We Keep Coding

Why does my for loop mess up all the parameters?

I am trying to parse some data from several web pages using javascript. I wrote a small parser for this purpose. The algorithm looks like this:
Open first URL from my .csv file
Find the data I need on the page
Save URL and data to a json file
My code executes 1. and 2. perfectly but sometimes messes up with number 3. Output looks like this:
URL 1 + data from URL 1 (correct line)
URL 2 + data from URL 2 (correct line)
URL 3 + data from URL 3 (correct line)
URL 4 + data from URL 4 (correct line)
URL 6(wrong URL) + data from another URL
URL 5(wrong URL) + data from another URL
URL 7 + data from URL 7 (correct line)
URL 8 + data from URL 8 (correct line)
URL 9 + data from URL 9 (correct line)
I assume the problem is that some pages load way too long which messes up the whole process. But I still don't understand why it sometimes saves the wrong data.
Heres my code:
var request = require('request');
var cheerio = require('cheerio');
var cloudscraper = require('cloudscraper');
var fs = require('fs');
var path = require('path');
var csvjson = require('csvjson');
//First, we read .csv file with our URL list
function getTheList() {
urlList = fs.readFileSync(path.join(__dirname, 'data.csv'), { encoding : 'utf8'});
var options = {
delimiter : ';', // optional
quote : '"' // optional
};
urlList = csvjson.toObject(urlList, options);
end = urlList.length;
logs = [];
//here we start the loop reading and saving data from each url
for (let p = 0; p < end; p += 1){
grabTheData(urlList, p)
}
}
//this code extracts the data from the page and saves it to a json file
function grabTheData(urlList, p){
setTimeout(function() {
url = url[p].ItemLink;
cloudscraper.get(url, function(err, res, body){
if (err) {
console.log(other.Time() + colors.yellow('Warn: ') + '- something went wrong with item ' + url);
callback();
} else {
var $ = cheerio.load(body);
/*
here are the lines which extract the data I need
dataIneed = ...;
*/
logs.push({
url, dataINeed
});
fs.writeFileSync('./logs.json', JSON.stringify(logs, null, 4));
}
});
//here I set a 2 seconds delay between each URL
}, 2000 * p);
}
getTheList()

The reason this is happening is that there is a potential mismatch between the callback result and the url variable in grabTheData.
Now there is a very quick fix for this, simple change the scope of the url variable like so:
function grabTheData(urlList, p){
setTimeout(function() {
// Set scope of url variable to block
let url = url[p].ItemLink;
cloudscraper.get(url, function(err, res, body){
if (err) {
console.log(other.Time() + colors.yellow('Warn: ') + '- something went wrong with item ' + url);
callback();
} else {
var $ = cheerio.load(body);
/*
here are the lines which extract the data I need
dataIneed = ...;
*/
logs.push({
url, dataINeed
});
fs.writeFileSync('./logs.json', JSON.stringify(logs, null, 4));
}
});
//here I set a 2 seconds delay between each URL
}, 2000 * p);
}
This should keep your results in order.
Here's another (IMHO much better) option, using promises and avoiding the use of setTimeout to separate calls. This should avoid any potential race condition, since the Promise.all call will preserve order:
async function getTheList() {
urlList = fs.readFileSync(path.join(__dirname, 'data.csv'), { encoding : 'utf8'});
var options = {
delimiter : ';', // optional
quote : '"' // optional
};
urlList = csvjson.toObject(urlList, options);
let promiseList = urlList.map(urlEntry => grabTheDataUpdated(urlEntry.ItemLink));
let logs = await Promise.all(promiseList);
fs.writeFileSync('./new_logs.json', JSON.stringify(logs, null, 4));
}
// Promisified version of cloudscraper.get
function getCloudScraperData(url) {
return new Promise((resolve, reject) => {
cloudscraper.get(url, (err, res, body) => {
if (err) {
reject(err);
} else {
resolve ( { url, res, body });
}
})
})
}
function getDataINeed(url, body) {
// Use cheerio to process data..
// Return mock data for now.. replace with actual data processed by cheerio..
return `data from ${url}`;
}
async function grabTheDataUpdated(url) {
try {
let result = await getCloudScraperData(url);
let dataINeed = getDataINeed(result.url, result.body);
return { url, dataINeed };
} catch (error) {
return { url, dataINeed: "Error occurred: " + error.message };
}
}

JavaScript: Appending a post request json string to existing json file

I have a simple express app that takes a post request with some JSON data. I'd like to take that data and append it to an existing json file (If it exists). Key value pairs may be different. My current version pushes an object to an array of objects. Ideally, I'd like to add just another key/value pair:
app.post('/notes', function(req, res){
var body = "";
req.on('data', function(chunk){
body += chunk;
});
req.on('end', function(){
fs.readFile(__dirname + '/data/notes.json', function(err, data){
if (err) throw err;
console.log(body);
var fileObj = JSON.parse(data.toString());
var postObj = JSON.parse(body);
fileObj.notes.push(postObj);
var returnjson = JSON.stringify(fileObj);
fs.writeFile(__dirname + '/data/notes.json', returnjson, function(err){
if (err) throw err;
res.send(returnjson);
});
});
});
});
Example of what might be in notes.json:
{"note": "Dear Diary: The authorities have removed the black pants from the couch"}
This works, but I'm having trouble wrapping my head around appending whatever json comes in the post (Let's assume there is no nested data in this case).
EDIT: Not the same as just appending to a file. Needs to append to an object within a file.

You can simply iterate through the post object with the help of for ... in loop, and add its properties to the file object. Keep in mind that in this case, if property keys are identical, their value will be overwritten. To avoid it you can make a verification with the help of Object.prototype.hasOwnProperty().
app.post('/notes', function(req, res){
var body = "";
req.on('data', function(chunk){
body += chunk;
});
req.on('end', function(){
fs.readFile(__dirname + '/data/notes.json', function(err, data){
if (err) throw err;
console.log(body);
var fileObj = JSON.parse(data.toString());
var postObj = JSON.parse(body);
for(var key in postObj) {
fileObj[key] = postObj[key];
}
var returnjson = JSON.stringify(fileObj);
fs.writeFile(__dirname + '/data/notes.json', returnjson, function(err){
if (err) throw err;
res.send(returnjson);
});
});
});
});
Here is for ... each statement, if you don't want to overwrite properties. New properties would be generated with suffixes like: _1, _2 etc. You can also use something like shortid to be sure that properties do not repeat, but it would be more ugly and less readable.
for(var key in postObj) {
if(fileObj.hasOwnProperty(key)) {
while(true) {
i++;
newKey = key + '_' + i;
if(fileObj.hasOwnProperty(newKey) == false) {
fileObj[newKey] = postObj[key];
break;
}
}
} else {
fileObj[key] = postObj[key];
}
}

Sending a file and some string value in single post request using angular controller

I am trying to upload an image to the server along with that I am sending a string value through $ upload post method. Some how I am able to process the requested file through the $upload method. I dont know how to access the string data that I send through this post request.Here is my controller code
$scope.string1="abcd";
createFile = $scope.files.slice(0);
console.log(createFile);
console.log("upload function");
if (createFile) { // console.log(file.name);
$scope.upload = $upload.upload({
url: '/uploadResortsImage', //upload.php script, node.js route, or servlet url
method: 'POST',
data: {
myObj: $scope.myModelObj,
$scope.dataone:$scope.string1,
},
file: createFile, // or list of files ($files) for html5 only
// fileName: this.name +'.pdf', //or ['1.jpg', '2.jpg', ...] // to modify the name of the file(s)
})
My appliaction stack is meanjs and Here goes my server code.
exports.upload = function (req, res) {
console.log("upload function");
console.log(req);
var filename = '',
target_path = '';
console.log(req.files);
var form = new formidable.IncomingForm();
form.parse(req, function (err, fields, files) {
console.log(files.file);
var tmp_path = files.file.path;
var tmp_path_string = String(tmp_path);
console.log(tmp_path_string);
filename = files.file.name;
var dir = "./public/uploads";
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir);
}
var dir1 = dir + '/' + resortid;
if (!fs.existsSync(dir1)) {
fs.mkdirSync(dir1);
}
var dest_path = dir1 + "/";
target_path = dest_path + filename;
console.log(dest_path);
console.log(filename);
console.log(target_path);
var target_path_string = String(target_path);
fs.rename(tmp_path_string, target_path_string, function (err) {
if (err) {
throw err;
console.log("fail");
console.log(err);
} else {
console.log("successful");
done(err, resort);
}
});
});
}

I think this what you need
data: {
myObj: $scope.myModelObj,
myObjB: $scope.string1,
},
Then on the server side, you would just access that myObjB property or whatever you decide to call it in the incoming form values collection depending on what your are using server side. Using $scope.dataone as a property name in your data object doesn't make sense as that's a scope property from elsewhere in your AngularJS code, stick to simple property names.

I was using formidabble plug in the node application.My code goes like this and I am able to access the send data objects in the server side.
form.parse(req, function (err, fields, files) {
var data_received= fields.myObjB;
}
Thanks.

Loading XML file into NodeJS as a JSON

I'm trying to load an XML file into my nodeJS project as a JSON and having some issued. The console.log(JSON.stringify(obj)) returns an undefined value.
var returnJSONResults = function(baseName, queryName) {
var XMLPath = "SomeFile.xml";
var rawJSON = loadXMLDoc(XMLPath);
function loadXMLDoc(filePath) {
var fs = require('fs');
var xml2js = require('xml2js');
var json;
try {
var fileData = fs.readFileSync(filePath, 'ascii');
var parser = new xml2js.Parser();
parser.parseString(fileData.substring(0, fileData.length), function (err, result) {
json = JSON.stringify(result);
console.log(JSON.stringify(result));
});
console.log("File '" + filePath + "/ was successfully read.\n");
return json;
} catch (ex) {...}
}
}();
I'm not sure what I'm doing wrong here but it is either unable to read the file (but doesn't return an error) or does't know how to stringify it.
Thnx.
UPDATE:
changed the parser.parseString and it is working now.
fs.readFile(__dirname + '/'+ filePath, function(err, data) {
parser.parseString(data, function (err, result) {
console.log(result);
console.log('XML converted to JSON');
tempJSON = result;
});
});

Take a look at the following runnable...
Copied your code. not many changes....
It works...
I suspect your XML has something wrong in it.... I created a simple valid XML...
If you add your XML there we can go further to see what's wrong.
Hope that helps.
Runnable link: Runnable Link Here
Shahar.

It seems in your code you doesn't have any variable "obj" . Form your code i think
console.log(JSON.stringify(json));
will work.

node.js + cheerio scrape: Passing an array of urls to download?

Firstly, here is my code as I've progressed so far:
var http = require("http");
// Utility function that downloads a URL and invokes
// callback with the data.
function download(url, callback) {
http.get(url, function(res) {
var data = "";
res.on('data', function (chunk) {
data += chunk;
});
res.on("end", function() {
callback(data);
});
}).on("error", function() {
callback(null);
});
}
var cheerio = require("cheerio");
var url = "http://www.bloglovin.com/en/blogs/1/2/all";
var myArray = [];
var a = 0;
var getLinks = function(){download(url, function(data) {
if (data) {
// console.log(data);
var $ = cheerio.load(data);
$(".content").each(function(i, e) {
var blogName = $(e).find(".blog-name").text();
var followLink = $(e).find("a").attr("href");
var blogSite = $(e).find(".description").text();
myArray[a] = [a];
myArray[a]["blogName"] = blogName;
myArray[a]["followLink"] = "http://www.bloglovin.com"+followLink;
myArray[a]["blogSite"] = blogSite;
a++;
console.log(myArray);
});
}
});
}
getLinks();
As you can see, followLinks is concatenated to followUrl, of which I'd like to pass through the 'url' download, so effectively I'll be scraping each of the pages using the same CSS rules, which will be added to the multidimensional array for the corresponding blogger.
How can I go about this?

I do something similar in one of my scraping jobs, but I use the async.js library to accomplish. Note that I'm also using the request module and cheerio.js in my scraping. I fetch and scrape rows of data from a single webpage, but suspect you could do something similar to fetch URLs and request / scrape them in the same manner.
I also admit this is quite basic coding, certainly could be optimized with a bit of refactoring. Hope it gives you some ideas at least...
First, I use request to fetch the page and call my parse function -
var url = 'http://www.target-website.com';
function(lastCallback) {
request(url, function(err, resp, body) {
if(!err) { parsePage(err, resp, body, lastCallback); }
else { console.log('web request error:' + resp.statusCode); }
}
}
Next, in my parsePage function, I load the website into Cheerio, fetch the HTML of each data row into an array, push my parseRow function and each HTML segment into another array, and use async.parallel to process each iteration -
var rows = [];
function parsePage(err, resp, body, callback1) {
var $ = cheerio.load(body);
$('div#targetTable tr').each(function(i, elem) {
rows.push($(this).html());
});
var scrRows = [];
rows.forEach(function(row) {
scrRows.push(function(callback2) {
parseRow(err, resp, row);
callback2();
});
async.parallel(scrRows, function() {
callback1();
});
}

Inside your loop, just create an object with the properties you scrape then push that object onto your array.
var blogInfo = {
blogName: blogName,
followLink: "http://www.bloglovin.com"+followLink;
blogSite: blogSite
};
myArray.push(blogInfo);

You have defined a = 0; So
myArray[a] = [a]; // => myArray[0] = [0]; myArray[0] becomes an array with 0 as only member in it
All these statements throw an error since Array can have only integer as keys.
myArray[a]["blogName"] = blogName;
myArray[a]["followLink"] = "http://www.bloglovin.com"+followLink;
myArray[a]["blogSite"] = blogSite;
Instead try this:
var obj = {
index: a,
blogName: blogName,
followLink: "http://www.bloglovin.com" + followLink,
blogSite: blogSite
}
myArray.push(obj);
console.log(myArray);