We Keep Coding

Why does my for loop mess up all the parameters?

I am trying to parse some data from several web pages using javascript. I wrote a small parser for this purpose. The algorithm looks like this:
Open first URL from my .csv file
Find the data I need on the page
Save URL and data to a json file
My code executes 1. and 2. perfectly but sometimes messes up with number 3. Output looks like this:
URL 1 + data from URL 1 (correct line)
URL 2 + data from URL 2 (correct line)
URL 3 + data from URL 3 (correct line)
URL 4 + data from URL 4 (correct line)
URL 6(wrong URL) + data from another URL
URL 5(wrong URL) + data from another URL
URL 7 + data from URL 7 (correct line)
URL 8 + data from URL 8 (correct line)
URL 9 + data from URL 9 (correct line)
I assume the problem is that some pages load way too long which messes up the whole process. But I still don't understand why it sometimes saves the wrong data.
Heres my code:
var request = require('request');
var cheerio = require('cheerio');
var cloudscraper = require('cloudscraper');
var fs = require('fs');
var path = require('path');
var csvjson = require('csvjson');
//First, we read .csv file with our URL list
function getTheList() {
urlList = fs.readFileSync(path.join(__dirname, 'data.csv'), { encoding : 'utf8'});
var options = {
delimiter : ';', // optional
quote : '"' // optional
};
urlList = csvjson.toObject(urlList, options);
end = urlList.length;
logs = [];
//here we start the loop reading and saving data from each url
for (let p = 0; p < end; p += 1){
grabTheData(urlList, p)
}
}
//this code extracts the data from the page and saves it to a json file
function grabTheData(urlList, p){
setTimeout(function() {
url = url[p].ItemLink;
cloudscraper.get(url, function(err, res, body){
if (err) {
console.log(other.Time() + colors.yellow('Warn: ') + '- something went wrong with item ' + url);
callback();
} else {
var $ = cheerio.load(body);
/*
here are the lines which extract the data I need
dataIneed = ...;
*/
logs.push({
url, dataINeed
});
fs.writeFileSync('./logs.json', JSON.stringify(logs, null, 4));
}
});
//here I set a 2 seconds delay between each URL
}, 2000 * p);
}
getTheList()

The reason this is happening is that there is a potential mismatch between the callback result and the url variable in grabTheData.
Now there is a very quick fix for this, simple change the scope of the url variable like so:
function grabTheData(urlList, p){
setTimeout(function() {
// Set scope of url variable to block
let url = url[p].ItemLink;
cloudscraper.get(url, function(err, res, body){
if (err) {
console.log(other.Time() + colors.yellow('Warn: ') + '- something went wrong with item ' + url);
callback();
} else {
var $ = cheerio.load(body);
/*
here are the lines which extract the data I need
dataIneed = ...;
*/
logs.push({
url, dataINeed
});
fs.writeFileSync('./logs.json', JSON.stringify(logs, null, 4));
}
});
//here I set a 2 seconds delay between each URL
}, 2000 * p);
}
This should keep your results in order.
Here's another (IMHO much better) option, using promises and avoiding the use of setTimeout to separate calls. This should avoid any potential race condition, since the Promise.all call will preserve order:
async function getTheList() {
urlList = fs.readFileSync(path.join(__dirname, 'data.csv'), { encoding : 'utf8'});
var options = {
delimiter : ';', // optional
quote : '"' // optional
};
urlList = csvjson.toObject(urlList, options);
let promiseList = urlList.map(urlEntry => grabTheDataUpdated(urlEntry.ItemLink));
let logs = await Promise.all(promiseList);
fs.writeFileSync('./new_logs.json', JSON.stringify(logs, null, 4));
}
// Promisified version of cloudscraper.get
function getCloudScraperData(url) {
return new Promise((resolve, reject) => {
cloudscraper.get(url, (err, res, body) => {
if (err) {
reject(err);
} else {
resolve ( { url, res, body });
}
})
})
}
function getDataINeed(url, body) {
// Use cheerio to process data..
// Return mock data for now.. replace with actual data processed by cheerio..
return `data from ${url}`;
}
async function grabTheDataUpdated(url) {
try {
let result = await getCloudScraperData(url);
let dataINeed = getDataINeed(result.url, result.body);
return { url, dataINeed };
} catch (error) {
return { url, dataINeed: "Error occurred: " + error.message };
}
}

Javascript multiple promises not working with q and request middle

So this is my dilemma. I have a list of movies, witch I have scraped from a website, then I want to add additional properties to my newly constructed object(json)
Now the omdi api witch I am using supports searching for a movie by title.
Then I make a get request using request and q middlewares. When I receive information from omdb api in the call back I add that data to the object.
Now the next part is where my problem lies. Now I want to return a new Request using data from the previous request. Now I make an new get Request and return it but then() func isin't returning anything. But I don't seem to realize what I am doing wrong.
Here is my code..
var promises = [];
films.forEach(function (film) {
// Get omdbapi information
promises.push(HttpService.getContent(configExternal.omodburl + '?t=' + film.title.trim() + '&y=' + film.year + '&plot=true&tomatoes=true&r=json').then(function (data) {
var result = JSON.parse(data);
if(Boolean(result.Response) === true) {
film.omdb.push(result);
}
var imdbid = result.imdbID;
return HttpService.getContent(configExternal.themoviedburl + imdbid + '/videos?api_key=' + configExternal.themoviedbkey);
}).then(function(data) {
film.trailers = [];
film.trailers.push(JSON.parse(data));
}).catch(function (err) {
logger.error().info('Error getting ' + film.title + ' from omdb, ErrorMessage : ' + err);
}));
});
//--------------------------------
// When all promises have finished
//--------------------------------
Promise.all(promises).then(function (data, err) {
// do stuff with the data
});
And here is my getContent func
var Service = {
getContent: function(url) {
var deferred = q.defer();
request(url, function (error, response, body) {
if (!error && response.statusCode == 200) {
deferred.resolve(body);
} else {
deferred.reject(error);
}
});
return deferred.promise;
}
};

Problem solved. There wasn't anything wrong with the request as Roamer said. But the moviedata base limits by 40 request per 10 sek witch I didn't know :)

How to chain two ajax requests with promises

I am having trouble with ajax/promises. I have two ajax requests total, with the second ajax call relying data to be returned by the first ajax call.
My first ajax call finds Latitude, Longitude, and country code of the value of #search.
My second ajax call finds the weather of that city, but the API URL is dependent on the Latitude, Longitude and country code that my first ajax call returns. So the second ajax call can't be started until the first one is finished.
My logic here is that var ajax1 is assigned a promise, and var ajax2 starts after ajax1.then() checks that ajax1's promise is resolved. Then ajax2 runs and returns another promise. Finally ajax2.done starts after it checks that ajax2's promise is resolved, and then starting my successWeatherFunction().
My problem is that ajax2.done is not working, as the console.log("test") is not showing up on the console. The two earlier console.logs, console.log(info) and console.log(weatherApiUrl) are working.
Thanks!
$("#search").keypress(function(event) {
if (event.which === 13) {
var searchCity = $("#search").val();
var jsonURL = "http://autocomplete.wunderground.com/aq?query=" + searchCity + "&cb=?"
var ajax1 = $.getJSON(jsonURL);
var ajax2 = ajax1.then(function(data) {
var info = [];
info.push(data["RESULTS"][0]["name"]);
info.push(data["RESULTS"][0]["c"]);
info.push(data["RESULTS"][0]["lat"]);
info.push(data["RESULTS"][0]["lon"]);
console.log(info);
var searchLat = info[2];
var searchLng = info[3];
var countryCode = info[1];
if (countryCode === "US") {
var weatherApiUrl = "https://api.forecast.io/forecast/{APIKEY}/" + searchLat + "," + searchLng + "?exclude=minutely" + "&callback=?";
} else {
var weatherApiUrl = "https://api.forecast.io/forecast/{APIKEY}/" + searchLat + "," + searchLng + "?exclude=minutely" + "?units=si" + "&callback=?";
console.log(weatherApiUrl);
}
return $.getJSON(weatherApiUrl);
});
ajax2.done(function(data){
console.log("test");
successCityWeather(data);
});

Your code use then and done. done is the old promises jQuery syntax so you should use only then.
The following code works for me :
$(function() {
$.get('/test').then(function() {
console.log('First request end');
return $.get('/test');
}).then(function() {
console.log('second request end');
});
});
But in your case, maybe a one of your request fail. Give a second parameter to then to log the error :
$.getJSON('...').then(function(data) {
console.log('success', data);
}, function(data) {
console.log('fail', data);
});

If not sure, always use always() handler. That way you will know if the request actually finished with error or not at all.
$.ajax( ...params... )
.always(function(jqXHR, textStatus) {
if (textStatus != "success") {
alert("Error: " + jqXHR.statusText); //error is always called .statusText
} else {
alert("Success: " + jqXHR.response); //might not always be named .response
}});

$.post(jsonURL)
.then(function (data) {
var info = [];
// some actions
return $.getJSON(weatherApiUrl);
})
.then(function(data, status, promise) {
// some actions
successCityWeather(data);
})

Parse cloud code loop through url and httpRequest

I just want to do a simple loop in my "alerts" objects, which contains an url, and a word.
For each alert, I do a httpRequest to check if the word is present in the response html code. I yes, I put the status to true.
I also want to update each time the "updatedTo" column, even if I don't find the word in the response html code, but I don't know why...
I wrote this cloud code, but it don't works, or it works sometimes only if I have only items with the word present.
Parse.Cloud.job("updateStatus", function(request, status) {
Parse.Cloud.useMasterKey();
var counter = 0;
var AlertItem = Parse.Object.extend("Alert");
var query = new Parse.Query(AlertItem);
query.each(function(alert) {
var alertTitle = alert.get("title");
var alertUrl = alert.get("url");
var alertStatus = alert.get("status");
var alertWords = alert.get("research");
console.log("Alert : " + alertTitle + " - Check if : " + alertWords + " is on : " + alertUrl)
promise = promise.then(function() {
return Parse.Cloud.httpRequest({
url: alertUrl,
headers: {
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25'
},
}).then(function(httpResponse) {
console.log("We succeded to access to the website");
var htmlCode = httpResponse.text;
if (htmlCode.indexOf(alertWords) >= 0) {
if (alertStatus == false) {
alert.set("status", true);
console.log("new status:true");
return alert.save();
}
} else {
alert.set("status", false);
console.log("new status:false");
//I do this to updated the "updatedTo" field, but it doesn't work
return alert.save();
}
// You need to return a Promise here if non of the above condition meet.
},
function(error) {
console.error('Request failed with response code ' + httpResponse.headers.Location);
// You need to return a rejected promise here.
}
});
});
return promise;
}).then(function() {
status.success('Status updated');
// Set the job's success status
}, function(error) {
// Set the job's error status
status.error("Uh oh, something went wrong.");
});
});

The query.each(callback, options) from documentation.
Iterates over each result of a query, calling a callback for each one. If the callback returns a promise, the iteration will not continue until that promise has been fulfilled. If the callback returns a rejected promise, then iteration will stop with that error. The items are processed in an unspecified order. The query may not have any sort order, and may not use limit or skip.
Parse.Cloud.job("updateStatus", function(request, status) {
Parse.Cloud.useMasterKey();
var counter = 0;
var AlertItem = Parse.Object.extend("Alert");
var query = new Parse.Query(AlertItem);
query.each(function(alert) {
var alertTitle = alert.get("title");
var alertUrl = alert.get("url");
var alertStatus = alert.get("status");
var alertWords = alert.get("research");
console.log("Alert : " + alertTitle + " - Check if : " + alertWords + " is on : " + alertUrl)
return Parse.Cloud.httpRequest({
url: alertUrl,
headers: {
'user-agent': 'A user classic agent'
},
success: function(httpResponse) {
console.log("We succeded to access to the website");
var htmlCode = httpResponse.text;
if (htmlCode.indexOf(alertWords) >= 0) {
if (alertStatus == false) {
alert.set("status", true);
console.log("new status:true");
return alert.save();
}
} else {
alert.set("status", false);
console.log("new status:false");
//I do this to updated the "updatedTo" field, but it doesn't work
return alert.save();
}
// You need to return a Promise here if non of the above condition meet.
},
error: function(httpResponse) {
console.error('Request failed with response code ' + httpResponse.headers.Location);
// You need to return a rejected promise here.
}
});
}).then(function() {
status.success('Status updated');
// Set the job's success status
}, function(error) {
// Set the job's error status
status.error("Uh oh, something went wrong.");
});
});

So, with any help it was difficult, but I finish to find another post who was close to what I need, I adapt it, and I success to use it, it works great with Promises :) :
var _ = require('underscore.js')
Parse.Cloud.job("updateStatus", function(request, response) {
var alerts = Parse.Object.extend("Alert");
var query = new Parse.Query(alerts);
query.equalTo("status", false);
query.find().then(function(alerts) {
var promise = Parse.Promise.as();
_.each(alerts, function(alert) {
var alertUrl = alert.get("url");
...
promise = promise.then(function() {
return Parse.Cloud.httpRequest({
url: alertUrl
}).then(function(httpResponse) {
...
},
function(error) {
...
});
});
});
return promise;
}).then(function() {
response.success("All status updated with success !");
},
function (error) {
response.error("Error: " + error.code + " " + error.message);
});
});

nodejs async nested calls

I would like to scrap an url:
1 request to get a list of elements
1 request on each result to get details
Here what I have:
var request = require('request')
, cheerio = require('cheerio')
, async = require('async')
, format = require('util').format;
var baseurl = 'http://magiccards.info';
async.waterfall([
function (callback) {
request(baseurl + '/sitemap.html', function (err, response, body) {
var sets = [];
var $ = cheerio.load(body);
$('a[href$="/en.html"]').each(function () {
sets.push({"name": $(this).text(), "code":$(this).attr('href').match(/\/([^)]+)\//)[1], "path": $(this).attr('href'), "translations":[]});
});
callback(null, sets);
});
},
function (sets, callback) {
console.log(sets);
async.eachSeries(sets, function (set, callback) {
console.log('SET ' + set.code.toUpperCase());
request(baseurl + set.path, function (err, response, body) {
var $ = cheerio.load(body);
$('body > a[href^="/' + set.code + '/"]').each(function () {
console.log(' %s (%s)', $(this).text(), $(this).attr('href'));
});
});
});
}
], function (err, result) {
console.log('ERR');
// result now equals 'done'
});
The problem is that the 2nd waterfall function run only once, if I replace the eachSeries with an each, the loop does run X times (but I need to wait for result).
Wath am I missing?

You need to call the eachSeries callback function. Otherwise async won't know that you are done. (1)
You also need to tell the waterfall function that you are done with that step, also by calling the callback function. (2)
function (sets, waterfallCallback) {
async.eachSeries(sets, function (set, seriesCallback) {
console.log('SET ' + set.code.toUpperCase());
request(baseurl + set.path, function (err, response, body) {
var $ = cheerio.load(body);
$('body > a[href^="/' + set.code + '/"]').each(function () {
console.log(' %s (%s)', $(this).text(), $(this).attr('href'));
});
seriesCallback(null); /* 1 */
});
}, waterfallCallback /* 2 */);
}