We Keep Coding

Why does my for loop mess up all the parameters?

I am trying to parse some data from several web pages using javascript. I wrote a small parser for this purpose. The algorithm looks like this:
Open first URL from my .csv file
Find the data I need on the page
Save URL and data to a json file
My code executes 1. and 2. perfectly but sometimes messes up with number 3. Output looks like this:
URL 1 + data from URL 1 (correct line)
URL 2 + data from URL 2 (correct line)
URL 3 + data from URL 3 (correct line)
URL 4 + data from URL 4 (correct line)
URL 6(wrong URL) + data from another URL
URL 5(wrong URL) + data from another URL
URL 7 + data from URL 7 (correct line)
URL 8 + data from URL 8 (correct line)
URL 9 + data from URL 9 (correct line)
I assume the problem is that some pages load way too long which messes up the whole process. But I still don't understand why it sometimes saves the wrong data.
Heres my code:
var request = require('request');
var cheerio = require('cheerio');
var cloudscraper = require('cloudscraper');
var fs = require('fs');
var path = require('path');
var csvjson = require('csvjson');
//First, we read .csv file with our URL list
function getTheList() {
urlList = fs.readFileSync(path.join(__dirname, 'data.csv'), { encoding : 'utf8'});
var options = {
delimiter : ';', // optional
quote : '"' // optional
};
urlList = csvjson.toObject(urlList, options);
end = urlList.length;
logs = [];
//here we start the loop reading and saving data from each url
for (let p = 0; p < end; p += 1){
grabTheData(urlList, p)
}
}
//this code extracts the data from the page and saves it to a json file
function grabTheData(urlList, p){
setTimeout(function() {
url = url[p].ItemLink;
cloudscraper.get(url, function(err, res, body){
if (err) {
console.log(other.Time() + colors.yellow('Warn: ') + '- something went wrong with item ' + url);
callback();
} else {
var $ = cheerio.load(body);
/*
here are the lines which extract the data I need
dataIneed = ...;
*/
logs.push({
url, dataINeed
});
fs.writeFileSync('./logs.json', JSON.stringify(logs, null, 4));
}
});
//here I set a 2 seconds delay between each URL
}, 2000 * p);
}
getTheList()

The reason this is happening is that there is a potential mismatch between the callback result and the url variable in grabTheData.
Now there is a very quick fix for this, simple change the scope of the url variable like so:
function grabTheData(urlList, p){
setTimeout(function() {
// Set scope of url variable to block
let url = url[p].ItemLink;
cloudscraper.get(url, function(err, res, body){
if (err) {
console.log(other.Time() + colors.yellow('Warn: ') + '- something went wrong with item ' + url);
callback();
} else {
var $ = cheerio.load(body);
/*
here are the lines which extract the data I need
dataIneed = ...;
*/
logs.push({
url, dataINeed
});
fs.writeFileSync('./logs.json', JSON.stringify(logs, null, 4));
}
});
//here I set a 2 seconds delay between each URL
}, 2000 * p);
}
This should keep your results in order.
Here's another (IMHO much better) option, using promises and avoiding the use of setTimeout to separate calls. This should avoid any potential race condition, since the Promise.all call will preserve order:
async function getTheList() {
urlList = fs.readFileSync(path.join(__dirname, 'data.csv'), { encoding : 'utf8'});
var options = {
delimiter : ';', // optional
quote : '"' // optional
};
urlList = csvjson.toObject(urlList, options);
let promiseList = urlList.map(urlEntry => grabTheDataUpdated(urlEntry.ItemLink));
let logs = await Promise.all(promiseList);
fs.writeFileSync('./new_logs.json', JSON.stringify(logs, null, 4));
}
// Promisified version of cloudscraper.get
function getCloudScraperData(url) {
return new Promise((resolve, reject) => {
cloudscraper.get(url, (err, res, body) => {
if (err) {
reject(err);
} else {
resolve ( { url, res, body });
}
})
})
}
function getDataINeed(url, body) {
// Use cheerio to process data..
// Return mock data for now.. replace with actual data processed by cheerio..
return `data from ${url}`;
}
async function grabTheDataUpdated(url) {
try {
let result = await getCloudScraperData(url);
let dataINeed = getDataINeed(result.url, result.body);
return { url, dataINeed };
} catch (error) {
return { url, dataINeed: "Error occurred: " + error.message };
}
}

Trying to reach JSON outside function, returns TypeError

I am trying to get JSON number outside function using Node.js.
Firstly, JSON is imported from URL:
var request = require('request');
var importedJSON;
request('http://private-38e18c-uzduotis.apiary-mock.com/config/cash-in', function (error, response, body) {
if (!error && response.statusCode == 200) {
importedJSON = JSON.parse(body);
}
processInputData();
}
);
Than object is added into variable :
function processInputData() {
var X ;
X = importedJSON.percents;
//console.log(X);
return XX;
}
After all, console.log() shows value, but when I want to make variable global :
var showX = processInputData();
console.log(showX);
I get stuck into wild error:
TypeError: Cannot read property 'percents' of undefined
I am lost, if anyone knows where I am missing the point. Many thanks for all possible help.
Looking forward,

Javascript multiple promises not working with q and request middle

So this is my dilemma. I have a list of movies, witch I have scraped from a website, then I want to add additional properties to my newly constructed object(json)
Now the omdi api witch I am using supports searching for a movie by title.
Then I make a get request using request and q middlewares. When I receive information from omdb api in the call back I add that data to the object.
Now the next part is where my problem lies. Now I want to return a new Request using data from the previous request. Now I make an new get Request and return it but then() func isin't returning anything. But I don't seem to realize what I am doing wrong.
Here is my code..
var promises = [];
films.forEach(function (film) {
// Get omdbapi information
promises.push(HttpService.getContent(configExternal.omodburl + '?t=' + film.title.trim() + '&y=' + film.year + '&plot=true&tomatoes=true&r=json').then(function (data) {
var result = JSON.parse(data);
if(Boolean(result.Response) === true) {
film.omdb.push(result);
}
var imdbid = result.imdbID;
return HttpService.getContent(configExternal.themoviedburl + imdbid + '/videos?api_key=' + configExternal.themoviedbkey);
}).then(function(data) {
film.trailers = [];
film.trailers.push(JSON.parse(data));
}).catch(function (err) {
logger.error().info('Error getting ' + film.title + ' from omdb, ErrorMessage : ' + err);
}));
});
//--------------------------------
// When all promises have finished
//--------------------------------
Promise.all(promises).then(function (data, err) {
// do stuff with the data
});
And here is my getContent func
var Service = {
getContent: function(url) {
var deferred = q.defer();
request(url, function (error, response, body) {
if (!error && response.statusCode == 200) {
deferred.resolve(body);
} else {
deferred.reject(error);
}
});
return deferred.promise;
}
};

Problem solved. There wasn't anything wrong with the request as Roamer said. But the moviedata base limits by 40 request per 10 sek witch I didn't know :)

Uncaught TypeError: undefined is not a function for request() in Titanium

I have a problem with 1 of my functions.
in app.js ia have th following:
//Including all functions
Ti.include('functions.js');
//Including the login screen
Ti.include('login.js');
//Including the register screen
Ti.include('register.js');
So all the functions are above the other files that could call a function.
In login.js I have the following code:
//'login' is the type
var request = request('login', loginUsernameInput.value, md5(loginPasswordInput.value));
Ti.API.info('request: ' + request);
if(request == true) {
alert('You are loggedin');
} else {
alert('Something went wrong');
}
The request function looks like this:
function request(type, username, password) {
//Database connection
var db = Ti.Network.createHTTPClient();
db.open('POST', 'http://myip/mobile_app/');
Ti.API.info('type: ' + type);
Ti.API.info('username: ' + username);
Ti.API.info('password: ' + password);
//If variables has been send
db.onload = function() {
var answer = this.responseText;
Ti.API.info('type answer: ' + typeof this.responseText);
if(answer == 'true') {
Ti.API.info('TEST');
return true;
} else {
return false;
}
};
//Variables to send
db.send({
type: type,
username: username,
password: md5(password)
});
//If there is an error
db.onerror = function(e) {
Ti.API.info('error: ' + JSON.stringify(e));
};
}
I know that this.responseText returns true and that the function md5() works aswell.
I know this because I also tested login.js when the code is like:
Ti.API.info('request: ' + request('login', loginUsernameInput.value, md5(loginPasswordInput.value)));
if(request('login', loginUsernameInput.value, md5(loginPasswordInput.value)) == true) {
alert('You are loggedin');
} else {
alert('Something went wrong');
}
The above also returns that function request() is undefined
So as soon as try to login I get the following error:
So my question is how can resolve the error?

I think the issue is that you redefine variable request to the return value of function request. You request function does not return anything. Therefore it returns undefined.
Now that you set request to undefined. You cannot use request() anymore, because you overwrote it.
Attempting to call an undefined value as a function, results in undefined is not a function
See demonstration: http://repl.it/UXE/1

Indeed, #Gabs00 is right when he says that you're redefining your request variable.
But the real question is how could you have avoid this problem ?
And the answer is in the way you're coding. By using Ti.include, you're not using the best practices recommended by Titanium.
Instead, you should do something like this :
In a file models/User.js :
exports.request = function(type, username, password) {
// Your code here
};
Then, instead of writing (in your app.js file) :
Ti.include('functions.js');
var request = request('login', loginUsernameInput.value, md5(loginPasswordInput.value));
You'll have :
var User = require('models/User');
User.request('login', loginUsernameInput.value, md5(loginPasswordInput.value));
But even with this code, it's sot satisfying enough...
What you should probably try is to declare as many functions as your requests. Even if your code is common for your 2 requests for now, I can assure you that it won't be that way very long.
Thus, there is the final code I recommend :
In your models/User.js file :
function request(type, username, password) {
// Your code
};
exports.login = function(username, password) {
request('login', username, password);
};
exports.register = function(username, password) {
request('register', username, password);
};
The cool thing with this code is that your request method is totally protected: the only code which can access this function is the one written in your models/User.js file. For the others, it's like this function doesn't even exist.
In you app.js file :
var User = require('models/User');
User.login(loginUsernameInput.value, md5(loginPasswordInput.value));
I think it's more readable this way, don't you?
If you want more information about this pattern, you should probably read these articles:
CommonJS modules
Titanium best practices
As you'll see, you should reuse this pattern for your windows too (your Ti.include('login.js'); and Ti.include('register.js');)

This question got answered here: http://developer.appcelerator.com/question/175412/function-not-returning-anything#280011
Edit:
In User.js:
function request(type, username, password, callback) {
db.onload = function() {
Ti.API.info('type antwoord: ' + typeof this.responseText);
callback( this.responseText );
};
}
exports.login = function(username, password, callback) {
request('login', username, password, callback);
};
and in login.js:
// last parameter is the callback function
User.login(loginUsernameInput.value, md5(loginPasswordInput.value), function(response){
Ti.API.info('login response: ' + response)
});
now i get true or false as response.

NodeJS unable to make a GET request asynchronously

I am a rookie in Nodejs and asynchronous programming. I am having a problem executing a GET request inside an asynchronous function. Here I am posting the whole code. I am trying to pull a list of all Urls , add them to a list and send the list for processing to another function.
My problem is with processing them. Inturn for each url I am executing a GET request to fetch the body and to look for image elements in it. I am looking to pass the Image url to a 3rd party api as a GET param. I am unable to execute the GET request as the control doesn't seem to reach there at all.
var async = require("async"),
request = require("request"),
cheerio = require("cheerio");
async.waterfall([
function(callback) {
var url = "someSourceUrl";
var linkList = [];
request(url, function(err, resp, body) {
var $ = cheerio.load(body);
$('.list_more li').each(function() {
//Find all urls and add them to a list
$(this).find('a').each(function() {
linkList.push($(this).attr('href'));
});
});
callback(null, linkList);
});
},
//pass all the links as a list to callback
function(liksListFetched, callback) {
for (var i in liksListFetched) {
callback(null, liksListFetched[i]);
}
}],
//***********My problem is with the below code**************
function(err, curUrl) {
var cuResp = "";
console.log("Currently Processing Url : " + curUrl);
request(curUrl, function(err, resp, body) {
var $ = cheerio.load(body);
var article = $("article");
var articleImage = article.find("figure").children('img').attr('src');
var responseGrabbed = "API response : ";
//check if there is an IMG element
if (articleImage === undefined) {
console.log("No Image Found.");
articleImage = 'none';
}
else {
//if there is an img element, pass this image url to an API,
//So do a GET call by passing imageUrl to the API as a GET param
request("http://apiurl.tld?imageurl=" + articleImage, function(error, response, resp) { //code doesn't seem to reach here
I would like to grab the response and concatenate it to the responseGrabbed var.
console.log(resp);
responseGrabbed += resp;
});
}
console.log(responseGrabbed);// api response never gets concatenated :(
console.log("_=_=_=_=_=_=__=_=_=_=_=_=__=_=_=_=_=_=__=_=_=_=_=_=_");
process.exit(0);
});
});
I appreciate if any one can help me understand the root cause. Thanks in advance.

request() is asynchronous, so when you're console logging the string, the string hasn't been built yet, you have to do the console log inside the callback :
request("http://apiurl.tld?imageurl=" + articleImage, function(error, response, resp) {
responseGrabbed += resp;
console.log(responseGrabbed);// api response never gets concatenated :(
console.log("_=_=_=_=_=_=__=_=_=_=_=_=__=_=_=_=_=_=__=_=_=_=_=_=_");
});
Same goes for terminating the process, which should be done when all the requests have finished