We Keep Coding

Call a Request function from outside the request

Im trying to make a webscraper(educational puposes), and I got really far, but this little issue is bugging me.
I made a request callback function, and im trying to get lines 75-78 to work. However to get this to work, I need PDF_LISTS and PDF_LINKS to initilaze to the right values.
I've already tried to make them global variables, and what not, for some reason that doesnt work. So my question is: How do I make a callback function that will call that for loop (75-78) and succesfully initilaze PDF_LISTS and PDF_LINKS to the correct values ?
(Dont worry I use this on educational content, with the prof's permission). First time posting here!
// URL_LINKS has the pdf links of the pages
PDF_LINKS = [];
// URL_LIST has the names of the pdf links
PDF_LIST = [];
function fillPDF(callback) {
request(url, function(err, res, body) {
$ = cheerio.load(body);
links = $('a'); //jquery get all hyperlinks
$(links).each(function(i, link) {
var value = $(link).attr('href');
// creates objects to hold the file
if (value.substring(value.length - 3, value.length) == "pdf") {
PDF_LINKS[i] = $(link).attr('href');
PDF_LIST[i] = $(link).text();
}
})
});
}
// must decleare fillPDF variable or else you wont initilze teh variables
fillPDF() {
//HERE I WANT PDF_LINKS and PDF_LIST to be intialized to 33.....
}
for (j = 0; j < PDF_LIST.length; j++) {
request(PDF_LINKS[j]).pipe(fs.createWriteStream(PDF_LIST[j]));
}

You may push your values into arrays using array's push method, avoiding array's element to be undefined.
You can put your final for loop into a function, and then use fillPDF();
You also need to call fillPDF's callback once the request is over.
PDF_LINKS = [];
PDF_LIST = [];
function fillPDF(callback) {
request(url, function(err, res, body) {
$ = cheerio.load(body);
links = $('a');
$(links).each(function(i, link) {
var value = $(link).attr('href');
if (value.slice(-3) == "pdf") {
PDF_LINKS.push(value);
PDF_LIST.push($(link).text());
}
})
callback();
});
}
function writePDF() {
for (j = 0; j < PDF_LIST.length; j++) {
request(PDF_LINKS[j]).pipe(fs.createWriteStream(PDF_LIST[j]));
}
}
fillPDF(writePDF);

How can I check if two array contains the same value, even if they are in different indexes?

I'm working on a personal project and I'm stump in the last part. I'm trying to only get the data for companies that have the same name of the companies in my database.
My goal at the end is to merge the two jsons obtained from the two following calls into one
Call one: $http.get('//localhost:8081/api/jobs').then(function(res)
Call two: localhost:8081/api/glassdoor/
Full Code:
$http.get('//localhost:8081/api/jobs').then(function(res) {
$scope.data = res.data; //data from the database
$scope.size = $scope.data.length; //length 132
for (var i = 0; i < $scope.size; i++) {
if ($scope.data[i].glassdoor !== null && $scope.data[i].glassdoor !== undefined) {
$scope.hasGlassdoor = [];
for (var i = 0; i < $scope.size; i++) {
if ($scope.data[i].glassdoor !== null && $scope.data[i].glassdoor !== undefined)
$scope.hasGlassdoor.push($scope.data[i]);
}
//Get the companies name that have glassdoor
$scope.company = [];
for (var j = 0; j < $scope.hasGlassdoor.length; j++) {
$scope.company.push($scope.hasGlassdoor[j].company);
}
//Create the URL calls for my glassdoor api
$scope.url = [];
for (var x = 0; x < $scope.company.length; x++) {
$scope.url.push('//localhost:8081/api/glassdoor/' + $scope.company[x]);
}
//For example : '//localhost:8081/api/glassdoor/A9.com'
//Get the Glassdoor data
var company = $scope.company;
for (var j = 0; j < $scope.url.length; j++) {
$http.get($scope.url[j]).then(function(response) {
$scope.gData = response.data;
$scope.gSize = $scope.gData.length;
$scope.gName = [];
//Get the names of the companies that glassdoor returns
for(var x = 0; x < $scope.gSize; x++){
if ($scope.gData[x] !== null && $scope.gData[x] !== undefined) {
if ($scope.gData[x].name !== null && $scope.gData[x].name !== undefined) {
$scope.gName.push($scope.gData[x].name);
}
}
}
//Now I'm trying to only get the names of the companies that are in my variable company
//For example '//localhost:8081/api/glassdoor/6sense
//returns data for 6sense, and a company named 6sense Technologies
//I'm trying to just get the data of 6sense
//
// TODO
//
// My try using loDash returns undefined.
// I'm trying to see if $scope.gName is in var Company.
//if(_.includes(company, $scope.gName)){
// gd.push($scope.gName);
//}
}); //this is what is calling the glassdoor api
}//end of the for loop for url.
} //if statement to check for null
} //first for loop
}).catch(function(error, res) {
console.log("Error:", error, res);
});
Right now I'm working on small list so I can fix this issue.
My goal at the end is to have this as my finished json:
[
{
"company": "23andMe"
"glassdoor":"https://www.glassdoor.com/Overview/Working-at-23andMe-EI_IE145899.11,18.htm"
"img":"https://www.23andme.com/static/img/icons/logo_alt.96cf7888b73d.svg"
"international":null
"link":"https://www.23andme.com/careers/"
"location":"Mountain View, CA"
"secondary":null
"third":null
"id":145899,
"name":"23andMe",
"website":"www.23andme.com",
"isEEP":true,
"exactMatch":true,
"industry":"Biotech & Pharmaceuticals",
"numberOfRatings":27,
"squareLogo":"https://media.glassdoor.com/sqll/145899/23andme-squarelogo.png",
"overallRating":"4.2",
"ratingDescription":"Very Satisfied",
"cultureAndValuesRating":"4.5",
"seniorLeadershipRating":"3.6",
"compensationAndBenefitsRating":"4.0",
"careerOpportunitiesRating":"3.4",
"workLifeBalanceRating":"4.3",
"recommendToFriendRating":80,
"sectorId":10005,
"sectorName":"Biotech & Pharmaceuticals",
"industryId":200021,
"industryName":"Biotech & Pharmaceuticals",
"featuredReview":{
"attributionURL":"https://www.glassdoor.com/Reviews/Employee-Review-23andMe-RVW11447587.htm",
"id":11447587,
"currentJob":true,
"reviewDateTime":"2016-08-03 15:05:20.157",
"jobTitle":"Customer Care Reporesentative",
"location":"Mountain View, CA",
"jobTitleFromDb":"Customer Care Reporesentative",
"headline":"Customer Care Representative",
"pros":"the environment-everyone is extremely genuine, smart, and friendly. management is very understanding and open. Executives are transparent with everything going on in the company\r\nbenefits-free gym, food every day, snacks, great health coverage, rooftop access, etc\r\nworkspace-facilities does a phenomenal job at keeping everything extremely clean and fixes all issues ASAP. I don't feel like I'm sitting a boring desk job all day, it's a fun place to be",
"cons":"Traffic through downtown mountain view can suck and the train can be kind of loud (I cannot think of a legitimate con, everything is awesome here)",
"overall":5,
"overallNumeric":5
},
"ceo":{
"name":"Anne Wojcicki",
"title":"CEO",
"numberOfRatings":15,
"pctApprove":100,
"pctDisapprove":0
}
}
]
Server.js that deals with the glassdoor call:
//Glassdoor api call
app.get('/api/glassdoor/:company', function(req, res, next) {
var company = req.params.company;
requestify.get('https://api.glassdoor.com/api/api.htm?t.p=PRODUCT&t.k=KEY&userip=0.0.0.0&useragent=&format=json&v=1&action=employers&q=' + company).then(function(response) {
// Get the response body (JSON parsed or jQuery object for XMLs)
gData = response.getBody();
gData = gData.response.employers;
res.json(gData);
});
});

Sort the array, then enumerate it. If previous === current you have a dupe.

This is how I solved it.
app.get('/api/glassdoor/:company', function(req, res, next) {
var company = req.params.company;
requestify.get('https://api.glassdoor.com/api/api.htm?t.p=PRODUCT&t.k=KEY&userip=0.0.0.0&useragent=&format=json&v=1&action=employers&q=' + company).then(function(response) {
// Get the response body (JSON parsed or jQuery object for XMLs)
gData = response.getBody();
gData = gData.response.employers;
//What I added to only send the data of the companies that where like 'company'
for (var i = 0; i < gData.length; i++) {
if (gData[i].name === company) {
gData = gData[i];
res.send(gData);
}
}
});
});
Which doesn't answer the overall question but that is how I solved my particular issue.

Async loading files with Angular into an object array and keeping their order

I'm not loading scripts, I'm reading XML files into an array of objects.
My first [dumb] attempt was something like:
for (var i = 1; i < n; i++) {
var filePath = "xml/chapter_"+i+".xml";
$http.get( filePath ).success(function (data) {
$scope.chaptersData.push (data._chapter);
});
}
I quickly figured out this was no good, because the array will be filled in the order the files finished loading, not when they started (a race condition) and the smaller ones will finish first. I can't use the value of 'i' because of course it gets to 'n' long before any of the loading finishes.
After that I thought success(function (data, i) { would work, but it didn't. I'm out of simple ideas, and before I come up with some kind of Rube Goldberg kludge I thought I would ask the wisdom of the Internets if there is a 'right' way to do this.

You can pass i into a function which performs the request.
for (var i = 1; i < n; i++) {
getFile(i);
}
function getFile(index){
var filePath = "xml/chapter_" + index + ".xml";
$http.get( filePath ).success(function (data) {
$scope.chaptersData[index] = data._chapter;
});
}
Within the getFile function, the value of the parameter index will not change as i changes in the outer function. So it will still be at its initial value when the success function executes and can be used for populating the array.

Just get data as an object like
{
order: [] // just your chapter id or whatever identifier value in desired order
data: // whatever you're getting now
}
In your .success (which is btw depreciated and you should use .then() instead) just do
orderedChapters = [];
for (var i = 0; i < order.length; i++) {
for (var j = 0; j < data.length; i++) {
if (order[i] == data[j].id) {
orderedChapters.push(data[j]);
}
}
}

Integrating asynchronous mongo call within an inner forEach loop

I got two loops, the outer loops over the users and the inner one loops over the venueID's of each user. Within the inner loop I want to look up the venue and attach it to an array defined in the outer look (userItem). However because forEach is synchronous and the mongo database look up is asynchronous the result always remains empty. I've tried to integrate this answer but to no avail. How to do this?
ret = [];
users.forEach(function(user) {
var userItem = user.getSanitised('ADM');
userItem.venues = [];
var tmp = [];
userItem.adminVenueIds.forEach(function(adminVenueId){
tmp.push(function(callback) {
Venue.findOne({_id:adminVenueId}, function(error, venue) {
callback(null, venue.toObject());
});
});
});
async.parallel(userItem.venues, function(err, result) {
/* this code will run after all calls finished the job or
when any of the calls passes an error */
if (err)
return console.log(err);
userItem.venues.push(result);
});
ret.push(userItem);
});
Tried the following as well but doesn't work also
users.forEach(function(user) {
var userItem = [];
async.series({
setUserItem : function(callback)
{
userItem = user.getSanitised('ADM');
callback(null, 'OK');
},
setUserVenues : function(callback)
{
userItem.venues = [];
user.adminVenueIds.forEach(function(adminVenueId,index) {
Venue.findOne({_id:adminVenueId}, function(error, venue) {
userItem.venues.push(venue.toObject());
if((index+1) == user.adminVenueIds.length)
callback(null, 'OK');
});
});
}
},
function(error, results) {
if(error)
winston.error(error);
ret.push(userItem);
}
);
});

You could simply put an if statement (in your case put the conditional as the array length) then when the loop is done you could then make it continue doing its thing by calling a function to continue (or put your code in there, but it will start to look messy)
var ret = [];
var test = [];
for (var i = 0; i < 20; i++) {
for (var x = 0; x < 20; x++) {
setTimeout(function() {
test.push("Test"+x);
if (x === 20) {
finishIt();
}
}, 300)
}
}
function finishIt() {
console.log(test);
ret.push(test);
}

I think you might want to look into using Mongoose. It is a NodeJS application layer on top of MongoDB that provides a more SQL like experience.
http://mongoosejs.com

I ended up with the following solution. It's dirty but I guess that's just nodejs being nodejs.
users.forEach(function(user) {
var userItem = user.getSanitised('ADM');
userItem.venues = [];
user.adminVenueIds.forEach(function(adminVenueId) {
Venue.findOne({_id:adminVenueId}, function(error, venue) {
userItem.venues.push(venue.toObject());
});
});
(function(){
if(userItem.venues.length == user.adminVenueIds.length) {
ret.push(userItem);
} else {
setTimeout(arguments.callee, 30);
}
})();
});

Simpy cannot iterate over javascript object?

I have scoured the other question/answer for this and implemented everything and I still cannot access the values of the object. Here's the code I am using:
function apply_voucher(voucher) {
var dates = $.parseJSON($("[name='dates']").val());
var voucher_yes_no = new Array();
var voucher_reduction = new Array();
if(voucher.length > 0)
{
$.each(dates, function(room_id, these_dates) {
$.post('/multiroom/check_voucher/'+voucher+'/'+room_id, function(result) {
if(result.result == 'ok') {
voucher_yes_no.push('yes');
voucher_reduction.push(result.voucher_reduction);
} else {
voucher_yes_no.push('no');
}
}, 'json');
});
// check if there are any yes's in the array
if('yes' in voucher_yes_no) {
console.log("no yes's");
} else {
console.log(voucher_reduction);
console.log(typeof voucher_reduction);
for (var prop in voucher_reduction) {
console.log(prop);
console.log(voucher_reduction[prop]);
if (voucher_reduction.hasOwnProperty(prop)) {
console.log("prop: " + prop + " value: " + voucher_reduction[prop]);
}
}
}
}
}
Apologies for the constant console logging - I'm just trying to track everything to make sure it's all doing what it should. The console output I get from this is below:
...which shows the object containing one value, "1.01" and my console.log of the typeof it to make sure it is actually an object (as I thought I was going mad at one point). After this there is nothing from inside the for-in loop. I have tried jquery's $.each() also to no avail. I can't understand why nothing I'm trying is working!

It does not work because the Ajax call is asynchronous!
You are reading the values BEFORE it is populated!
Move the code in and watch it magically start working since it will run after you actually populate the Array!
function apply_voucher(voucher) {
var room_id = "169";
var dates = $.parseJSON($("[name='dates']").val());
var voucher_reduction = new Array();
$.post('/multiroom/check_voucher/'+voucher+'/'+room_id, function(result) {
if(result.result == 'ok') {
voucher_reduction.push(result.voucher_reduction);
}
console.log(voucher_reduction);
console.log(typeof voucher_reduction);
for (var prop in voucher_reduction) {
console.log(prop);
console.log(voucher_reduction[prop]);
if (voucher_reduction.hasOwnProperty(prop)) {
console.log("prop: " + prop + " value: " + voucher_reduction[prop]);
}
}
}, 'json');
}
From what it looks like, you plan on making that Ajax call in a loop. For this you need to wait for all of the requests to be done. You need to use when() and then(). It is answered in another question: https://stackoverflow.com/a/9865124/14104

Just to say for future viewers that changing the way I did this to use proper deferred objects and promises, which blew my head up for a while, but I got there! Thanks for all the help, particularly #epascarello for pointing me in the right direction :) As soon as I started doing it this way the arrays began behaving like arrays again as well, hooray!
Here's the final code:
function apply_voucher(voucher) {
var booking_id = $("[name='booking_id']").val();
var dates = $.parseJSON($("[name='dates']").val());
if(voucher.length > 0) {
var data = []; // the ids coming back from serviceA
var deferredA = blah(data, voucher, dates); // has to add the ids to data
deferredA.done(function() { // if blah successful...
var voucher_yes_no = data[0];
var voucher_reduction = data[1];
if(voucher_yes_no.indexOf("yes") !== -1)
{
console.log("at least one yes!");
// change value of voucher_reduction field
var reduction_total = 0;
for(var i = 0; i < voucher_reduction.length; i++) {
reduction_total += voucher_reduction[i];
}
console.log(reduction_total);
}
else
{
console.log("there are no yes's");
}
});
}
}
function blah(data, voucher, dates) {
var dfd = $.Deferred();
var voucher_yes_no = new Array();
var voucher_reduction = new Array();
var cycles = 0;
var dates_length = 0;
for(var prop in dates) {
++dates_length;
}
$.each(dates, function(room_id, these_dates) {
$.post('/multiroom/check_voucher/'+voucher+'/'+room_id, function(result) {
if(result.result == 'ok') {
voucher_reduction.push(result.voucher_reduction);
voucher_yes_no.push('yes');
} else {
voucher_yes_no.push('no');
}
++cycles;
if(cycles == dates_length) {
data.push(voucher_yes_no);
data.push(voucher_reduction);
dfd.resolve();
}
}, 'json');
});
return dfd.promise();
}

Can you show how voucher_reduction is defined?
I am wondering where the second line of the debug output comes from, the one starting with '0'.

in this line:
console.log(vouncher_reduction[prop]);
^
The name of the variable is wrong (then) and probably that is breaking your code.

I think there are no problem with your loop.
But perhaps with your object.
Are you sure what properties has enumerable ?
Try to execute this to check :
Object.getOwnPropertyDescriptor(voucher_reduction,'0');
If it return undefined, the property was not exist.