We Keep Coding

Call a Request function from outside the request

Im trying to make a webscraper(educational puposes), and I got really far, but this little issue is bugging me.
I made a request callback function, and im trying to get lines 75-78 to work. However to get this to work, I need PDF_LISTS and PDF_LINKS to initilaze to the right values.
I've already tried to make them global variables, and what not, for some reason that doesnt work. So my question is: How do I make a callback function that will call that for loop (75-78) and succesfully initilaze PDF_LISTS and PDF_LINKS to the correct values ?
(Dont worry I use this on educational content, with the prof's permission). First time posting here!
// URL_LINKS has the pdf links of the pages
PDF_LINKS = [];
// URL_LIST has the names of the pdf links
PDF_LIST = [];
function fillPDF(callback) {
request(url, function(err, res, body) {
$ = cheerio.load(body);
links = $('a'); //jquery get all hyperlinks
$(links).each(function(i, link) {
var value = $(link).attr('href');
// creates objects to hold the file
if (value.substring(value.length - 3, value.length) == "pdf") {
PDF_LINKS[i] = $(link).attr('href');
PDF_LIST[i] = $(link).text();
}
})
});
}
// must decleare fillPDF variable or else you wont initilze teh variables
fillPDF() {
//HERE I WANT PDF_LINKS and PDF_LIST to be intialized to 33.....
}
for (j = 0; j < PDF_LIST.length; j++) {
request(PDF_LINKS[j]).pipe(fs.createWriteStream(PDF_LIST[j]));
}

You may push your values into arrays using array's push method, avoiding array's element to be undefined.
You can put your final for loop into a function, and then use fillPDF();
You also need to call fillPDF's callback once the request is over.
PDF_LINKS = [];
PDF_LIST = [];
function fillPDF(callback) {
request(url, function(err, res, body) {
$ = cheerio.load(body);
links = $('a');
$(links).each(function(i, link) {
var value = $(link).attr('href');
if (value.slice(-3) == "pdf") {
PDF_LINKS.push(value);
PDF_LIST.push($(link).text());
}
})
callback();
});
}
function writePDF() {
for (j = 0; j < PDF_LIST.length; j++) {
request(PDF_LINKS[j]).pipe(fs.createWriteStream(PDF_LIST[j]));
}
}
fillPDF(writePDF);

Log only shows one of four rows of data

I am writing a small Node js application for automatic vehicle location system.
Here is the code for where I am getting trouble.
markerData contains 4 rows but only in the log I can see the last row.
for (var i = 0, len = markerData.length; i < len; i++) {
var thisMarker = markerData[i];
sql.connect(config, function (err) {
var request = new sql.Request();
request.input('myval', sql.Int, thisMarker.id);
request.query('SELECT d.id, d.name, d.lastupdate, p.latitude, p.longitude, p.speed, p.course FROM dbo.devices AS d INNER JOIN dbo.positions AS p ON d.positionid = p.id AND d.id = p.deviceid WHERE (d.id = #myval)', function (err, recordset2) {
if (typeof recordset2 != 'undefined') {
thisMarker.position.lat = recordset2[0].latitude;
thisMarker.position.long = recordset2[0].longitude;
console.log(recordset2[0].id);
}
});
});
}
Please help me to solve the issue.

As var is not a block level variable in terms of scope, when `sql' module takes time to connect to the database asynchronously, the synchronous loop may change the value of the variable that's why you have the last row printed since the variable holds the reference to the last object at the time of successful connection.
Instead of _.each, I would recommend to use async module with async.each since you have few asynchronous operation to get rid of a synchronous loop.
You can check for samples here,
http://justinklemm.com/node-js-async-tutorial/
Here is your updated code with async.each
-> Install async module with npm install async --save
-> Then add the below reference in the required place,
// Reference
var async = require('async');
-> Modified code:
sql.connect(config, function (err) {
if(err) {
console.log('Connection error: ');
console.log(err);
} else {
async.each(markerData, function(thisMarker, callback) {
var request = new sql.Request();
request.input('myval', sql.Int, thisMarker.id);
request.query('SELECT d.id, d.name, d.lastupdate, p.latitude, p.longitude, p.speed, p.course FROM dbo.devices AS d INNER JOIN dbo.positions AS p ON d.positionid = p.id AND d.id = p.deviceid WHERE (d.id = #myval)', function (err, recordset2) {
if(err) {
console.log(err);
callback();
} else {
if (typeof recordset2 != 'undefined') {
thisMarker.position.lat = recordset2[0].latitude;
thisMarker.position.long = recordset2[0].longitude;
console.log(recordset2[0].id);
} else {
console.log('Recordset empty for id: ' + thisMarker.id);
}
callback();
}
});
}, function(err){
if(err) {
console.log(err);
}
});
}
});

I'm not entirely sure how your library works, but presumably recordset2 is an array of records. recordset2[0] is therefore the first record. If you want the next one you should probably try recordset2[1] and so on and so forth.
Arrays: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array
You'll probably need to loop through all the elements in the array at some point. use a for loop for that:
for (var i = 0; i < recordset2.length; i++ {
console.log(recordset2[i])
}
That will print out everything your query returns.

Increment var in callback node js

Is any possible to increment var licznik in this block of code?
I try sth like this, But always receives 0. Could someone explain me what I'm doing wrong?
rows.forEach(function(record) {
var licznik = 0;
var offer = manager.createOffer('76561198252553560');
inventory.forEach(function(item) {
if(licznik <= record.amount) {
if(item.market_hash_name == record.real_name) {
var asid = item.assetid;
(function(licznik){
connection.query('SELECT count(id) as wynik FROM used where asset_id = \'' + asid + '\'', function(err, wiersze) {
if (wiersze[0].wynik == 0) {
var employee = {
asset_id: asid,
trans_id: record.tid
};
connection.query('INSERT INTO used SET ?', employee, function(err, res) {
if (err) throw err;
offer.addMyItem(item);
console.log(licznik);
&licznik++;
});
}
});
})(licznik);
}
}
});
});

As the comment on your original question points out, I have no context for what this code is actually trying to accomplish. What I can tell you is that your callbacks supplied to connection.query are NOT fired on each iteration of the forEach. The whole reason connection.query takes a callback is because you don't know when the operation will complete. Node is designed to be asynchronous so all it does on each iteration of the forEach loop is begin the query. The callback supplied to the query could be invoked at any time which also means that a query that fired after another query could potentially fire its callback before the callback from the first query. It just depends on how long each query takes.
If you need licznik to be incremented on every iteration of the forEach then you need to increment it after your if statement.
rows.forEach(function(record) {
var licznik = 0;
var offer = manager.createOffer('76561198252553560');
inventory.forEach(function(item) {
if(licznik <= record.amount) {
// .... omitted for brevity
}
licznik++; // <-- increment here, outside of the closure.
});
});
Again, I have zero clue what you're actually trying to do with that variable so this may not solve your real problem, but that's the way to get it to increment in that loop.
PS - You may not be understanding that you actually have two licznik variables here. You create a new one when you wrap all your logic in a closure function like you did. If you change the variable declared at the top of your closure function you'll see that it's not the same variable as the one outside the closure function.
rows.forEach(function(record) {
var licznik = 0;
var offer = manager.createOffer('76561198252553560');
inventory.forEach(function(item) {
if(licznik <= record.amount) {
if(item.market_hash_name == record.real_name) {
var asid = item.assetid;
(function(licznik2) { // <-- notice this is a new variable
connection.query('SELECT count(id) as wynik FROM used where asset_id = \'' + asid + '\'', function(err, wiersze) {
if (wiersze[0].wynik == 0) {
var employee = {
asset_id: asid,
trans_id: record.tid
};
connection.query('INSERT INTO used SET ?', employee, function(err, res) {
if (err) throw err;
offer.addMyItem(item);
console.log(licznik2);
licznik2++;
});
}
});
})(licznik);
}
}
});
});

Two async module functions work asynchronously

I tried to scrap for thousands of pages. So I used async.timesSeries and async.waterfall. Each of functions work synchronously very well but they don't work together. What can I do?
The logic is simple.
Because I want to scrape pages are "http://udb.kr/local/category/390101?page="1~1167, async.timesSeries loop 1 to 1167
async.waterfall scraps components of pages
but messages that console shows me looks like this
info.NM values // just for explain, It shows me each attires of obj because I insert console.log(info.NM) for verifying.
info.NM values
info.NM values
info.NM values and randomly ----- page number -----
...
['done',
'done',
'done',
'done',
'done',
...
'done']
info.NM values again
.../Users/Snark/Dev/job_apply/cheerio_job_app_list.js:29
if (tObj[m+1].children != 0) {info.nAddr = tObj[m+1].firstChild.data}else{info.nAddr = null};
^
TypeError: Cannot read property 'children' of undefined
at /Users/Snark/Dev/job_apply/cheerio_job_app_list.js:29:17
at fn (/Users/Snark/node_modules/async/lib/async.js:746:34)
at /Users/Snark/node_modules/async/lib/async.js:1212:16
at /Users/Snark/node_modules/async/lib/async.js:166:37
at /Users/Snark/node_modules/async/lib/async.js:706:43
at /Users/Snark/node_modules/async/lib/async.js:167:37
at /Users/Snark/node_modules/async/lib/async.js:1208:30
at Request._callback (/Users/Snark/Dev/job_apply/cheerio_job_app_list.js:21:6)
at Request.self.callback (/Users/Snark/node_modules/request/request.js:198:22)
at emitTwo (events.js:87:13)
And this is js code.
var request = require("request"),
cheerio = require("cheerio"),
jsonfile = require("jsonfile"),
fs = require("fs"),
async = require("async");
var info = {},
dbArray = [];
var url = "http://udb.kr/local/category/390101?page=";
async.timesSeries(1166, function(n, next) {
var page = n + 1
async.waterfall([
function(callback) {
request(url + page, function(error, response, html) {
if (error) {
throw error
};
var $ = cheerio.load(html),
tObj = $('tbody tr td');
callback(null, tObj);
});
},
function(tObj, callback) {
for (var m = 0; m < 150; m = m + 5) {
if (tObj[m]) {
info.NM = tObj[m].firstChild.children[0].data
} else {
info.NM = null
};
if (tObj[m + 1].children != 0) {
info.nAddr = tObj[m + 1].firstChild.data
} else {
info.nAddr = null
};
console.log(info.NM);
dbArray.push(info);
}
callback(dbArray, callback);
},
function(dbArray, callback) {
fs.appendFile('./jobDB_l.json', JSON.stringify(dbArray), function (err) {
if (err)
throw err;
});
callback(null, 'done');
}
], function(err, result) {
console.log('----- ' +page+ '-----');
});
next(null, 'done');
}, function(err, result) {
console.log(result)
});

To get these to work together where you are using waterfall inside of each timesSeries iteration, you need to call the timesSeries done callback from the completion callback for the waterfall call. Right now, you are calling it long before that which means that timesSeries won't wait for the waterfall to be done.
You can do that by changing this:
], function(err, result) {
console.log('----- ' +page+ '-----');
});
next(null, 'done');
to this:
], function(err, result) {
console.log('----- ' +page+ '-----');
next(null, 'done');
});
It also seems odd that you have a hard-coded for loop limit of m < 150 rather than using the actual length of the content. You can easily run off the end of the content and potentially cause problems.
And, your error handling probably won't work well either. If you throw inside of the async request() callback, that's not going to go anywhere. You need much better error handling such as calling callback(error) to pass the error on to async.waterfall().
You also may want to surround all your DOM walking in a try/catch so if you throw any exceptions there, you can catch them yourself, analyze them and then fix the code.

if (tObj[m+1] && tObj[m+1].children != 0)

Node/Express - How to wait until For Loop is over to respond with JSON

I have a function in my express app that makes multiple queries within a For Loop and I need to design a callback that responds with JSON when the loop is finished. But, I'm not sure how to do this in Node yet. Here is what I have so far, but it's not yet working...
exports.contacts_create = function(req, res) {
var contacts = req.body;
(function(res, contacts) {
for (var property in contacts) { // for each contact, save to db
if( !isNaN(property) ) {
contact = contacts[property];
var newContact = new Contact(contact);
newContact.user = req.user.id
newContact.save(function(err) {
if (err) { console.log(err) };
}); // .save
}; // if !isNAN
}; // for
self.response();
})(); // function
}; // contacts_create
exports.response = function(req, res, success) {
res.json('finished');
};

There are a few problems with your code besides just the callback structure.
var contacts = req.body;
(function(res, contacts) {
...
})(); // function
^ you are redefining contacts and res in the parameter list, but not passing in any arguments, so inside your function res and contacts will be undefined.
Also, not sure where your self variable is coming from, but maybe you defined that elsewhere.
As to the callback structure, you're looking for something like this (assuming contacts is an Array):
exports.contacts_create = function(req, res) {
var contacts = req.body;
var iterator = function (i) {
if (i >= contacts.length) {
res.json('finished'); // or call self.response() or whatever
return;
}
contact = contacts[i];
var newContact = new Contact(contact);
newContact.user = req.user.id
newContact.save(function(err) {
if (err)
console.log(err); //if this is really a failure, you should call response here and return
iterator(i + 1); //re-call this function with the next index
});
};
iterator(0); //start the async "for" loop
};
However, you may want to consider performing your database saves in parallel. Something like this:
var savesPending = contacts.length;
var saveCallback = function (i, err) {
if (err)
console.log('Saving contact ' + i + ' failed.');
if (--savesPending === 0)
res.json('finished');
};
for (var i in contacts) {
...
newContact.save(saveCallback.bind(null, i));
}
This way you don't have to wait for each save to complete before starting the next round-trip to the database.
If you're unfamiliar with why I used saveCallback.bind(null, i), it's basically so the callback can know which contact failed in the event of an error. See Function.prototype.bind if you need a reference.