I am trying to nest several callbacks with the goal of programmatically navigating a page using POST requests. I have a list of different files I am trying to download, and want to assign each of them a different filename so that I can make asynchronous download calls and perform OCR on them.
The callback that actually downloads the PDF is here:
filename = "doodle" + Math.floor((Math.random() * 100) + 1) + ".pdf";
request.post(
{
url:'https://ecorp.sos.ga.gov/BusinessSearch/DownloadFile',
form: {
'documentId': body
},
headers: {
'Referer': 'https://ecorp.sos.ga.gov/BusinessSearch/BusinessFilings'
}
}, getPDF).pipe(fs.createWriteStream(filename));
Notice that the request sends the resulting page to a getPdf function, which actually opens the PDF for OCR after the request has been made. Currently I have a filename hardcoded in the OCR method, because I do not know how to pass an additional variable to the callback function. I would like to be able to write to a randomly generated file, and in the callback method retrieve that filename.
function getPDF (error, response, body) {
if(!error){
console.log(body);
filename = "/doodle" + Math.floor((Math.random() * 100) + 1) + ".pdf";
//console.log(__dirname + filename);
/*pdfText(__dirname + "/doodle.pdf", function(err, chunks) {
//console.log(chunks);
});*/
var pdf_path = __dirname + filename;
//option to extract text from page 0 to 10
var option = {from: 0, to: 10};
var pdf_body;
pdfUtil.pdfToText(pdf_path, option, function(err, data) {
pdf_body = data;
var result;
try{
var namez = /AUTHORIZED SIGNATURE:\s+(.+?)\s{2}/.exec(data)[1];
var emailz = /Email: (.+?)\s{2}/.exec(data)[1];
result = {query:query, info:{name: namez, email: emailz}};
} catch (err){
//result = {query:query, error:"non-selectable text in PDF"};
}
results.push(result);
//console.log(JSON.stringify(result));
});
}
}
the library I am using is called request
and the documentation shows the following:
The callback argument gets 3 arguments:
An error when applicable (usually from http.ClientRequest object)
An http.IncomingMessage object
The third is the response body (String or Buffer, or JSON object if the json option is supplied)
The most straight forward approach would be to simply change the signature of getPDF to
function getPDF (filename, error, response, body) {...}
then you'd call it from within an anonymous function as the callback handler of request.post:
request.post({...}, function(error, response, body){
getPDF( 'filename', error, response, body);
});
E6 notation makes this even nicer:
request.post({...}, (error, response, body)=>{
getPDF( 'filename', ...arguments);
});
You can accomplish this using a more sophisticated method by partially applying the additional argument and then calling it in point-free fashion:
function getPDF (filename, error, response, body, ) {...}
getPdf2 = getPDF.bind(null, "filename");
request.post({...}, getPdf2);
The approaches amount to to the same thing and I might use either depending on circumstances. Arguably, the intent of the second approach is easier to read. It's certainly more fashionable.
Related
I'm fairly new to javascript and I'm still trying to grasp callback concepts. Even then, I'm not sure I have an issue with callbacks or ajax itself.
So I have this callback function which basically takes an array of objects somewhere else in my code.
function powerAmplifierInfo(id, cb) {
ApiGet('/system/radio-frequency/tx/' + id + '/power-amplifier', function (err, result) {
if (err && !TESTING) {
alertError("Connection error: " + err.statusText);
return;
} else if (result && result.result.code !== "OK") {
alertError(result.result.message);
return;
}
if (TESTING) {
result = { powerAmplifier: "TEST model", uptime: Date.now() };
}
cb(result.powerAmplifier);
});
}
This function uses another function which uses ajax to send a get request to some server.
function ApiGet(urlSuffix, cb) {
var url = apiPrefix + "/" + urlSuffix;
$.ajax({
url: url,
type: 'GET',
success: function (result) {
cb(null, result);
},
error: function (err) {
cb(err, null);
}
});
}
Now, the request chains in my list works but sometimes requests get mixed up together when I rerun the code in loop as it seems ajax switch some port orders while sending requests. I found out this issue while troubleshooting my local network with Wireshark. Here's an image to show it:
The Get requests are sent but between the 3rd and the 4th ones, there is a port mixup. As we can see, the server replies without the port order mixup.
I'm wondering, is this a known issue or would there be a problem with my code, or even my server? I do not get this error when sending a single object in my callback functions instead of an object array.
Thanks in advance!
I'm using AWS S3 API wrapper. To download files from cloud I call the following wrapper:
aws.s3.downloadFile(bucket, fileName, cbDownloadOk, cbDownloadErr);
Inside this function we build parameters' container and then call the official AWS S3 API:
s3.listObjectsV2(params, function(err, data) {
if (err) {
cbDownloadErr(err); // an error occurred
} else {
cbDownloadOk(data); // successful response
}
});
Now I want to be able to print the name of downloaded file, e.g.:
const cbDownloadOk = function (data) {
console.log("File " + fileName + " downloaded");
}
The problem is that I can't change the implementation of wrapper and hence, to change the signature of the cbDownloadOk callback.
Is there any way to pass to cbDownloadOk the fileName without changing the implementation of the wrapper? Is usage of global variable is the only way to do that?
Update:
As we can learn from the answer, this question deals with the currying.
What you're looking for is probably a curried function. You can call it with filename and it will give you exactly what you want: a function accepts exactly one parameter, data, and has access to the additional parameter (filename).
const mySuccessCb = filename => data => {
// data and name are available here
console.log("File " + filename + " downloaded");
}
Usage:
aws.s3.downloadFile(myBucket, filename, mySuccessCb(filename), cbDownloadErr);
If I'm understanding your question correctly, you just need to ensure cbDownloadOk is defined in a scope that has access to fileName:
aws.s3.downloadFile(
bucket,
fileName,
(data) => console.log("File " + fileName + " downloaded"),
cbDownloadErr
);
Or you could pass a function that calls cbDownloadOk with fileName as an additional parameter:
aws.s3.downloadFile(
bucket,
fileName,
(data) => cbDownloadOk(data, fileName),
cbDownloadErr
);
I have an endpoint defined at /api/profile which accepts post parameters.
var http = require('http');
var serverConfig = require('../server.config.js');
var request = require('request');
module.exports = function(server){
server.route({
method: 'POST',
path: '/api/profile',
handler: getProfileData
});
function getProfileData(request, reply){
var battleTag = request.payload.battleTag;
getProfileDataHttp(battleTag, function(err, data){
if(err){
reply(new Error(err));
}
reply(data);
});
}
function getProfileDataHttp(battleTag, callback){
var key = serverConfig.battleNet.apiKey;
var tag = encodeURIComponent(battleTag);
var url = 'https://eu.api.battle.net/d3/profile/'+ tag + '/?locale=en_GB&callback=JSON_CALLBACK&apikey=' + key;
console.log(url);
request(url,function(error, response, body){
if(error){
callback(err);
}
if(!error && response.statusCode ==200){
callback(null, body);
}
});
}
};
it is calling an api with a json callback, when I am receiving the data it is in format:
JSON_CALLBACK({ json data here})
how can I get this endpoint to return just the json data, I have tried JSON.parse() but it causes errors in the server.
the angular service that calls this endpoint is like below:
function getProfileData(battleTag){
var defer = $q.defer();
var tag = validTag(battleTag);
if(!tag){
defer.reject('Invalid Tag please use format 1[a-z]11[a-z0-9]#4[0-9]');
return defer.promise;
}
$http.post('/api/profile', {
battleTag: battleTag
})
.success(function(data){
if(data.reason){
defer.resolve(data.reason);
}
defer.resolve(data);
})
.error(function(err){
defer.reject(err);
});
return defer.promise;
}
the call would work when using $http.jsonp in angular however I had to create the server to hide the secret key from the client
Your question is a bit confusing. You are talking about JSONP, but you want to fetch the data directly.
The whole point of JSONP is to return the data encapsulated inside a function that you choose. You then simply have to execute it.
If you want the data in a direct way, don't use JSONP. Simply do a "normal" call.
After having a quick look at the Battle.net API, it seems that to get the data directly, you should simply omit the 'callback' parameter in the URL of your request.
Thus, your request URL would looks like that:
var url = 'https://eu.api.battle.net/d3/profile/'+ tag + '/?locale=en_GB&apikey=' + key;
I am a rookie in Nodejs and asynchronous programming. I am having a problem executing a GET request inside an asynchronous function. Here I am posting the whole code. I am trying to pull a list of all Urls , add them to a list and send the list for processing to another function.
My problem is with processing them. Inturn for each url I am executing a GET request to fetch the body and to look for image elements in it. I am looking to pass the Image url to a 3rd party api as a GET param. I am unable to execute the GET request as the control doesn't seem to reach there at all.
var async = require("async"),
request = require("request"),
cheerio = require("cheerio");
async.waterfall([
function(callback) {
var url = "someSourceUrl";
var linkList = [];
request(url, function(err, resp, body) {
var $ = cheerio.load(body);
$('.list_more li').each(function() {
//Find all urls and add them to a list
$(this).find('a').each(function() {
linkList.push($(this).attr('href'));
});
});
callback(null, linkList);
});
},
//pass all the links as a list to callback
function(liksListFetched, callback) {
for (var i in liksListFetched) {
callback(null, liksListFetched[i]);
}
}],
//***********My problem is with the below code**************
function(err, curUrl) {
var cuResp = "";
console.log("Currently Processing Url : " + curUrl);
request(curUrl, function(err, resp, body) {
var $ = cheerio.load(body);
var article = $("article");
var articleImage = article.find("figure").children('img').attr('src');
var responseGrabbed = "API response : ";
//check if there is an IMG element
if (articleImage === undefined) {
console.log("No Image Found.");
articleImage = 'none';
}
else {
//if there is an img element, pass this image url to an API,
//So do a GET call by passing imageUrl to the API as a GET param
request("http://apiurl.tld?imageurl=" + articleImage, function(error, response, resp) { //code doesn't seem to reach here
I would like to grab the response and concatenate it to the responseGrabbed var.
console.log(resp);
responseGrabbed += resp;
});
}
console.log(responseGrabbed);// api response never gets concatenated :(
console.log("_=_=_=_=_=_=__=_=_=_=_=_=__=_=_=_=_=_=__=_=_=_=_=_=_");
process.exit(0);
});
});
I appreciate if any one can help me understand the root cause. Thanks in advance.
request() is asynchronous, so when you're console logging the string, the string hasn't been built yet, you have to do the console log inside the callback :
request("http://apiurl.tld?imageurl=" + articleImage, function(error, response, resp) {
responseGrabbed += resp;
console.log(responseGrabbed);// api response never gets concatenated :(
console.log("_=_=_=_=_=_=__=_=_=_=_=_=__=_=_=_=_=_=__=_=_=_=_=_=_");
});
Same goes for terminating the process, which should be done when all the requests have finished
I'm a long time PHP (CodeIgniter & WordPress) developer that only recently wanted to learn a few other languages. I've set out to learn Ruby (on Rails, and Sinatra), Python (w/ Flask framework) and Javascript with node.js.
I decided to create the most basic application I can think of, a URL expander, using each of these languages. I have managed to create a working version in every language, except node.js and Javascript.
I kinda know my problem, I know it is related to callbacks. I know I'm not doing it right. I get the basic idea of callbacks, but I just cannot figure out how to fix this mess I have created.
This is my whole code:
var http = require('http');
var url = require('url');
function expand() {
var short = url.parse('http://t.co/wbDrgquZ');
var options = {
host: short.hostname,
port: 80,
path: short.pathname
};
function longURL(response) {
console.log(response.headers.location);
}
http.get(options, longURL);
}
function start() {
function onRequest(request, response) {
console.log("Request received.");
response.writeHead(200, {
"Content-Type": "text/plain"
});
response.write("Hello World");
expand();
response.end();
}
http.createServer(onRequest).listen(8888);
console.log("Server has started.");
}
start();
The server starts, and when a request is made, it calls the expand function which returns the expanded URL in the terminal. I'm trying to get it to print in the browser.
Any help is appreciated. Thanks in advance.
You've made a few flaws.
You should rewrite expand to pass the url in and pass a callback in. Any function that does anything asynchronous generally has the signature (data, callback) in node. This basically allows you to say I want this function to do something then tell me when it's done.
function expand(urlToParse, callback) {
// note we pass in the url this time
var short = url.parse(urlToParse);
var options = {
host: short.hostname,
port: 80,
path: short.pathname
};
// note we store the clientRequest object temporarily
var clientRequest = http.get(options, extractRealURL);
// Always attach the error handler and forward any errors
clientRequest.on("error", forwardError);
function extractRealURL(res) {
callback(null, res.headers.location);
}
function forwardError(error) {
callback(err);
}
}
Here the callback is expected to have the signature of (err, data) which almost all callbacks in node have. We've also added error handling which is a must.
We now change onRequest to actually call expand properly
function onRequest(request, response) {
// parse the incoming url. true flag unpacks the query string
var parsedUrl = url.parse(request.url, true),
// extract the querystring url.
// http://localhost:8888/?url=http://t.co/wbDrgquZ
urlToExpand = parsedUrl.query.url;
// call expand with the url and a callback
expand(urlToExpand, writeResponse);
function writeResponse(error, newUrl) {
// handle the error case properly
if (error) {
response.writeHead(500, { 'Content-Type': 'text/plain'});
// early return to avoid an else block
return response.end(error.message);
}
response.writeHead(200, { 'Content-Type': 'text/plain'});
// write the new url to the response
response.end(newUrl);
}
}
Here we have added error handling logic and also unpacked the actual url to expand from the query string.
Generally the pattern of doSomething<data, callback<err, result>> works very well in node.js.
It's the exact same as let result = doSomething<data> mayThrow err that you expect in your normal blocking languages except it's asynchronous.
Note that the alternative option of passing the ServerResponse object into the function is frowned upon, by doing so your creating unnecessary hard coupling between the expand function and the server response.
The expand function should only expand an url and return the expanded url, it has no business doing IO itself.
Full code
A callback is just a word to describe a function that we pass to some other code for that other code to invoke.
In your example, onRequest is a callback function that gets passed to createServer to be invoked whenever a request is received.
I think the issue you're having is that you're expecting expand() to have access to all the same variables/parameters that the onRequest function has access to. This isn't the case.
You need pass the response object to expand(). Because the call to expand creates a new callback longURL for the http.get call, it will have access to the response object that you passed in.
function expand( resp ) {
// receive the original response object, and end the response when ready
var short = url.parse('http://t.co/wbDrgquZ');
var options = {
host: short.hostname,
port: 80,
path: short.pathname
};
function longURL( response ) {
console.log(response.headers.location);
resp.end( response.headers.location ); // end the original response
}
http.get(options, longURL);
}
function start() {
function onRequest(request, response) {
console.log("Request received.");
response.writeHead(200, {
"Content-Type": "text/plain"
});
response.write("Hello World");
expand( response ); // pass this response object to expand
}
http.createServer(onRequest).listen(8888);
console.log("Server has started.");
}
You weren't sending the response as a parameter to the expand function and also you were calling response.end() before the expand() function could write anything, here's the corrected version:
var http = require('http');
var url = require('url');
function expand(res) {
var short = url.parse('http://t.co/wbDrgquZ');
var options = {
host: short.hostname,
port: 80,
path: short.pathname
};
function longURL(response){
console.log(response.headers.location);
res.end("<br />" + response.headers.location);
}
http.get(options, longURL);
}
function start() {
function onRequest(request, response) {
console.log("Request received.");
response.writeHead(200, {"Content-Type": "text/html"});
response.write("Hello World");
expand(response);
}
http.createServer(onRequest).listen(8888);
console.log("Server has started.");
}
start();