I'm trying to understand how to write code with promises.
Check my code plz. This is right?
Node.js + request:
request(url, function (error, response, body) {
if (!error && response.statusCode == 200) {
var jsonpData = body;
var json;
try {
json = JSON.parse(jsonpData);
} catch (e) {
var startPos = jsonpData.indexOf('({');
var endPos = jsonpData.indexOf('})');
var jsonString = jsonpData.substring(startPos+1, endPos+1);
json = JSON.parse(jsonString);
}
callback(null, json);
} else {
callback(error);
}
});
Node.js + bluebird + request:
request.getAsync(url)
.spread(function(response, body) {return body;})
.then(JSON.parse)
.then(function(json){console.log(json)})
.catch(function(e){console.error(e)});
How to check response status? I should use if from first example or something more interesting?
You can simply check if the response.statusCode is not 200 in the spread handler and throw an Error from that, so that the catch handler will take care of it. You can implement it like this
var request = require('bluebird').promisifyAll(require('request'), {multiArgs: true});
request.getAsync(url).spread(function (response, body) {
if (response.statusCode != 200)
throw new Error('Unsuccessful attempt. Code: ' + response.statusCode);
return JSON.parse(body);
}).then(console.log).catch(console.error);
And if you notice, we return the parsed JSON from the spread handler, because JSON.parse is not an async function, so we don't have to do it in a separate then handler.
One way to check the status code:
.spread(function(response, body) {
if (response.statusCode !== 200) {
throw new Error('Unexpected status code');
}
return body;
})
Related
Trying to scrape the front page of a website (www.ozbargain.com) to return any content in the a tag that holds a reference to xbox but nothing is being returned to console. I believe the issue is with the if statement with :contains.
var fs = require('fs'),
request = require('request'),
cheerio = require('cheerio');
url = 'http://www.ozbargain.com.au';
request(url, function(error, response, html) {
if (!error && response.statusCode == 200) {
var $ = cheerio.load(html);
if($("a:contains('Xbox')").length) {
//console.log(this);
var el = $(this);
var log = el.text();
console.log(log);
} else {
console.log('hey');
}
}
});
The html block I'm after. In particulare, I want the a tag;
<h2 class="title" id="title214252">Free on Xbox One, Xbox 360, PS3, PS4: Tales from the Borderlands (Episode 1)</h2>
The Cheerio syntax for contains is slightly different than jQuery. Ommit the single quotes around the string you're searching for an it should work:
$("a:contains(Xbox)")
Assigned the selector to a variable then called the text method.
request(url, function(error, response, html) {
if (!error && response.statusCode == 200) {
var $ = cheerio.load(html);
var $el = $("a:contains('Xbox')");
if ($el.length) {
console.log($el.text());
} else {
console.log('hey');
}
}
});
This code will not show html in console it show error
var request = require('request');
var thai_url = "http://pantip.com/tag/Honda_(มอเตอร์ไซค์)";
request(thai_url, function (error, response, html) {
if (!error && response.statusCode === 200) {
console.log(html);
} else {
console.log("Error");
}
});
But I change thai_url from มอเตอร์ไซค์ to %E0%B8%A1%E0%B8%AD%E0%B9%80%E0%B8%95%E0%B8%AD%E0%B8%A3%E0%B9%8C%E0%B9%84%E0%B8%8B%E0%B8%84%E0%B9%8C
Like This
var request = require('request');
var thai_url = "http://pantip.com/tag/Honda_(%E0%B8%A1%E0%B8%AD%E0%B9%80%E0%B8%95%E0%B8%AD%E0%B8%A3%E0%B9%8C%E0%B9%84%E0%B8%8B%E0%B8%84%E0%B9%8C)";
request(thai_url, function (error, response, html) {
if (!error && response.statusCode === 200) {
console.log(html);
} else {
console.log("Error");
}
});
The second code working right. Can I change มอเตอไซค์ to %E0%B8%A1%E0%B8...(more)? Or If it's possible, How am I doing for http://pantip.com/tag/Honda_(มอเตอร์ไซค์) working with npm request ? Thank you for any help.
First one is working fine. Just log the error instead of "error" string.
Make sure your IDE supports utf character. otherwise on saving it might be changing it. That why you might get error.
Update. Now I can solve this problem by this
var thai_url = encodeURI("http://pantip.com/tag/Honda_(มอเตอร์ไซค์)");
Use encodeURI. http://www.javascripter.net/faq/escape.htm
I am trying to implement this (.net) example in Node.js using request and iconv-lite (its an HTTP web service that requires url requests are encoded in ISO-8859-1):
WebClient wc = new WebClient();
var encoding = Encoding.GetEncoding("iso-8859-1");
var url = new StringBuilder();
url.Append("https://url.com?");
url.Append("¶m=" + HttpUtility.UrlEncode(foo, encoding));
wc.Encoding = encoding;
return wc.DownloadString(url.ToString());
The problem is with encoding the URL (doesnt work). I am trying to do the same GET request, in which the URL must be encoded as ISO-8859-1. However, by doing something like this:
var options = {
url : iconv.encode(url, 'ISO-8859-1').toString(),
method: 'get',
headers : {
'Content-Type': 'application/x-www-form-urlencoded; charset=ISO-8859-1'
}
};
request(options, function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body);
}else {
console.log('err: ' + error);
}
});
It's still not sent as a ISO-8859-1 string. Any clues on how to get this to work like with the .NET example?
I think it's decoding the response, not encoding the url. Try this:
var https = require('https'),
qs = require('querystring');
// ...
var url = 'https://url.com/?' + qs.stringify({ param: 'foo' });
https.get(url, function(res) {
if (res.statusCode === 200) {
var bufs = [], bufsize = 0;
res.on('data', function(data) {
bufs.push(data);
bufsize += data.length;
}).on('end', function() {
var buffer = Buffer.concat(bufs, bufsize),
body = iconv.decode(buffer, 'iso-8859-1');
console.log('Body: ' + body);
});
return;
} else
console.log('Non-OK status code: ' + res.statusCode);
res.resume();
});
I currently have an endpoint which is used to circumvent CORS that takes in any image URL and returns the contents in data-uri format. This works well, but I think it would be much more efficient to use a streaming solution where the response is built as a stream from the image request.
Here is the working, non-stream version:
app.get('/api/image/convert', function (req, res) {
// TODO: use request piping for significantly more efficient throughput
request.get(req.query.url, function (error, response, body) {
if (!error && response && response.statusCode >= 200 && response.statusCode < 300) {
var data = "data:" + response.headers["content-type"] + ";base64," + new Buffer(body).toString('base64')
res.send(data)
} else {
res.send((response && response.statusCode) || 500, error)
}
})
})
My question is would it be more efficient to pipe the request.get to the result and, if so, how would this look?
Thanks!
I've try this with google maps example, and it works
var request = require('request');
app.get('/', function(req, res){
var url = "http://maps.googleapis.com/maps/api/staticmap?center=Brooklyn+Bridge,New+York,NY&zoom=13&size=600x300&maptype=roadmap" +
"&markers=color:blue%7Clabel:S%7C40.702147,-74.015794&markers=color:green%7Clabel:G;%7C40.711614,-74.012318" +
"&markers=color:red%7Ccolor:red%7Clabel:C%7C40.718217,-73.998284&sensor=false";
request.get(url).pipe(res);
});
github:request
I'm tryign to build a very simple scraper function for nodeJS - just a function that I can pass a URL to, and it returns the scraped data as var data.
I'm completely new to Node.js and can't work out why the following isn't working:
var request = require('request');
var cheerio = require('cheerio');
function scrape(url) {
console.log("Scraping: " + url);
request(url, function(err, resp, body) {
if (err) {
throw err;
}
var html = cheerio.load(body);
return html;
});
}
var data = scrape('http://www.stackoverflow.com');
$ = data;
var logo = $('#hlogo a').text();
console.log(logo);
The above code should return "Stack Overflow" but obviously does not. When I run this in the console I get an error:
var logo = $('#hlogo a').text();
^
TypeError: Property '$' of object #<Object> is not a function
Any ideas why this isn't working for me?
Your data will be undefined, because scrape function does not return a value, additionaly it asynchronous.
You need change logic to something like this:
function scrape(url, oncomplete) {
console.log("Scraping: " + url);
request(url, function(err, resp, body) {
if (err) {
throw err;
}
var html = cheerio.load(body);
oncomplete(html);
});
}
scrape('http://www.stackoverflow.com', function(data) { /* do work here*/ });