I am trying to access a non utf-8 website using request module. Response is garbled for this request.
var request = require('request');
request('http://www.alc.co.jp/', function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body) // Print the web page.
}
});
Even after setting the encoding option to Shift_JIS I am seeing garbled Japanese text.
You need to do the conversion yourself. The example code below uses node-iconv.
var Iconv = require('iconv').Iconv;
var request = require('request');
request({
uri: 'http://www.jalan.net/',
encoding: null,
}, function (error, response, body) {
if (!error && response.statusCode == 200) {
body = new Iconv('shift_jis', 'utf-8').convert(body).toString();
console.log(body); // Print the web page.
}
});
The encoding: null parameter asks request not to convert the Buffer (a byte array) into String yet.
We pass this buffer to Iconv for converting into another Buffer of UTF-8 encoding.
Now this Buffer is good for being converted into a String.
(BTW, http://www.alc.co.jp has switched to UTF-8, so I substituted with another site.)
Related
I used get method of request module to get content of external site. If encoding of external site is utf-8, it is ok, but it has display error with other encodings such as shift-jis
function getExternalUrl(request, response, url){
mod_request.get(url, function (err, res, body) {
//mod_request.get({uri: url, encoding: 'binary'}, function (err, res, body) {
if (err){
console.log("\terr=" + err);
}else{
var result = res.body;
// Process res.body
response.write(result);
}
response.end();
});
}
How can I get content of external site with correct encoding?
I found the way to do:
Get with binary encoding
var mod_request = require('request');
mod_request.get({ uri: url, encoding: 'binary', headers: headers }, function(err, res, body) {});
Create a Buffer with binary format
var contentBuffer = new Buffer(res.body, 'binary');
Get real encoding of page by detect-character-encoding npm
var mod_detect_character_encoding = require('detect-character-encoding');
var charsetMatch = mod_detect_character_encoding(contentBuffer);
Convert page to utf-8 by iconv npm
var mod_iconv = require('iconv').Iconv;
var iconv = new mod_iconv(charsetMatch.encoding, 'utf-8');
var result = iconv.convert(contentBuffer).toString();
P/S: This way is only applied for text file (html, css, js). Please do not apply for image file or others which is not text
What other libraries i can use with request so I can convert it to an array of bytes?
var request = require('request');
request('https://randomuser.me/api/portraits/men/81.jpg', function (error, response, body) {
if (!error && response.statusCode == 200) {
}
})
ImageMagic, which has several 3rd party JS SDKs, would work.
This code will not show html in console it show error
var request = require('request');
var thai_url = "http://pantip.com/tag/Honda_(มอเตอร์ไซค์)";
request(thai_url, function (error, response, html) {
if (!error && response.statusCode === 200) {
console.log(html);
} else {
console.log("Error");
}
});
But I change thai_url from มอเตอร์ไซค์ to %E0%B8%A1%E0%B8%AD%E0%B9%80%E0%B8%95%E0%B8%AD%E0%B8%A3%E0%B9%8C%E0%B9%84%E0%B8%8B%E0%B8%84%E0%B9%8C
Like This
var request = require('request');
var thai_url = "http://pantip.com/tag/Honda_(%E0%B8%A1%E0%B8%AD%E0%B9%80%E0%B8%95%E0%B8%AD%E0%B8%A3%E0%B9%8C%E0%B9%84%E0%B8%8B%E0%B8%84%E0%B9%8C)";
request(thai_url, function (error, response, html) {
if (!error && response.statusCode === 200) {
console.log(html);
} else {
console.log("Error");
}
});
The second code working right. Can I change มอเตอไซค์ to %E0%B8%A1%E0%B8...(more)? Or If it's possible, How am I doing for http://pantip.com/tag/Honda_(มอเตอร์ไซค์) working with npm request ? Thank you for any help.
First one is working fine. Just log the error instead of "error" string.
Make sure your IDE supports utf character. otherwise on saving it might be changing it. That why you might get error.
Update. Now I can solve this problem by this
var thai_url = encodeURI("http://pantip.com/tag/Honda_(มอเตอร์ไซค์)");
Use encodeURI. http://www.javascripter.net/faq/escape.htm
I currently have an endpoint which is used to circumvent CORS that takes in any image URL and returns the contents in data-uri format. This works well, but I think it would be much more efficient to use a streaming solution where the response is built as a stream from the image request.
Here is the working, non-stream version:
app.get('/api/image/convert', function (req, res) {
// TODO: use request piping for significantly more efficient throughput
request.get(req.query.url, function (error, response, body) {
if (!error && response && response.statusCode >= 200 && response.statusCode < 300) {
var data = "data:" + response.headers["content-type"] + ";base64," + new Buffer(body).toString('base64')
res.send(data)
} else {
res.send((response && response.statusCode) || 500, error)
}
})
})
My question is would it be more efficient to pipe the request.get to the result and, if so, how would this look?
Thanks!
I've try this with google maps example, and it works
var request = require('request');
app.get('/', function(req, res){
var url = "http://maps.googleapis.com/maps/api/staticmap?center=Brooklyn+Bridge,New+York,NY&zoom=13&size=600x300&maptype=roadmap" +
"&markers=color:blue%7Clabel:S%7C40.702147,-74.015794&markers=color:green%7Clabel:G;%7C40.711614,-74.012318" +
"&markers=color:red%7Ccolor:red%7Clabel:C%7C40.718217,-73.998284&sensor=false";
request.get(url).pipe(res);
});
github:request
Using Node v0.2.0 I am trying to fetch an image from a server, convert it into a base64 string and then embed it on the page in an image tag. I have the following code:
var express = require('express'),
request = require('request'),
sys = require('sys');
var app = express.createServer(
express.logger(),
express.bodyDecoder()
);
app.get('/', function(req, res){
if(req.param("url")) {
var url = unescape(req.param("url"));
request({uri:url}, function (error, response, body) {
if (!error && response.statusCode == 200) {
var data_uri_prefix = "data:" + response.headers["content-type"] + ";base64,";
var buf = new Buffer(body);
var image = buf.toString('base64');
image = data_uri_prefix + image;
res.send('<img src="'+image+'"/>');
}
});
}
});
app.listen(3000);
Note: This code requires "express" and "request". And of course, node. If you have npm installed, it should be as simple as "npm install express" or "npm install request".
Unfortunately, this doesn't work as expected. If I do the conversion with the Google logo, then I get the following at the beginning of the string:
77+9UE5HDQoaCgAAAA1JSERSAAABEwAAAF8IAwAAAO+/ve+/ve+/vSkAAAMAUExURQBzCw5xGiNmK0t+U++/vQUf77+9BiHvv70WKO+/vQkk77+9D
However if I use an online Base64 encoder with the same image, then it works perfectly. The string starts like this:
iVBORw0KGgoAAAANSUhEUgAAARMAAABfCAMAAAD8mtMpAAADAFBMVEUAcwsOcRojZitLflOWBR+aBiGQFiipCSS8DCm1Cya1FiyNKzexKTjDDSrLDS
Where am I going wrong that this isn't working correctly? I have tried so many different js base64 implementations and they all don't work in the same way. The only thing I can think of is that I am trying to convert the wrong thing into base64, but what should I convert if that is the case?
The problem is encoding and storing binary data in javascript strings. There's a pretty good section on this under Buffers at http://nodejs.org/api.html.
Unfortunately, the easiest way to fix this involved changing the request npm. I had to add response.setEncoding('binary'); on line 66 just below var buffer; in /path/to/lib/node/.npm/request/active/package/lib/main.js. This will work fine for this request but not others. You might want to hack it so that this is only set based on some other passed option.
I then changed var buf = new Buffer(body) to var buf = new Buffer(body, 'binary');. After this, everything worked fine.
Another way to do this, if you really didn't want to touch the request npm, would be to pass in an object that implements Writable Stream in the responseBodyStream argument to request. This object would then store the streamed data from the response in it's own buffer. Maybe there is a library that does this already... i'm not sure.
I'm going to leave it here for now, but feel free to comment if you want me to clarify anything.
EDIT
Check out comments. New solution at http://gist.github.com/583836
The following code (available at https://gist.github.com/804225)
var URL = require('url'),
sURL = 'http://nodejs.org/logo.png',
oURL = URL.parse(sURL),
http = require('http'),
client = http.createClient(80, oURL.hostname),
request = client.request('GET', oURL.pathname, {'host': oURL.hostname})
;
request.end();
request.on('response', function (response)
{
var type = response.headers["content-type"],
prefix = "data:" + type + ";base64,",
body = "";
response.setEncoding('binary');
response.on('end', function () {
var base64 = new Buffer(body, 'binary').toString('base64'),
data = prefix + base64;
console.log(data);
});
response.on('data', function (chunk) {
if (response.statusCode == 200) body += chunk;
});
});
should also produce a data URI without requiring any external modules.
This works for me using request:
const url = 'http://host/image.png';
request.get({url : url, encoding: null}, (err, res, body) => {
if (!err) {
const type = res.headers["content-type"];
const prefix = "data:" + type + ";base64,";
const base64 = body.toString('base64');
const dataUri = prefix + base64;
}
});
No need for any intermediate buffers. The key is to set encoding to null.