We Keep Coding

Can't get Image data through req.body, I am using postman and sending post request with binary data that is an image

I am using postman and sending a post request with binary data as an image, The image data is obtained in request.body but I am not able to write it in a file.
Code for the same is : fs.createWriteStream('/home/user/Downloads/Documents/4.jpg').write(req.body);
But the data is copied in hexadecimal format and I am not able to open the image.

I'm not sure if the following code helps, but this is what i have, hope you can play-around it and solve your problem
router.post('/api/data/logo/:name',upload.single('logo'), (req, res) => {
var newImg = fs.readFileSync(req.file.path);
var encImg = newImg.toString('base64');
req.body.imageUrl = req.file.path || '';
req.body.contentType = req.file.mimetype || '';
req.body.size = req.file.size || '';
req.body.logo = Buffer(encImg, 'base64') || '';
if (!req.userId) return res.status(405).send(JSON.stringify({error: 'Not logged in'}));
dataController.image(req.userId, req.params.name, req.body, (err, data) => {
if (err) return res.status(500).end();
return res.status(200).send(JSON.stringify({form: data}));
});
});

Nodejs error encoding when get external site's content

I used get method of request module to get content of external site. If encoding of external site is utf-8, it is ok, but it has display error with other encodings such as shift-jis
function getExternalUrl(request, response, url){
mod_request.get(url, function (err, res, body) {
//mod_request.get({uri: url, encoding: 'binary'}, function (err, res, body) {
if (err){
console.log("\terr=" + err);
}else{
var result = res.body;
// Process res.body
response.write(result);
}
response.end();
});
}
How can I get content of external site with correct encoding?

I found the way to do:
Get with binary encoding
var mod_request = require('request');
mod_request.get({ uri: url, encoding: 'binary', headers: headers }, function(err, res, body) {});
Create a Buffer with binary format
var contentBuffer = new Buffer(res.body, 'binary');
Get real encoding of page by detect-character-encoding npm
var mod_detect_character_encoding = require('detect-character-encoding');
var charsetMatch = mod_detect_character_encoding(contentBuffer);
Convert page to utf-8 by iconv npm
var mod_iconv = require('iconv').Iconv;
var iconv = new mod_iconv(charsetMatch.encoding, 'utf-8');
var result = iconv.convert(contentBuffer).toString();
P/S: This way is only applied for text file (html, css, js). Please do not apply for image file or others which is not text

How to download a file from node server(using only node modules, no express, etc)

Im currently writing a handler for a download feature. When the user clicks on the download button from his\her browser the download handler is called, which will then initiate the download(mp3 files only). I had this working on php, but I have since changed everything on my project to Node and I can't seem to get this last part working on Node.
This is the php code I had working before:
<?php
header("Content-Type: application/octet-stream");
header("Content-Disposition: attachment; filename=".($_GET['title']));
readfile($_GET['path']);
?>
This is the the new code for Node:
function download(response, request){
var body = [];
request.on('data', function(chunk) {
body.push(chunk);
});
request.on('end', function() {
body = Buffer.concat(body).toString();
var data = qs.parse(body);
var title = data.songTitle;
var filePath = __dirname + "/../myproject/songs/"+title+".mp3";
fs.open(filePath,'r',function(err, fd){
if(!err){
fs.readFile(fd, function(err, data){
if(!err){
var rs = fs.createReadStream(filePath);
response.writeHead(200, {"Content-Type": "application/octet-stream",
"Content-Disposition": "attachment; filename="+title+".mp3",
"Content-Length" : data.length});
rs.pipe(response);
response.on("end",function(){
fs.close(fd);
});
}else{
console.log("Error while trying to read: ", err);
}
});
}else{
console.log("Error could not open: ", err);
}
});
});
When trying to download, I do not get any errors but nothing happens. I have also tried "audio/mpeg3" for content-type, and nothing. Any ideas on what's going on? I'm trying to do this without using third-party modules.
Please note that the function download in not passed as the callback function of http.createServer(). So the order of response and request is not the issue :)

It looks like you switched request and response. Also, instead of using fs.open()/fs.readFile() to determine file size, you can use fs.stat(), which should be much more resource-friendly (since it doesn't require loading the entire file into memory first):
function download(request, response) {
var body = [];
request.on('data', function(chunk) {
body.push(chunk);
});
request.on('end', function() {
var data = qs.parse(Buffer.concat(body).toString());
var title = data.songTitle;
var filePath = title + '.mp3';
fs.stat(filePath, function(err, stats) {
if (err) {
response.statusCode = 500;
return response.end();
}
response.writeHead(200, {
"Content-Type" : "application/octet-stream",
"Content-Disposition" : "attachment; filename="+title+".mp3",
"Content-Length" : stats.size,
});
fs.createReadStream(filePath).pipe(response);
});
});
}

Error while creating file by fs on Node.js server

I have a Node.js server and I'm trying to parse captcha by uploading bmp image to server. There are two problems I'm facing.
Image is not uploaded correctly. When I try to open Image it gives me error "BMP image has bogus header data".
Currently I'm reading buffer from uploaded data and parsing it to extract captcha string. Captcha string is arbitrary (results shows correct last 3 characters instead of 6).
This is code I'm using:
app.use (function(req, res, next) {
console.log("statrt");
var data="";
req.on('data', function(chunk) {
data += chunk;
});
req.on('end', function() {
console.log("end");
req.body = data;
next();
});
});
var DoneInSync = fibrous(function(buffer){
var val = captcha.getCaptcha(buffer);
console.log("this" + val);
return val;
});
app.post('/', function (req,res){
buffer = new Buffer(req.body);
fs.writeFileSync("captchas_ass.bmp", buffer);
var val = DoneInSync.sync(buffer);
res.write("Yoing -> " + val);
res.end();
});
Captcha parser code runs perfectly, and is tested throughly offline.
Main problem I'm facing is uploading BMP file to server . Once done I can do this
var buf = fs.readFileSync("captcha.bmp");
and pass buf to captcha.getCaptcha(buf) and get result.
Can someone explain where it uploading is wrong?

Your middleware is stringifying the incoming data, which it shouldn't be doing. Try this:
app.use (function(req, res, next) {
var data = [];
req.on('data', function(chunk) {
data.push(chunk);
});
req.on('end', function() {
console.log("end");
req.body = Buffer.concat(data); // `req.body` will be a Buffer
next();
});
});
This would only work if you're uploading the data as a "raw" POST request. If you're using something like multipart/formdata, this won't work either. For that you should use a middleware like multer.

Node.js Server: Image Upload / Corruption Issues

So I'm trying to write a basic file server in Node.js, and all the images I've tried uploading and storing on it are coming back as corrupted. The problem seems to have something to do with the way that Node Buffers handle being converted to UTF-8 and back again (which I have to do in order to get the POST body headers out and away from the binary data).
Here's a simple Node server that shows my current approach and the problems I've been having:
var http = require('http');
var server = http.createServer(function(request, response) {
if (request.method === "GET") {
// on GET request, output a simple web page with a file upload form
var mypage = '<!doctype html><html><head><meta charset="utf-8">' +
'<title>Submit POST Form</title></head>\r\n<body>' +
'<form action="http://127.0.0.1:8008" method="POST" ' +
'enctype="multipart/form-data"> <input name="upload" ' +
'type="file"><p><button type="submit">Submit</button>' +
'</p></form></body></html>\r\n';
response.writeHead(200, {
"Content-Type": "text/html",
"Content-Length": mypage.length
});
response.end(mypage);
} else if (request.method === "POST") {
// if we have a return post request, let's capture it
var upload = new Buffer([]);
// get the data
request.on('data', function(chunk) {
// copy post data
upload = Buffer.concat([upload, chunk]);
});
// when we have all the data
request.on('end', function() {
// convert to UTF8 so we can pull out the post headers
var str = upload.toString('utf8');
// get post headers with a regular expression
var re = /(\S+)\r\nContent-Disposition:\s*form-data;\s*name="\w+";\s*filename="[^"]*"\r\nContent-Type: (\S+)\r\n\r\n/i,
reMatch = str.match(re);
var lengthOfHeaders = reMatch[0].length,
boundary = reMatch[1],
mimeType = reMatch[2];
// slice headers off top of post body
str = str.slice(lengthOfHeaders);
// remove the end boundary
str = str.replace("\r\n" + boundary + "--\r\n", '');
// convert back to buffer
var rawdata = new Buffer(str, 'utf8');
// echo back to client
response.writeHead(200, {
"Content-Type": mimeType
});
response.end(rawdata);
});
}
});
server.listen(8008);
console.log("server running on port 8008");
To test it, run the script in node and go to 127.0.0.1:8008 in your browser. Try uploading an image and submitting the form. The image comes back as corrupt every time -- even though the script should just be directly echoing the image data back to the browser.
So does anyone know what I'm doing wrong here? Is there a better way to handle POST body headers in Node that I haven't figured out yet? (And before anyone says anything, no, I don't want to use Express. I want to figure out and understand this problem.)

The problem seems to have something to do with the way that Node Buffers handle being converted to UTF-8 and back again
I guess you are right about that, convert to UTF-8 is a bad idea, but can do it just to work with the file and get the headers and boundaries positions, but keep the buffer file untouched, and when you have all the positions to get the header and boundary out of the file just copy the buffer to a new buffer like that
originalBuffer.copy(newBuffer,0, positionHeader, positionEndBoundary)
var http = require('http');
var fs = require('fs');
var connections = 0;
var server = http.createServer(function (req, res) {
connections++;
console.log(req.url,"connections: "+connections);
if(req.url == '/'){
res.writeHead(200, { 'content-type': 'text/html' });
res.end(
'<form action="/upload" enctype="multipart/form-data" method="post">' +
'<input type="file" name="upload" multiple="multiple"><br>' +
'<input type="submit" value="Upload">' +
'</form>'
);
}
var body = new Buffer([]);
if (req.url == '/upload') {
req.on('data', function (foo) {
//f.write(foo);
body = Buffer.concat([body,foo]);
if(isImage(body.toString())){
console.log("é imagem do tipo "+isImage(body.toString()));
}
else{
console.log("Não é imagem");
res.end("Não é imagem");
}
console.log(body.length, body.toString().length);
});
req.on('end', function () {
// console.log(req.headers);
//I converted the buffer to "utf 8" but i kept the original buffer
var str = body.toString();
console.log(str.length);
imageType = isImage(body.toString());
//get the index of the last header character
//I'm just using the string to find the postions to cut the headers and boundaries
var index = str.indexOf(imageType)+(imageType+"\r\n\r\n").length;
// var headers= str.slice(0,index).split(';');
// console.log(headers);
//Here comes the trick
/*
*I have to cut the last boundaries, so i use the lastIndexOf to cut the second boundary
* And maybe that is the corruption issues, because, I'm not sure, but I guess
* the UTF-8 format only use 7bits to represent all characters, and the buffer can use 8bits, or two hex,
*So, i need to take the difference here (body.length-str.length)
*/
var indexBoundayToBuffer = str.lastIndexOf('------WebKitFormBoundary')+(body.length-str.length);
console.log(index, indexBoundayToBuffer);
//maybe you can change this to use less memory, whatever
var newBuffer = Buffer.alloc(body.length);
/*
*And now use the index, and the indexBoudayToBuffer and you will have only the binary
*/
body.copy(newBuffer,0,index,indexBoundayToBuffer);
// f.end();
//file type
var type = imageType.substr("image/".length);
console.log("END");
fs.writeFile("nameFile."+type,newBuffer,function(err,ok){
if(err){
console.log(err);
return false;
}
res.end();
});
});
}
});
function isImage(str){
if(str.indexOf('image/png')!=-1) return 'image/png';
else if(str.indexOf('image/jpeg')!=-1) return 'image/jpeg';
else if(str.indexOf('image/bmp'!=-1)) return 'image/bmp';
else if(str.indexOf('image/gif'!=-1)) return 'image/gif';
else false;
}
var port = process.env.PORT || 8080;
server.listen(port, function () {
console.log('Recording connections on port %s', port);
});

You really shouldn't use regular expressions like that to parse multipart payloads as it can easily make trying to parse your image data very unreliable. There are modules on npm that parse forms for you such as busboy, multiparty, or formidable. None of them use regular expressions and they don't require Express.