We Keep Coding

NodeJS: Failed to fetch image every 6th attempt

I am trying to build a form for hotel owners. They can submit main images from the hotel, and later on images from the different room types. As soon as a user selects images with the file browser in the file input, they get uploaded automatically and saved on the server. When they submit the whole form, the images are uploaded to Imagekit and the link is saved in a database.
I want to show a progress bar while the images are uploading, and show a small thumbnail of the image when it is finished. In addition, there is a delete button to delete the images from the server if the user made a mistake.
So far so good, everything works fine, but after uploading 5 images (bulk or one by one), the 6th "get thumbnail" fetch method fails. I cannot upload or delete any other image anymore. When I try to reload the page, the error shows in the console (as I print it out for test purposes), and then I can proceed to upload 5 images again or delete others, until it occurs again. I have not defined a limit in association with the number "5" (e.g. a for loop) and I also have not defined a file size limit (e.g. 10MB which might be filled after the 5th picture).
TypeError: Failed to fetch
at getThumbnail (hotel:1144:13)
at XMLHttpRequestUpload.<anonymous> (hotel:1127:21)
However, the 6th image is still uploaded to the server, with the right file name, in the right directory.
This is my code to pick the images from the input and rename them:
function getFiles(roomId) {
const fileInput = document.getElementById('images'+roomId)
const dropForm = document.getElementById('drop-form'+roomId)
dropForm.addEventListener('click', () => {
fileInput.click();
fileInput.onchange = ({target}) => {
for(file of target.files) {
let backendFileName = `${Date.now().toString()}---${file.name.replace(/\s+/g, '')}`
if(file && !fileArray.includes(backendFileName.split('---')[1]+roomId)) {
// shorten filename if too long
let frontendFileName = file.name;
if(frontendFileName.length >= 20) {
let splitName = frontendFileName.split('.');
frontendFileName = splitName[0].substring(0, 12) + "... ." + splitName[1]
}
uploadFile(file, backendFileName, frontendFileName, roomId);
} else {
console.log('File not existing or already uploaded')
}
}
fileInput.value = null
}
})
}
The variable frontendFileName is irrelevant for the backend, its only there to shorten the file name if its too long. I distinguish the files by adding a Date.now() with 3 dashes (---) in front of the name.
I have defined a fileArray to check whether the images has already been uploaded. I am sure that this is not causing the problem, as I already tried removing it entirely from all the functionality.
This is my function to upload the images directly to the server and display the progress area:
function uploadFile(file, backendFileName, frontendFileName, roomId) {
let formData = new FormData()
var progressArea = document.getElementById('progress-area'+roomId)
var uploadedArea = document.getElementById('uploaded-area'+roomId)
formData.append('images', file)
fileArray.push(backendFileName.split('---')[1]+roomId)
let xhr = new XMLHttpRequest();
xhr.open("POST", "/images/upload/"+roomId+"/"+backendFileName, true);
xhr.upload.addEventListener('progress', ({loaded, total}) => {
let fileLoaded = Math.floor((loaded / total) * 100)
let fileTotal = Math.floor(total / 1000)
let fileSize;
fileTotal < 1024 ? fileSize = fileTotal + " KB" : fileSize = (loaded / (1024 * 1024)).toFixed(2) + " MB"
let progressHTML = `<li class="row">
<i class="fas fa-file-image"></i>
<div class="content">
<div class="details">
<span class="name">${frontendFileName} • Uploading</span>
<span class="percent">${fileLoaded} %</span>
</div>
<div class="progress-bar">
<div class="progress" style="width: ${fileLoaded}%"></div>
</div>
</div>
</li>`;
progressArea.innerHTML = progressHTML;
if(loaded == total) {
progressArea.innerHTML = "";
let uploadedHTML = `<li class="row" id="uploaded_${backendFileName}">
<div class="content">
<img class="thumbnail" id="img${backendFileName}">
<div class="details">
<span class="name">${frontendFileName} • Uploaded</span>
<span class="size">${fileSize}</span>
</div>
</div>
<div class="icons">
<i style="cursor: pointer;" class="far fa-trash-alt" id="delImg_${backendFileName}"></i>
</div>
</li>`;
uploadedArea.insertAdjacentHTML('afterbegin', uploadedHTML)
// get thumbnail from server
getThumbnail(backendFileName, roomId);
// add functionality to delete button
document.getElementById('delImg_'+backendFileName).onclick = () => {
deleteImage(backendFileName, roomId)
}
}
})
if(formData) {
xhr.send(formData);
}
}
I save the images via multer. These are my multer properties and my router function to save the images:
const storage = multer.diskStorage({
destination: (req, file, cb) => {
var dir = path.join(__dirname, '../public/images/', req.session.passport.user, req.params.directory)
if(!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true }, err => cb(err, dir))
}
cb(null, dir)
},
filename: (req, file, cb) => {
cb(null, req.params.fileName);
}
})
const upload = multer({
storage: storage
});
router.post('/upload/:directory/:fileName', upload.any('images'), (req, res) => {
if(req.files) {
console.log('Uploading image ' + req.params.fileName + ' to room directory ' + req.params.directory)
} else {
console.log('No images to upload')
}
})
And my function to get back the thumbnail from the server:
function getThumbnail(backendFileName, directory) {
console.log('Getting thumbnail')
fetch('/images/thumbnail/'+directory+"/"+backendFileName)
.then(response => {
console.log('got thumbnail')
response.json()
.then(data => {
document.getElementById('img'+backendFileName).src = "data:image/png;base64, "+data.image
console.log(data.message)
})
})
.catch(err => {
console.log(err)
})
}
Last but not least, my router function for finding and sending back the thumbnail:
router.get('/thumbnail/:directory/:fileName', (req, res) => {
const dirPath = path.join(__dirname, '../public/images/', req.session.passport.user, req.params.directory, req.params.fileName)
fs.readFile(dirPath, { encoding: 'base64' }, (err, data) => {
if(err) {
console.error(err)
res.send({'error': err})
}
if(data) {
console.log('Sending thumbnail')
res.json({ 'image':data , 'message':'image found'});
} else {
res.json({ 'message': 'no image found'})
}
})
})
I also checked the backend, the upload function works as the image is saved on the server, but the thumbnail function is not receiving a request from the sixth image.
I really need help on this one as it is confusing me for a week now.
Cheers!

Your upload middleware (see below) does not contain a res.end statement or similar, therefore the browser will never receive a response to the upload request. This means that for every new upload attempt by a given user, their browser must open a new parallel connection to your server.
And (here's where the number 5 comes in) browsers have a limit on the number of parallel HTTP/1.1 connections they can make to one server, and this limit might well be 5.
router.post('/upload/:directory/:fileName', upload.any('images'), (req, res) => {
if(req.files) {
console.log('Uploading image ' + req.params.fileName + ' to room directory ' + req.params.directory)
} else {
console.log('No images to upload')
}
res.end(); // add this line to your code
})

Pass data from res.write() to formidable filename

I use node to allow users to upload a file:
var http = require('http');
var formidable = require('formidable');
var fs = require('fs');
http.createServer(function (req, res) {
if (req.url == '/fileupload') {
var form = new formidable.IncomingForm();
form.parse(req, function (err, fields, files) {
var oldpath = files.filetoupload.path;
var newpath = 'team_1_uploads/' + files.filetoupload.name + files.filetoupload.token;
fs.rename(oldpath, newpath, function (err) {
if (err) throw err;
res.write(' FILE UPLOADED!');
res.end();
});
});
} else {
res.writeHead(200, {'Content-Type': 'text/html'});
res.write('<div align="center">');
res.write('<html>');
res.write('<body>');
res.write('<form action="fileupload" method="post" enctype="multipart/form-data">');
res.write('<input id="file_input" type="file" name="filetoupload" style="margin-left: 100px; margin-bottom: 10px; color: transparent"><br>');
res.write('<input id="submit_button" type="submit">');
res.write('</form>');
res.write('</body>');
res.write('</html>');
res.write('</div>');
return res.end();
}
}).listen(3131);
As you can see, I am trying to append a token onto the filename. The upload button gets served by node through an iframe on the front-end. I can pass the token to res.write() using postMessage, by adding the following script to res.write():
res.write("<script>window.addEventListener('message', function(event) { document.getElementById('file_input').dataset.token = event.data; global_hold_token = event.data; })</script>");
This sets the token to the form element by using the data attribute on the form element. The message is received from the front end by using postMessage:
$('#my_frame')[0].contentWindow.postMessage(token, '*')
I thought I could then parse the data attribute using formidable. But node doesn't seem able to access the data attribute on the form element, even though it can access the name.

The ?token=xxxxxx is a GET parameter. You can get these from the req object, under the req.query object
http://expressjs.com/en/api.html#req.query
So in your case, it will be in req.query.token

How can I get a req.body from a form that is not undefined?

Whenever I submit a form with information it is returned as undefined. I have posted the code below. If I include the (enctype="multipart/form-data") in my form I dont receive anything for the body (req.body). However, if I dont include it I receive a body but the file processing does not work and the page just keeps loading.
app.post('/processupload', function(req, res) {
var date = new Date();
titles.push(req.body.pTitle);
descriptions.push(req.body.postDescription);
dates.push(date.toString());
file_names.push(req.body.fUpload);
console.log(req);
var form = new formidable.IncomingForm();
form.parse(req, function(err, fields, files)
{
if(err) return res.redirect(303, '/error');
});
form.on('end', function(fields, files)
{
var temp_path = this.openedFiles[0].path;
var file_name = this.openedFiles[0].name;
var new_location = __dirname + '/public/images/';
fs.copy(temp_path, new_location + file_name);
res.redirect(303, 'home');
});
});
<form action="/processupload" enctype="multipart/form-data" method="POST" id="uploadForm" name="postForm">
<p align="center" id="pUploadForm" name="pPostForm"><label for="photoTitle">Photo Title: </label>
<input type="text" id="photoTitle" name="pTitle"><br>
<br><input type="file" id="fileUpload" name="fUpload"><br>
<br><label for="photoCaption">Photo Caption: </label><br>
<textarea rows="10" cols="50" id="photoCaption" name="postDescription"></textarea><br><br>
</p>
</form>

I created a project few weeks back that had photo upload. I used Angular and Node. But it should still work without Angular only using Node. I used multer npm package.
var AWS = require('aws-sdk');
var s3 = new AWS.S3();
var multer = require('multer');
var upload = multer({ storage: multer.memoryStorage() }); //Save photo in memory
router.post('/processupload', upload.single('photo'), function(req, res, next){
var bucketName = process.env.BUCKET_NAME;
var file = req.file;
var filename = file.originalname;
var ext = _.last(filename.split('.'))
var keyName = uuid.v4() + '.' + ext;
var url = process.env.AWS_URL + bucketName + '/' + keyName;
var params = { Bucket: bucketName, Key: keyName, Body: file.buffer, ACL: 'public-read' };
s3.putObject(params, function(err, data) {
if (err){
return res.status(400).send(err)
} else{
console.log("Successfully uploaded data to myBucket/myKey");
console.log("The URL is", url);
res.send(url)
}
});
});
This helped me uploading images then gives me back the image url from the S3 Bucket. But you can handle that file as you want. Multer allows you to access to req.file so you can do whatever you need to do, in this example I created a unique id in order to get a url to send back to the front-end and therefore use it a source somehow. This is a form working with this code:
<form action="/testupload" method='post' enctype='multipart/form-data'>
<input type="file" name="photo" id="photo" multiple=false>
<button type="submit">Submit</button>
</form>
Something important that took a long time to debug though the name="photo" in the form must be reflected by the upload.single('photo') middleware. I hope this helps, there are so many ways go around this, this is just one.
Sources:
https://www.npmjs.com/package/multer
http://docs.aws.amazon.com/AmazonS3/latest/UG/UploadingObjectsintoAmazonS3.html

How to save Data in MongoDB from forms

I have a form. I need to get text from my form to save it in MongoDB.
tweets.ejs:
<form method="post" action="/tweets">
<input type="text" id="txt" name="text"/>
<input type="button" id="btn" value="Touch me">
</form>
Here is my route file tweets.js:
var Message = require('models/messages');
exports.get = function(req,res) {
res.render('tweets')
};
I use mongoose schema(models/messages.js):
var mongoose = require('../libs/mongoose'),
Schema = mongoose.Schema;
var MessageSchema = new Schema({
message: String,
date: Date
});
var Message = mongoose.model('Message', MessageSchema);
module.exports = Message;
I tried set var m = req.body.text in tweets.js, but I think it's absolutely wrong way
exports.post = function(req,res){
var m = new Message;
m.message = req.body.text;
m.save(); }
Explain me how to do it right please!

in your routes or app file route should be
var tweets = require("tweets");
app.post("/tweets", tweets.post);
in your tweets.js file
var Message = require('models/messages');
exports.post = function(req,res){
console.log(req.body.text)
var msg = {message:req.body.text,date:new Date()};
Message(msg).save(function(error,data){
if (data){
console.log("Save "+ JSON.stringify(data));
res.send({statud:"OK",msg:data})
}
else{
res.send({statud:"Cancel"})
}
});
}

Express auth app not hangs on form submit

I am trying an Example from Jump start node.js(chapterote 1 Authentication).I wrote all the program and created all the files and folders witch is needed for chapter 1.For those who dont know Chapter 1 is about using mongolab cloud Service.
form.html
<form action="/signup" method="post">
<div>
<label>Username:</label>
<input type="text" name="username"/><br/>
</div>
<div>
<label>Password:</label>
<input type="password" name="password"/>
</div>
<div><input type="submit" value="Sign Up"/></div>
</form>`
lib/db.js
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
module.exports.mongoose = mongoose;
module.exports.Schema = Schema;
// Connect to cloud database
var username = "user"
var password = "password";
var address = ' #dbh42.mongolab.com:27427/nockmarket';
connect();
// Connect to mongo
function connect() {
var url = 'mongodb://' + username + ':' + password + address;
console.log('[*] not reaching here');
mongoose.connect(url);
}
function disconnect() {mongoose.disconnect()}
models/User.js
var db = require('../lib/db');
var UserSchema = new db.Schema({
username : {type: String, unique: true}
, password : String
})
var MyUser = db.mongoose.model('User', UserSchema);
// Exports
module.exports.addUser = addUser;
// Add user to database
function addUser(username, password, callback) {
var instance = new MyUser();
instance.username = username;
instance.password = password;
instance.save(function (err) {
if (err) {
callback(err);
}
else {
callback(null, instance);
}
});
}
When I sumbits the form app hangs and its not calling connect() function witch connects to the mongolab it just waits for finish but nothing happens.
thanks,