NodeJS: Failed to fetch image every 6th attempt - javascript

I am trying to build a form for hotel owners. They can submit main images from the hotel, and later on images from the different room types. As soon as a user selects images with the file browser in the file input, they get uploaded automatically and saved on the server. When they submit the whole form, the images are uploaded to Imagekit and the link is saved in a database.
I want to show a progress bar while the images are uploading, and show a small thumbnail of the image when it is finished. In addition, there is a delete button to delete the images from the server if the user made a mistake.
So far so good, everything works fine, but after uploading 5 images (bulk or one by one), the 6th "get thumbnail" fetch method fails. I cannot upload or delete any other image anymore. When I try to reload the page, the error shows in the console (as I print it out for test purposes), and then I can proceed to upload 5 images again or delete others, until it occurs again. I have not defined a limit in association with the number "5" (e.g. a for loop) and I also have not defined a file size limit (e.g. 10MB which might be filled after the 5th picture).
TypeError: Failed to fetch
at getThumbnail (hotel:1144:13)
at XMLHttpRequestUpload.<anonymous> (hotel:1127:21)
However, the 6th image is still uploaded to the server, with the right file name, in the right directory.
This is my code to pick the images from the input and rename them:
function getFiles(roomId) {
const fileInput = document.getElementById('images'+roomId)
const dropForm = document.getElementById('drop-form'+roomId)
dropForm.addEventListener('click', () => {
fileInput.click();
fileInput.onchange = ({target}) => {
for(file of target.files) {
let backendFileName = `${Date.now().toString()}---${file.name.replace(/\s+/g, '')}`
if(file && !fileArray.includes(backendFileName.split('---')[1]+roomId)) {
// shorten filename if too long
let frontendFileName = file.name;
if(frontendFileName.length >= 20) {
let splitName = frontendFileName.split('.');
frontendFileName = splitName[0].substring(0, 12) + "... ." + splitName[1]
}
uploadFile(file, backendFileName, frontendFileName, roomId);
} else {
console.log('File not existing or already uploaded')
}
}
fileInput.value = null
}
})
}
The variable frontendFileName is irrelevant for the backend, its only there to shorten the file name if its too long. I distinguish the files by adding a Date.now() with 3 dashes (---) in front of the name.
I have defined a fileArray to check whether the images has already been uploaded. I am sure that this is not causing the problem, as I already tried removing it entirely from all the functionality.
This is my function to upload the images directly to the server and display the progress area:
function uploadFile(file, backendFileName, frontendFileName, roomId) {
let formData = new FormData()
var progressArea = document.getElementById('progress-area'+roomId)
var uploadedArea = document.getElementById('uploaded-area'+roomId)
formData.append('images', file)
fileArray.push(backendFileName.split('---')[1]+roomId)
let xhr = new XMLHttpRequest();
xhr.open("POST", "/images/upload/"+roomId+"/"+backendFileName, true);
xhr.upload.addEventListener('progress', ({loaded, total}) => {
let fileLoaded = Math.floor((loaded / total) * 100)
let fileTotal = Math.floor(total / 1000)
let fileSize;
fileTotal < 1024 ? fileSize = fileTotal + " KB" : fileSize = (loaded / (1024 * 1024)).toFixed(2) + " MB"
let progressHTML = `<li class="row">
<i class="fas fa-file-image"></i>
<div class="content">
<div class="details">
<span class="name">${frontendFileName} • Uploading</span>
<span class="percent">${fileLoaded} %</span>
</div>
<div class="progress-bar">
<div class="progress" style="width: ${fileLoaded}%"></div>
</div>
</div>
</li>`;
progressArea.innerHTML = progressHTML;
if(loaded == total) {
progressArea.innerHTML = "";
let uploadedHTML = `<li class="row" id="uploaded_${backendFileName}">
<div class="content">
<img class="thumbnail" id="img${backendFileName}">
<div class="details">
<span class="name">${frontendFileName} • Uploaded</span>
<span class="size">${fileSize}</span>
</div>
</div>
<div class="icons">
<i style="cursor: pointer;" class="far fa-trash-alt" id="delImg_${backendFileName}"></i>
</div>
</li>`;
uploadedArea.insertAdjacentHTML('afterbegin', uploadedHTML)
// get thumbnail from server
getThumbnail(backendFileName, roomId);
// add functionality to delete button
document.getElementById('delImg_'+backendFileName).onclick = () => {
deleteImage(backendFileName, roomId)
}
}
})
if(formData) {
xhr.send(formData);
}
}
I save the images via multer. These are my multer properties and my router function to save the images:
const storage = multer.diskStorage({
destination: (req, file, cb) => {
var dir = path.join(__dirname, '../public/images/', req.session.passport.user, req.params.directory)
if(!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true }, err => cb(err, dir))
}
cb(null, dir)
},
filename: (req, file, cb) => {
cb(null, req.params.fileName);
}
})
const upload = multer({
storage: storage
});
router.post('/upload/:directory/:fileName', upload.any('images'), (req, res) => {
if(req.files) {
console.log('Uploading image ' + req.params.fileName + ' to room directory ' + req.params.directory)
} else {
console.log('No images to upload')
}
})
And my function to get back the thumbnail from the server:
function getThumbnail(backendFileName, directory) {
console.log('Getting thumbnail')
fetch('/images/thumbnail/'+directory+"/"+backendFileName)
.then(response => {
console.log('got thumbnail')
response.json()
.then(data => {
document.getElementById('img'+backendFileName).src = "data:image/png;base64, "+data.image
console.log(data.message)
})
})
.catch(err => {
console.log(err)
})
}
Last but not least, my router function for finding and sending back the thumbnail:
router.get('/thumbnail/:directory/:fileName', (req, res) => {
const dirPath = path.join(__dirname, '../public/images/', req.session.passport.user, req.params.directory, req.params.fileName)
fs.readFile(dirPath, { encoding: 'base64' }, (err, data) => {
if(err) {
console.error(err)
res.send({'error': err})
}
if(data) {
console.log('Sending thumbnail')
res.json({ 'image':data , 'message':'image found'});
} else {
res.json({ 'message': 'no image found'})
}
})
})
I also checked the backend, the upload function works as the image is saved on the server, but the thumbnail function is not receiving a request from the sixth image.
I really need help on this one as it is confusing me for a week now.
Cheers!

Your upload middleware (see below) does not contain a res.end statement or similar, therefore the browser will never receive a response to the upload request. This means that for every new upload attempt by a given user, their browser must open a new parallel connection to your server.
And (here's where the number 5 comes in) browsers have a limit on the number of parallel HTTP/1.1 connections they can make to one server, and this limit might well be 5.
router.post('/upload/:directory/:fileName', upload.any('images'), (req, res) => {
if(req.files) {
console.log('Uploading image ' + req.params.fileName + ' to room directory ' + req.params.directory)
} else {
console.log('No images to upload')
}
res.end(); // add this line to your code
})

Related

Flask-wtf with trix-editor - How do I upload images

I'm using Flask with one of my wtforms TextAreaFields mapped to Trix-Editor. All works well except for images using the built toolbar attach button.
I'd like to save the images to a directory on the backend and have a link to it in the trix-editor text. I'm saving this to a database.
I can make this work by adding an <input type='file'/>in my template like so:
{{ form.description }}
<trix-editor input="description"></trix-editor>
<input type="file"/>
and the following javascript which I found somewhere as an example.
document.addEventListener('DOMContentLoaded', ()=> {
let contentEl = document.querySelector('[name="description"]');
let editorEl = document.querySelector('trix-editor');
document.querySelector('input[type=file]').addEventListener('change', ({ target })=> {
let reader = new FileReader();
reader.addEventListener('load', ()=> {
let image = document.createElement('img');
image.src = reader.result;
let tmp = document.createElement('div');
tmp.appendChild(image);
editorEl.editor.insertHTML(tmp.innerHTML);
target.value = '';
}, false);
reader.readAsDataURL(target.files[0]);
});
// document.querySelector('[role="dump"]').addEventListener('click', ()=> {
// document.querySelector('textarea').value = contentEl.value;
// });
});
This saves the image embedded in the text. I don't want that because large images will take up a lot of space in the database and slow down loading of the editor when I load this data back into it from the database.
It is also ugly having the extra button when Trix has an attachment button in it's toolbar. So, I'd like to be able to click the toolbar button and have it upload or if that is too hard, have the built in toolbar button save the image embedded.
To save the images to a folder instead of embedded, the Trix-editor website says to use this javascript https://trix-editor.org/js/attachments.js
In this javascript I have to provide a HOST so I use
var HOST = "http://localhost:5000/upload/"
and I set up a route in my flask file:
#tickets.post('/_upload/')
def upload():
path = current_app.config['UPLOAD_DIRECTORY']
if request.method == 'POST':
if 'file' not in request.files:
flash('No file part')
return redirect(request.url)
file = request.files['file']
if file.filename == '':
flash('No selected file')
return redirect(request.url)
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
session["id"] = filename
file.save(os.path.join(path, filename))
return send_from_directory(path, filename)
I can select an image and it shows in the editor and it uploads to the directory on my backend as expected. But when I save the form the location of the image is not in in the document text (should be in there as something like <img src="uploads/image.png>
On the python console I see
"POST /_upload/ HTTP/1.1" 404 -
I can make this go away if I change the return on my route to something like return "200" But all the examples I have seen about uploading files have this or a render_template. I don't want to render a template so I'm using this although I don't really understand what it does.
I'm assuming I need to return something the javasript can use to embed the image link in the document. But I'm a total newbie (like you didn't figure that out already) so I don't know what to do for the return statement (assuming this is where the problem lies).
If anyone else is trying to figure this out this is what I ended up doing.
Still needs a but of tweaking but works.
First I modified the example javascript for uploading to use Fetch instead of XMLHttpRequest
const editor = document.querySelector('trix-editor');
(function() {
HOST = '/_upload/'
addEventListener("trix-attachment-add", function(event) {
if (event.attachment.file) {
uploadFileAttachment(event.attachment)
}
// get rid of the progress bar as Fetch does not support progress yet
// this code originally used XMLHttpRequest instead of Fetch
event.attachment.setUploadProgress(100)
})
function uploadFileAttachment(attachment) {
uploadFile(attachment.file, setAttributes)
function setAttributes(attributes) {
attachment.setAttributes(attributes)
alert(attributes)
}
}
function uploadFile(file, successCallback) {
var key = createStorageKey(file)
var formData = createFormData(key, file)
fetch(HOST, {method: 'POST', body: formData}).then(function(response){
response.json().then(function(data){
alert(data.file, data.status)
if (data.status == 204) {
var attributes = {
url: HOST + key,
href: HOST + key + "?content-disposition=attachment"
}
console.log(attributes)
successCallback(attributes)
}
})
})
}
function createStorageKey(file) {
var date = new Date()
var day = date.toISOString().slice(0,10)
var name = date.getTime() + "-" + file.name
return [day, name ].join("/")
}
function createFormData(key, file) {
var data = new FormData()
data.append("key", key)
data.append("Content-Type", file.type)
data.append("file", file)
return data
}
})();
Then modified my Flask route (which I'll refactor, this was just slapped together to make it work):
def upload():
path = current_app.config['UPLOAD_DIRECTORY']
new_path = request.form["key"].split('/')[0]
file_upload_name = os.path.join(path, request.form["key"])
print(file_upload_name)
upload_path = os.path.join(path, new_path)
if request.method == 'POST':
if 'file' not in request.files:
flash('No file part')
return redirect(request.url)
file = request.files['file']
if file.filename == '':
flash('No selected file')
return redirect(request.url)
if file and allowed_file(file.filename):
if not os.path.exists(upload_path):
os.mkdir(upload_path)
filename = secure_filename(file.filename)
session["id"] = filename
attachment = os.path.join(upload_path, filename)
file.save(attachment)
file.close()
os.rename(attachment, file_upload_name)
print(os.listdir(upload_path))
return jsonify({'file': attachment, 'status': 204})
return f'Nothing to see here'
Anyway, I hope that helps as it took me ages to figure out.

running function more than one times returns "error: Can't set headers after they are sent"

Program runs as it should until I try to use the function once again and it returns the error: Can't set headers after they are sent.
I'm creating this app to familiarize myself with nodejs and javascript and I've been reading about the error and it seems to be an issue when sending more than one response to a request. I started using res.setHeader before knowing this, but I read that res.header could avoid this problem, it didn't solve it but I kept it.
HTML:
<!doctype html>
<head>
<link rel="stylesheet" type="text/css" href="youtube2music.css">
<title>
Youtube2Music
</title>
</head>
<body>
<div id="cabeza">
<h1>
Youtube 2 Music
</h1>
<p>
just paste your link below and download your song.
</p>
</div>
<div id="down-part">
<input id="myUrl" class='myUrl-input'>
</input>
<button type="button" class="download_button">
Download
</button>
</div>
</body>
<script src='youtube2music.js'></script>
</html>
Javascript:
var urlinput = document.querySelector('.myUrl-input'); // gets url inputbox
var button = document.querySelector('.download_button'); // gets download button
button.addEventListener('click', () => {
console.log(urlinput.value); // prints in console the url
sendUrl(urlinput.value); // sends url to function to start the request
});
// function to make requst
function sendUrl(URL){
window.location.href = `http://localhost:4000/?URL=${URL}`; // makes the video request to nodejs server
}
index.js < node file:
var eventEmitter = new events.EventEmitter();
var sfn;
appi.use(cors());
const {app, BrowserWindow} = require('electron')
function createWindow(){
let win = new BrowserWindow({width:800, height:600});
win.loadFile('index.html');
}
app.on('ready', createWindow)
appi.listen(4000, () => {
console.log('server at port 4000');
});
appi.get('/',(req,res)=>{
var URL = req.query.URL;
ytdl.getInfo(URL, function(err,info){
if(err) throw err;
var songTitle = info.title;
sfn = filenamify(songTitle);
eventEmitter.emit('name_ready');
});
var startDownload = function(){
let stream = ytdl(URL, {
quality: 'highestaudio',
});
res.header('Content-Disposition', 'attachment; filename=' + sfn + '.mp3');
res.header('Content-type', 'audio/mpeg');
proc = new ffmpeg({source: stream})
proc.withAudioCodec('libmp3lame').toFormat('mp3').output(res).run();
}
eventEmitter.on('name_ready', startDownload);
})
as it is works for the first input but asking for another output results in error, why is it really returning this error and how can it be avoided?
There are several problems with your current setup:
Try not to use event emitter for signaling events within an HTTP request, it wasn't made for this.
With HTTP requests, try not to use global variables for data received during the request, when two requests come in at the same time, they may get confused and get sent the wrong data.
appi.listen(4000, () => {
console.log('server at port 4000');
});
appi.get('/', (req,res)=> {
const { URL } = req.query;
ytdl.getInfo(URL, (err,info) => {
if(err) throw err;
const songTitle = info.title;
const sfn = filenamify(songTitle);
let stream = ytdl(URL, {
quality: 'highestaudio',
});
res.set('Content-Disposition', 'attachment; filename=' + sfn + '.mp3');
res.set('Content-type', 'audio/mpeg');
const proc = new ffmpeg({source: stream})
proc.withAudioCodec('libmp3lame').toFormat('mp3').output(res).run();
});
})

Convert html to canvas and download as zip file

I am trying to download different sections of a page as jpeg. There are two ways I'm going about it; One is to include a download button in every section and when it is clicked, the section is downloaded as jpeg; The other is to include a button atop the page and when it is clicked, all the sections are downloaded.
The download section by section code works well but the issue arises when I try to do the download all option, It downloads files of type file instead of jpeg pictures.
When I logged the url I'm supposed to download from, I find out that it is empty but it isn't inside the html2canvas function.
I am using html2canvas to convert html to canvas and JSZip to zip it.
function urlToPromise(url) {
return new Promise(function(resolve, reject) {
JSZipUtils.getBinaryContent(url, function (err, data) {
if(err) {
reject(err);
} else {
resolve(data);
console.log(data);
}
});
});
}
function getScreen(){
var caption = $('#caption-input').val();
var allSections = $("#content").children().unbind();
var allSectionsArray = $.makeArray(allSections);
console.log(allSectionsArray);
var zip = new JSZip(); //Instantiate zip file
var url = "";
for(var i = 0; i < allSectionsArray.length; i++){
console.log("Currently at " + allSectionsArray[i].id);
var queryId = allSectionsArray[i].id.toString();
html2canvas(document.querySelector("#"+queryId)).then(function(canvas) {
$("#blank").attr('href',canvas.toDataURL('image/jpeg', 1.0));
$("#blank").attr('download',caption + ".jpeg");
//$("#blank")[0].click();
url = $("#blank").attr('href');
console.log(url);
});
console.log(url);
var filename = "image " + (i+1);
zip.file(filename, urlToPromise(url),{binary:true}); //Create new zip file with filename and content
console.log('file ' + (i+1) + ' generated');
console.log(filename+ "\n" + url);
}
//Generate zip file
generateZipFile(zip);
}
function generateZipFile(zip){
zip.generateAsync({type:"blob"})
.then(function callback(blob) {
saveAs(blob, "example.zip");
console.log("zip generated");
});
}

Upload images to uploads folder _ JavaScript

Users will be able to either send a text post(input type="text") or image post(input type="file"). But they won't be sending both
Here's my form (in Jade):
form#addPost(action="/uploads", method="post", placeholder='Add your ideas here...')
input#postinput(type="text", name="contents" placeholder="Add your ideas here...")
div.privacy(class="onoffswitch", id='privacytog')
input(type="checkbox" name="onoffswitch" class="onoffswitch-checkbox" id="myonoffswitch" checked)
label(class="onoffswitch-label" for="myonoffswitch")
span(class="onoffswitch-inner")
span(class="onoffswitch-switch")
<div style="height:0px;overflow:hidden">
<input type="file" id="fileInput" name="fileInput" accept="image/*">
</div>
input#submit1(type="submit", value="Post")
And Here's my app.js(server-side code)
var util = require("util");
var fs = require("fs");
var bodyParser = require('body-parser');
var multer = require("multer");
app.use(multer({
dest: "./public/uploads/"
}));
app.post("/uploads", function(req, res) {
var data = req.body;
var id = req.user._id;
var username = req.user.username;
var date = Date();
var onOff = false;
if (req.body.onoffswitch) {
onOff = true;
}
//Images upload to uploads folder
if (req.files) {
console.log(util.inspect(req.files));
if (req.files.fileInput.size === 0) {
return next(new Error("Hey, first would you select a file?"));
}
fs.exists(req.files.fileInput.path, function(exists) {
if(exists) {
res.end("Got your file!");
} else {
res.end("Well, there is no magic for those who don’t believe in it!");
}
});
}
User.findById(id, function(err, user) {
if (err) return handleErr(err);
var uid = shortid.generate();
newPost = {
//If sending down an Image use data.fileInput not contents
contents: [data.contents || '/img/'+data.fileInput],
_id: uid,
privacy: onOff,
username: req.user.username,
date: date,
rating: Number(0),
uwv: []
};
user.posts.push(newPost);
user.save(function(err, user){
if(err) return handleErr(err);
if(newPost.privacy === 'false'){
for (var i = 0; i < user.followers.length; i++) {
User.findOne({username:user.followers[i]}, function(err, follower){
follower.discover.push(newPost)
follower.save();
});
}
}
});
});
}
Images are being uploaded and saved to uploads folder. However when posting just a text post(only filling in input type="text") it keeps throwing back the error: Cannot read property 'size' of undefined
Typically browsers will not send the file field if there is no file selected, so there is no way for the backend to know that there was such a field.
Instead just check for the existence of the file field: if (!req.files.fileInput). You may also want to check that the file is not empty: if (!req.files.fileInput || !req.files.fileInput.size)

Is it possible to forward a stream from one function to another?

Please have a look at the following sample code.
As you can see, it uses busboy to parse incoming form data and write incoming files to disc.
Let's assume these are just image files because my sample code makes use of imgur.com. (a content-length header doesn't need to be sent)
The imgurUpload() function makes use of node-form-data
Is it somehow possible to stream the image files additionally, without the need to buffer them complete, to imgur.com? (to the imgurUpload() function and use it within node-form-data?)
Server / listener:
var http = require('http'),
Busboy = require('busboy'),
FormData = require('form-data'),
fs = require('fs')
http.createServer(function(req, res) {
if (req.method === 'POST') {
var busboy = new Busboy({
headers: req.headers
})
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
//pipe the stream to disc
file.pipe(fs.createWriteStream('1st-' + filename))
//pipe the stream a 2nd time
file.pipe(fs.createWriteStream('2nd-' + filename))
/* how to connect things together? */
})
busboy.on('finish', function() {
res.writeHead(200, {
'Connection': 'close'
})
res.end("upload complete")
})
return req.pipe(busboy)
} else if (req.method === 'GET') {
var stream = fs.createReadStream(__dirname + '/index.html')
stream.pipe(res)
}
}).listen(80, function() {
console.log('listening for requests')
})
HTML test form (index.html)
<!doctype html>
<head></head>
<body>
<form action="/upload" method="post" enctype="multipart/form-data">
<input type="file" name="file">
<input type="submit" value="submit">
</form>
</body>
</html>
Sample function that submits an image file to imgur.com:
function imgurUpload(stream) {
var form = new FormData()
//form.append('Filedata', fs.createReadStream('test.jpg'))
form.append('Filedata', /* how to connect things together? */X )
form.submit('http://imgur.com/upload', function(err, res) {
if (err) throw err
var body = ''
res.on('data', function(chunk) { body += chunk })
res.on('end', function() { console.log('http://imgur.com/' + JSON.parse(body).data.hash) })
})
}
Update (regarding mscdex answer)
_stream_readable.js:748
throw new Error('Cannot switch to old mode now.');
^
Error: Cannot switch to old mode now.
at emitDataEvents (_stream_readable.js:748:11)
at FileStream.Readable.pause (_stream_readable.js:739:3)
at Function.DelayedStream.create (\path\node_modules\form-data\node_modules\combined-stream\node_modules\delayed-stream\lib\delayed_stream.js:35:12)
at FormData.CombinedStream.append (\path\node_modules\form-data\node_modules\combined-stream\lib\combined_stream.js:45:30)
at FormData.append (\path\node_modules\form-data\lib\form_data.js:43:3)
at imgurUpload (\path\app.js:54:8)
at Busboy.<anonymous> (\path\app.js:21:4)
at Busboy.emit (events.js:106:17)
at Busboy.emit (\path\node_modules\busboy\lib\main.js:31:35)
at PartStream.<anonymous> (\path\node_modules\busboy\lib\types\multipart.js:205:13)
You can append Readable streams as shown in node-form-data's readme. So this:
form.append('Filedata', stream);
should work just fine.
Then in your file event handler:
imgurUpload(file);

Categories