uploading multiple files to s3 with sailsjs - javascript

I am not using the package of multer because I am not using express so I am not sure how multer can work with sailsjs
Anyways, I am trying to upload multiple files to s3, at first I worked with for loop which did not work because for loop is synchronous and file upload is asynchronous.
But then I googled that using recurrsive would work so I tried it but somehow it still didn't though.
Files are uploaded but then the size isn't right for all of them.
Somehow the size might be bigger / smaller then when I download the file let's say if it's a doc file, either I get error saying it's not a msdoc file or what's inside is all scrambled. If it's a pdf, it'll say failed to open the pdf file.
If I try only with one file, it works sometimes but not always though.
Did I do something wrong with the codes below?
s3_upload_multi: async function(req){
try {
let fieldName = req._fileparser.upstreams[0].fieldName;
let files = req.file(fieldName)._files;
let return_obj = [];
const upload_rec = files => {
if (files.length <= 0) return return_obj;
const f = files.pop();
const fileUpload = f.stream;
const s3 = new AWS.S3();
// https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property
s3.putObject(({ // uses s3 sdk
Bucket: sails.config.aws.bucket,
Key: 'blahblahblahblahblah',
Body: fileUpload._readableState.buffer.head.data, // buffer from file
ACL: 'public-read',
}, function ( err, data ) {
if (err) reject(err);
return_obj.push(data);
console.log(return_obj, 'return_obj');
});
return upload_rec(files);
};
upload_rec(files);
} catch (e) {
console.log(e, 'inside UploadService');
return false;
}
}
Thanks in advance for any advices and suggestions

Related

Why does Busboy yield inconsistent results when parsing FLAC files?

I have an Express server that receives FormData with an attached FLAC audio file. The code works as expected for several files of varying size (10 - 70MB), but some of them get stuck in the 'file' event and I cannot figure out why this happens. It is even more strange when a file that previously did not fire the file.on('close', => {}) event, as can be seen in the documentation for Busboy, suddenly does so, with the file being successfully uploaded.
To me, this seems completely random, as I have tried this with a dozen files of varying size and content type (audio/flac & audio/x-flac), and the results have been inconsistent. Some files will, however, not work at all, even if I attempt to parse them many times over. Whereas, certain files can be parsed and uploaded, given enough attempts?
Is there some error that I fail to deal with in the 'file' event? I did try to listen to the file.on('error', => {}) event, but there were no errors to be found. Other answers suggest that the file stream must be consumed for the 'close' event to proceed, but I think that file.pipe(fs.createWriteStream(fileObject.filePath)); does that, correct?
Let me know if I forgot to include some important information in my question. This has been bothering me for about a week now, so I am happy to provide anything of relevance to help my chances of overcoming this hurdle.
app.post('/upload', (request, response) => {
response.set('Access-Control-Allow-Origin', '*');
const bb = busboy({ headers: request.headers });
const fields = {};
const fileObject = {};
bb.on('file', (_name, file, info) => {
const { filename, mimeType } = info;
fileObject['mimeType'] = mimeType;
fileObject['filePath'] = path.join(os.tmpdir(), filename);
file.pipe(fs.createWriteStream(fileObject.filePath));
file.on('close', () => {
console.log('Finished parsing of file');
});
});
bb.on('field', (name, value) => {
fields[name] = value;
});
bb.on('close', () => {
bucket.upload(
fileObject.filePath,
{
uploadType: 'resumable',
metadata: {
metadata: {
contentType: fileObject.mimeType,
firebaseStorageDownloadToken: fields.id
}
}
},
(error, uploadedFile) => {
if (error) {
console.log(error);
} else {
db.collection('tracks')
.doc(fields.id)
.set({
identifier: fields.id,
artist: fields.artist,
title: fields.title,
imageUrl: fields.imageUrl,
fileUrl: `https://firebasestorage.googleapis.com/v0/b/${bucket.name}/o/${uploadedFile.name}?alt=media&token=${fields.id}`
});
response.send(`File uploaded: ${fields.id}`);
}
}
);
});
request.pipe(bb);
});
UPDATE: 1
I decided to measure the number of bytes that were transferred upon each upload with file.on('data', (data) => {}), just to see if the issue was always the same, and it turns out that this too is completely random.
let bytes = 0;
file.on('data', (data) => {
bytes += data.length;
console.log(`Loaded ${(bytes / 1000000).toFixed(2)}MB`);
});
First Test Case: Fenomenon - Sleepy Meadows Of Buxton
Source: https://fenomenon.bandcamp.com/track/sleepy-meadows-of-buxton
Size: 30.3MB
Codec: FLAC
MIME: audio/flac
Results from three attempts:
Loaded 18.74MB, then became stuck
Loaded 5.05MB, then became stuck
Loaded 21.23MB, then became stuck
Second Test Case: Almunia - New Moon
Source: https://almunia.bandcamp.com/track/new-moon
Size: 38.7MB
Codec: FLAC
MIME: audio/flac
Results from three attempts:
Loaded 12.78MB, then became stuck
Loaded 38.65, was successfully uploaded!
Loaded 38.65, was successfully uploaded!
As you can see, the behavior is unpredictable to say the least. Also, those two successful uploads did playback seamlessly from Firebase Storage, so it really worked as intended. What I cannot understand is why it would not always work, or at least most of the time, excluding any network-related failures.
UPDATE: 2
I am hopelessly stuck trying to make sense of the issue, so I have now created a scenario that closely resembles my actual project, and uploaded the code to GitHub. It is pretty minimal, but I did add some additional libraries to make the front-end pleasant to work with.
There is not much to it, other than an Express server for the back-end and a simple Vue application for the front-end. Within the files folder, there are two FLAC files; One of them is only 4.42MB to prove that the code does sometimes work. The other file is much larger at 38.1MB to reliably illustrate the problem. Feel free to try any other files.
Note that the front-end must be modified to allow files other than FLAC files. I made the choice to only accept FLAC files, as this is what I am working with in my actual project.
You'll need to write the file directly when BusBoy emits the file event.
It seems there is a race condition if you rely on BusBoy that prevents the file load from being completed. If you load it in the file event handler then it works fine.
app.post('/upload', (request, response) => {
response.set('Access-Control-Allow-Origin', '*');
const bb = busboy({
headers: request.headers
});
const fileObject = {};
let bytes = 0;
bb.on('file', (name, file, info) => {
const {
filename,
mimeType
} = info;
fileObject['mimeType'] = mimeType;
fileObject['filePath'] = path.join(os.tmpdir(), filename);
const saveTo = path.join(os.tmpdir(), filename);
const writeStream = fs.createWriteStream(saveTo);
file.on('data', (data) => {
writeStream.write(data);
console.log(`Received: ${((bytes += data.length) / 1000000).toFixed(2)}MB`);
});
file.on('end', () => {
console.log('closing writeStream');
writeStream.close()
});
});
bb.on('close', () => {
console.log(`Actual size is ${(fs.statSync(fileObject.filePath).size / 1000000).toFixed(2)}MB`);
console.log('This is where the file would be uploaded to some cloud storage server...');
response.send('File was uploaded');
});
bb.on('error', (error) => {
console.log(error);
});
request.pipe(bb);
});

folders and files are not visible after uploading file though multer

I am working on a small project. discussing Step by step
At first I am uploading zip files though multer
extracting those files (How can I call extract function after completing upload using multer?)
After extracting those I am trying to filter those files
after filtering those files I want to move some files to another directory
in my main index.js I have
A simple route to upload files which is working
// MAIN API ENDPOINT
app.post("/api/zip-upload", upload, async (req, res, next) => {
console.log("FIles - ", req.files);
});
Continuous checking for if there is any zip file that needs to unzip but the problem is after uploading it's not showing any files or dir
// UNZIP FILES
const dir = `${__dirname}/uploads`;
const files = fs.readdirSync("./uploads");
const filesUnzip = async () => {
try {
if (fs.existsSync(dir)) {
console.log("files - ", files);
for (const file of files) {
console.log("file - ", file);
try {
const extracted = await extract("./uploads/" + file, { dir: __dirname + "/uploads/" });
console.log("Extracted - ",extracted);
// const directories = await fs.statSync(dir + '/' + file).isDirectory();
} catch (bufErr) {
// console.log("------------");
console.log(bufErr.syscall);
}
};
// const directories = await files.filter(function (file) { return fs.statSync(dir + '/' + file).isDirectory(); });
// console.log(directories);
}
} catch (err) {
console.log(err);
}
return;
}
setInterval(() => {
filesUnzip();
}, 2000);
Moving files to static directory but here is the same problem no directory found
const getAllDirs = async () => {
// console.log(fs.existsSync(dir));
// FIND ALL DIRECTORIES
if (fs.existsSync(dir)) {
const directories = await files.filter(function (file) { return fs.statSync(dir + '/' + file).isDirectory(); });
console.log("Directories - ",directories);
if (directories.length > 0) {
for (let d of directories) {
const subdirFiles = fs.readdirSync("./uploads/" + d);
for (let s of subdirFiles) {
if (s.toString().match(/\.xml$/gm) || s.toString().match(/\.xml$/gm) !== null) {
console.log("-- ", d + "/" + s);
const move = await fs.rename("uploads/" + d + "/" + s, __dirname + "/static/" + s, (err) => { console.log(err) });
console.log("Move - ", move);
}
}
}
}
}
}
setInterval(getAllDirs, 3000);
There are so many issues with your code, I don't know where to begin:
Why are you using fs.xxxSync() methods if all your functions are async? Using xxxSync() methods is highly discouraged because it's blocking the server (ie parallel requests can't/won't be accepted while a sync reading is in progress). The fs module supports a promise api ...
Your "Continuous checking" for new files is always checking the same (probably empty) files array because it seems you are executing files = fs.readdirSync("./uploads"); only once (probably at server start, but I can't tell for sure because there isn't any context for that snippet)
You shouldn't be polling that "uploads" directory. Because as writing a file (if done properly) is an asynchronous process, you may end up reading incomplete files. Instead you should trigger the unzipping from your endpoint handler. Once it is hit, body.files contains the files that have been uploaded. So you can simply use this array to start any further processing instead of frequently polling a directory.
At some points you are using the callback version of the fs API (for instance fs.rename(). You cannot await a function that expects a callback. Again, use the promise api of fs.
EDIT
So I'm trying to address your issues. Maybe I can't solve all of them because of missing infomation, but you should get the general idea.
First of all, you shuld use the promise api of the fs module. And also for path manipulation, you should use the available path module, which will take care of some os specific issues.
const fs = require('fs').promises;
const path = require('path');
Your API endpoint isn't currently returning anything. I suppose you stripped away some code, but still. Furthermore, you should trigger your filehandling from here, so you don't have to do directory polling, which is
error prone,
wasting resources and
if you do it synchronously like you do blocks the server
app.post("/api/zip-upload", upload, async (req, res, next) => {
console.log("FIles - ", req.files);
//if you want to return the result only after the files have been
//processed use await
await handleFiles(req.files);
//if you want to return to the client immediately and process files
//skip the await
//handleFiles(req.files);
res.sendStatus(200);
});
Handling the files seems to consist of two different steps:
unzipping the uploaded zip files
copying some of the extracted files into another directory
const source = path.join(".", "uploads");
const target = path.join(__dirname, "uploads");
const statics = path.join(__dirname, "statics");
const handleFiles = async (files) => {
//a random folder, which will be unique for this upload request
const tmpfolder = path.join(target, `tmp_${new Date().getTime()}`);
//create this folder
await fs.mkdir(tmpfolder, {recursive: true});
//extract all uploaded files to the folder
//this will wait for a list of promises and resolve once all of them resolved,
await Promise.all(files.map(f => extract(path.join(source, f), { dir: tmpfolder })));
await copyFiles(tmpfolder);
//you probably should delete the uploaded zipfiles and the tempfolder
//after they have been handled
await Promise.all(files.map(f => fs.unlink(path.join(source, f))));
await fs.rmdir(tmpfolder, { recursive: true});
}
const copyFiles = async (tmpfolder) => {
//get all files/directory names in the tmpfolder
const allfiles = await fs.readdir(tmpfolder);
//get their stats
const stats = await Promise.all(allfiles.map(f => fs.stat(path.join(tmpfolder, f))));
//filter directories only
const dirs = allfiles.filter((_, i) => stats[i].isDirectory());
for (let d of dirs) {
//read all filenames in the subdirectory
const files = await fs.readdir(path.join(tmpfolder, d)));
//filter by extension .xml
const xml = files.filter(x => path.extname(x) === ".xml");
//move all xml files
await Promise.all(xml.map(f => fs.rename(path.join(tmpfolder, d, f), path.join(statics, f))));
}
}
That should do the trick. Of course you may notice there is no error handling with this code. You should add that.
And I'm not 100% sure about your paths. You should consider the following
./uploads refers to a directory uploads in the current working directory (whereever that may be)
${__dirname}/uploads refers to a directory uploads which is in the same directory as the script file currently executing Not sure if that is the directory you want ...
./uploads and ${__dirname}/uploads may point to the same folder or to completely different folders. No way knowing that without additional context.
Furthermore in your code you extract the ZIP files from ./uploads to ${__dirname}/uploads and then later try to copy XML files from ./uploads/xxx to ${__dirname}/statics, but there won't be any directory xxx in ./uploads because you extracted the ZIP file to a (probably) completely different folder.

Ipfs-mini cat APi's output buffer seems like corrupted for the hash pointing the image file

I am a newbie to both Javascript and ipfs and I am trying an experiment to fetch an image buffer from the ipfs hash "QmdD8FL7N3kFnWDcPSVeD9zcq6zCJSUD9rRSdFp9tyxg1n" using ipfs-mini node module.
Below is my code
const IPFS = require('ipfs-mini');
const FileReader = require('filereader');
var multer = require('multer');
const ipfs = initialize();
app.post('/upload',function(req,res){
upload(req,res, function(err){
console.log(req.file.originalname);
ipfs.cat('QmdD8FL7N3kFnWDcPSVeD9zcq6zCJSUD9rRSdFp9tyxg1n', function(err, data){
if(err) console.log("could not get the image from the ipfs for hash " + ghash);
else {
var wrt = data.toString('base64');
console.log('size ; ' + wrt.length);
fs.writeFile('tryipfsimage.gif',wrt, (err) =>{
if(err)console.log('can not write file');
else {
//console.log(data);
ipfs.stat('QmdD8FL7N3kFnWDcPSVeD9zcq6zCJSUD9rRSdFp9tyxg1n', (err, data)=>{
// console.log(hexdump(wrt));
});
console.log("files written successfully");
}
});
}
});
});
});
function initialize() {
console.log('Initializing the ipfs object');
return new IPFS({
host: 'ipfs.infura.io',
protocol: 'https'
});
}
I could view the image properly in the browser using the link below "https://ipfs.io/ipfs/QmdD8FL7N3kFnWDcPSVeD9zcq6zCJSUD9rRSdFp9tyxg1n", but if I open the file 'tryipfsimage.gif' in which I dump the return buffer of the cat API in above program, the content of the image seems corrupted. I am not sure what the mistake I am doing in the code. it would be great If someone points me the mistake.
From ipfs docs https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#javascript---ipfsfilescatipfspath-callback
file in the callback is actually a Buffer so by toString('base64')'ing it you are writing actual base64 into the .gif file - no need to do this. you can pass the Buffer directly to the fs.writeFile api with
fs.writeFile('tryipsimage.gif', file, ...
For larger files I would recommend using the ipfs catReadableStream, where you can do something more like:
const stream = ipfs.catReadableStream('QmdD8FL7N3kFnWDcPSVeD9zcq6zCJSUD9rRSdFp9tyxg1n')
// don't forget to add error handlers to stream and whatnot
const fileStream = fs.createWriteStream('tryipsimage.gif')
stream.pipe(fileStream);

Merging PDFs in Node

Hi i'm trying to merge pdf's of total of n but I cannot get it to work.
I'm using the Buffer module to concat the pdf's but it does only apply the last pdf in to the final pdf.
Is this even possible thing to do in node?
var pdf1 = fs.readFileSync('./test1.pdf');
var pdf2 = fs.readFileSync('./test2.pdf');
fs.writeFile("./final_pdf.pdf", Buffer.concat([pdf1, pdf2]), function(err) {
if(err) {
return console.log(err);
}
console.log("The file was saved!");
});
There are currently some libs out there but they do all depend on either other software or programming languages.
What do you expect to get when you do Buffer.concat([pdf1, pdf2])? Just by concatenating two PDFs files you won’t get one containing all pages. PDF is a complex format (basically one for vector graphics). If you just added two JPEG files you wouldn’t expect to get a big image containing both pictures, would you?
You’ll need to use an external library. https://github.com/wubzz/pdf-merge might work for instance.
HummusJS is another PDF manipulation library, but without a dependency on PDFtk. See this answer for an example of combining PDFs in Buffers.
Aspose.PDF Cloud SDK for Node.js can merge/combine pdf documents without depending on any third-party API/Tool. However, currently, it works with cloud storage(Aspose Internal Storage, Amazon S3, DropBox, Google Drive Storage, Google Cloud Storage, Windows Azure Storage, FTP Storage). In near future, we will provide support to merge files from the request body(stream).
const { PdfApi } = require("asposepdfcloud");
const { MergeDocuments }= require("asposepdfcloud/src/models/mergeDocuments");
var fs = require('fs');
pdfApi = new PdfApi("xxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxx", "xxxxxxxxxxxxxxxxxxxxx");
const file1 = "dummy.pdf";
const file2 = "02_pages.pdf";
const localTestDataFolder = "C:\\Temp";
const names = [file1, file2];
const resultName = "MergingResult.pdf";
const mergeDocuments = new MergeDocuments();
mergeDocuments.list = [];
names.forEach( (file) => {
mergeDocuments.list.push(file);
});
// Upload File
pdfApi.uploadFile(file1, fs.readFileSync(localTestDataFolder + "\\" + file1)).then((result) => {
console.log("Uploaded File");
}).catch(function(err) {
// Deal with an error
console.log(err);
});
// Upload File
pdfApi.uploadFile(file2, fs.readFileSync(localTestDataFolder + "\\" + file2)).then((result) => {
console.log("Uploaded File");
}).catch(function(err) {
// Deal with an error
console.log(err);
});
// Merge PDF documents
pdfApi.putMergeDocuments(resultName, mergeDocuments, null, null).then((result) => {
console.log(result.body.code);
}).catch(function(err) {
// Deal with an error
console.log(err);
});
//Download file
const outputPath = "C:/Temp/" + resultName;
pdfApi.downloadFile(outputPath).then((result) => {
fs.writeFileSync(localPath, result.body);
console.log("File Downloaded");
}).catch(function(err) {
// Deal with an error
console.log(err);
});

How to upload image to S3 using Node

I am writing an Express app that takes in a base64 encoded string that represents an image. Right now, i'm not really sure how I can take that string and upload the image to AWS S3, so i'm reading in the encoded image string data, decoding it, writing a file using fs, and then trying to upload. I have this working for an endpoint that just takes in a raw file, and all of its content is correctly uploaded to AWS s3.
Now when I try to do what I described above, i'm able to upload to S3, but the file has 0kb and is empty, and i'm not sure why. I tested just taking the stringData and writing a file to a test file, and it works. However, when I try uploading to s3, the file shows but it's empty. Here is my code:
router.post('/images/tags/nutritionalInformation/image/base64encoded', function (req, res) {
console.log(req.body.imageString);
var base64Stream = req.body.imageString;
var imgDecodedBuffer = decodeBase64Image(base64Stream);
console.log(imgDecodedBuffer);
// write to image file
var prefix = guid().toString() + ".jpg";
var filePath = './uploads/' + prefix;
console.log(filePath);
fs.writeFile(filePath, imgDecodedBuffer.data, function(err) {
console.log(err);
});
var stream = fs.createReadStream(filePath);
console.log(stream);
return s3fsImpl.writeFile(prefix, stream).then(function () {
fs.unlink(filePath, function (err) {
if (err) {
console.error(err);
}
});
});
})
Here are the relevant import statements:
var fs = require('fs');
var s3fs = require('s3fs');
var multiparty = require('connect-multiparty'),
multipartyMidleware = multiparty();
var s3fsImpl = new s3fs('blahblah', {
accessKeyId: 'ACCESS_KEY_ID',
secretAccessKey: 'SECRET'
});
Any help would be greatly appreciated!
If you simply just pass in the buffer, which I presume is in your imgDecodedBuffer.data value, it should work.

Categories