We Keep Coding

Node.js AWS Lambda putObject doesn't seem to finish

I have a Lambda function that is meant to download a directory of files from s3, convert them, delete the old files, and upload the new output files back to s3. The output for each file will be at least one file and a folder.
Everything seems to be working as intended, except for the upload. No errors are thrown, it just ends without putting.
I'm a novice, so feel free to point out I've done it all wrong.
exports.handler = async ({ dirName }) => {
// const jsonIn = JSON.parse(event.body);
// const dirName = jsonIn.dirName;
const localDir = `/tmp/${dirName}`;
const params = {
Bucket: 'to-pdf-test',
Delimiter: '/',
Prefix: dirName + '/',
StartAfter: dirName + '/'
};
var s3List;
var localList = [];
execSync(`mkdir ${localDir}`);
try {
s3List = await s3.listObjectsV2(params).promise();
} catch (e) {
throw e;
}
await Promise.all(
s3List.Contents.map(async (file) => {
let f = await getFiles(file);
localList.push(f);
})
).then(res => {console.log('Get Successful' + res) } )
.catch(err => {console.log('error' + err) } );
await Promise.all(
localList.map(async (file) => {
convertFile(file);
})
).then(res => {console.log('Convert Successful' + res) } )
.catch(err => {console.log('error' + err) } );
dirSync(localDir, async (filePath, stat) => {
let bucketPath = filePath.substring(5);
let uploadParams = { Bucket: 'to-pdf-test',
Key: `${bucketPath}`,
Body: fs.readFileSync(filePath) };
console.log('DS fPath ' + filePath);
console.log('DS bPath ' + bucketPath);
console.log(uploadParams.Body);
try {
let res = await s3.putObject(uploadParams).promise();
console.log('Upload Complete', res);
} catch (e) {
console.log('Error', e);
}
});
};
async function getFiles(file) {
let filePath = `/tmp/${file.Key}`;
let fileParams = {
Bucket: 'to-pdf-test',
Key: file.Key
};
try {
const { Body: inputFileBuffer } = await s3.getObject(fileParams).promise();
fs.writeFileSync(filePath, inputFileBuffer);
} catch (e) {
throw (e);
}
return filePath;
}
function convertFile(file) {
const noPath = getFilename(file);
const fPath = getFilePath(file);
if (path.extname(noPath) === '.msg') {
execSync(`cd ${fPath} && ${command} ${noPath}`);
} else {
console.log(`${noPath} not run. Not .msg`);
}
fs.unlinkSync(file);
}
function getFilename(fullPath) {
return fullPath.replace(/^.*[\\\/]/, '');
}
function getFilePath(fullPath) {
return fullPath.substring(fullPath.lastIndexOf('/'), 0);
}
function dirSync(dirPath, callback) {
fs.readdirSync(dirPath).forEach((name) => {
var filePath = path.join(dirPath, name);
var stat = fs.statSync(filePath);
if (stat.isDirectory()) {
dirSync(filePath, callback);
} else {
callback(filePath, stat);
}
});
}
I had the upload working in a previous version of this function, so thanks to this post for when it was working.

My solution for the moment - Read the local directory separately, push the paths of the files to localList then .map the array with all the paths to upload them.
localList = [];
//read dir and push to localList array
await dirSync(localDir, (filePath, stat) => {
localList.push(filePath);
});
console.log(localList);
await Promise.all(
localList.map( async (file) => {
let bucketPath = file.substring(5);
let uploadParams = {
Bucket: 'to-pdf-test',
Key: bucketPath,
Body: fs.readFileSync(file) };
console.log('Uploading', file);
await s3.putObject(uploadParams).promise()
.then((res) => {console.log('Upload Successful', bucketPath) } )
.catch((err) => {console.log('error' + err) } );
})
);
If there is better (or proper) way to do this, someone let me know :)

converting req.body to an array

I have a file input to upload multiple images and saving them to MongoDB, but the images uploaded to me as a string, I need to upload them with an array. I update it with the server code...
router.post('/multiple-upload',uploadImages,resizeImages,getResultImages , (req, res, next) => {
const thing = new Thing({
filename: req.body.filename,
images: req.body.images
});
thing.save().then(
() => {
res.status(201).json({
message: 'Post saved successfully!'
});
}
).catch(
(error) => {
res.status(400).json({
error: error
});
}
);
});
const uploadFiles = upload.array('images', 10);
exports.uploadImages = (req, res, next) => {
uploadFiles(req, res, (err) => {
if (err instanceof multer.MulterError) {
if (err.code === 'LIMIT_UNEXPECTED_FILE') {
return res.send('Too many files to upload !');
}
} else if (err) {
return res.send(err);
}
next();
});
};
exports.resizeImages = async (req, res, next) => {
if (!req.files) return next();
req.body.images = [];
await Promise.all(
req.files.map(async (files,i) => {
// const filename = files.originalname.replace(/\..+$/, '');
const newFilename = `tour-${files.originalname}-${Date.now()}-${i + 1}.jpeg`;
await sharp(files.buffer)
.resize(640, 320)
.toFormat('jpeg')
.jpeg({ quality: 90 })
.toFile(`upload/${newFilename}`);
req.body.images.push(newFilename);
// console.log(req.body.images);
})
);
next();
};
exports.getResultImages = async (req, res,next) => {
if (req.body.images.length <= 0) {
return res.send(`You must select at least 1 image !`);
}
const images = req.body.images.map((image) => '' + image + '').join(', ');
// res.status(200).json(req.body.images)
req.body.images = images;
res.locals.images = images;
console.log();
next();
return res.json(req.body.files);
// res.status(200).json({
// data:{
// images
// }
// });
};
I have a file input to upload multiple images and saving them to MongoDB, but the images uploaded to me as a string, I need to upload them with an array. I update it with the server code...

it works with me by deleting join(', ')
in this line const images = req.body.images.map((image) => '' + image + '').join(', ');

Firebase function storage POST image issue

I have encountered a problem when following a maximilian schwarzmüller course, which has otherwise been great: https://www.youtube.com/watch?v=qZ1EFnFOGvE
The image logs in the Firebase console as uploaded, recognises the type of file/size etc. But continually loads and never displays the image. I use a post request in POSTMAN to upload the image.
When I upload manually to firebase on their UI, everything works fine.
My code:
const functions = require("firebase-functions");
const os = require("os");
const path = require("path");
const spawn = require("child-process-promise").spawn;
const cors = require("cors")({ origin: true });
const Busboy = require("busboy");
const fs = require("fs");
const gcconfig = {
projectId: "REDACTED",
keyFilename: "REDACTED"
};
const gcs = require("#google-cloud/storage")(gcconfig);
//
exports.onFileChange = functions.storage.object().onFinalize(event => {
const object = event.data;
const bucket = object.bucket;
const contentType = object.contentType;
const filePath = object.name;
console.log("File change detected, function execution started");
if (object.resourceState === "not_exists") {
console.log("We deleted a file, exit...");
return;
}
if (path.basename(filePath).startsWith("resized-")) {
console.log("We already renamed that file!");
return;
}
const destBucket = gcs.bucket(bucket);
const tmpFilePath = path.join(os.tmpdir(), path.basename(filePath));
const metadata = { contentType: contentType };
return destBucket
.file(filePath)
.download({
destination: tmpFilePath
})
.then(() => {
return spawn("convert", [tmpFilePath, "-resize", "500x500", tmpFilePath]);
})
.then(() => {
return destBucket.upload(tmpFilePath, {
destination: "resized-" + path.basename(filePath),
metadata: metadata
});
});
});
exports.uploadFile = functions.https.onRequest((req, res) => {
cors(req, res, () => {
if (req.method !== "POST") {
return res.status(500).json({
message: "Not allowed"
});
}
const busboy = new Busboy({ headers: req.headers });
let uploadData = null;
busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {
const filepath = path.join(os.tmpdir(), filename);
uploadData = { file: filepath, type: mimetype };
file.pipe(fs.createWriteStream(filepath));
});
busboy.on("finish", () => {
const bucket = gcs.bucket("REDACTED");
bucket
.upload(uploadData.file, {
uploadType: "media",
metadata: {
metadata: {
contentType: uploadData.type
}
}
})
.then(() => {
res.status(200).json({
message: "It worked!"
});
})
.catch(err => {
res.status(500).json({
error: err
});
});
});
busboy.end(req.rawBody);
});
});
My security rules:
rules_version = '2';
service firebase.storage {
match /b/{bucket}/o {
match /{allPaths=**} {
allow read, write:if true;
}
}
}

Node js pause while loop wait until functions inside get executed completely?

I am coding a post request which downloads all URL HTML,zips them and email it back. This all should happen in the backend. I am storing all the data in an array and extract the first element to start these operations.
I have while loop inside which I am calling some functions. Each function gets executed at a certain time.
I used async, await and promises to make sure they run one after the
other.
Coming to my problem.
My while loop starts getting executed again before all the
functions inside it are executed.
app.post('/?', async (req, res) => {
var urls = req.query.urls
var email = req.query.email;
var new_stack = [urls, email]
stack.push(new_stack)
res.send("Mail sent")
if (isFunctionRunning === false) { //initially it is false
console.log(isFunctionRunning, stack.length)
send_mails();
}
});
const getGoogleIndexHTML = (url) => {
return new Promise((resolve, reject) => {
request(url, (err, res, body) => err ? reject(err) : resolve(body))
})
}
const some_function_to_download = async (url) => {
try {
const a = url.split(".")
let googleIndexHTML = await getGoogleIndexHTML(url)
await fs.writeFile(directory + '/' + a[1] + '.html', googleIndexHTML, (err) => {
if (err) throw err
})
console.log('File created.')
} catch (err) {
console.log(err)
}
}
const html_to_zip_file = async () => {
await zipper.zip(directory, function (error, zipped) {
if (!error) {
zipped.compress();
zipped.save('./package.zip', function (error) {
if (!error) {
console.log("Saved successfully !");
}
});
} else {
console.log(error)
}
})
}
const send_mails = async () => {
while (stack.length > 0) {
isFunctionRunning = true
var a = stack.shift()
var urls = a[0]
var collection_urls = urls.split(",");
var to_email = a[1]
rimraf(directory, function () {
console.log("done");
});
fs.mkdirSync(directory);
for (url of collection_urls) {
await some_function_to_download(url); // 5 sec per download
}
await html_to_zip_file() // takes 5 sec to zip
.then(result => {
transporter.sendMail(set_mail_options(to_email)) //2 sec to send mail
.then(result => {
console.log("Mail sent")
})
.catch(err => {
console.log(err)
})
})
.catch(err => {
console.log(err)
})
console.log("reached") // this is reached before zip is done and mail sent. I want to prevent this
}
isFunctionRunning = false
}

You need to return transporter.sendMail in sendMail, fs.writeFile in someFunctionToDownload and zipper.zip in htmlToZipFile otherwise the await won't work as expected (I'm assuming that they actually do return promises, I'm only familiar with fs.writeFile)
Also: CamelCase is used in JS, not snake_case 🙃
And are you sure rimraf is synchronous?
const sendMails = async () => {
while (stack.length > 0) {
isFunctionRunning = true;
const [urls, toEmail] = stack.shift();
var collectionUrls = urls.split(",");
rimraf(directory, function() {
console.log("done");
});
await fs.mkdir(directory);
await Promise.All(collectionUrls.map(someFunctionToDownload)); // 5 sec per download
await htmlToZipFile() // takes 5 sec to zip
.then(result => transporter.sendMail(set_mail_options(toEmail))) //2 sec to send mail
.then(result => {
console.log("Mail sent");
})
.catch(err => {
console.log(err);
});
console.log("reached"); // this is reached before zip is done and mail sent. I want to prevent this
}
isFunctionRunning = false;
};
const someFunctionToDownload = async url => {
const a = url.split(".");
const googleIndexHTML = await getGoogleIndexHTML(url);
return fs.writeFile(`${directory}/${a[1]}.html`, googleIndexHTML, err => {
if (err) throw err;
});
};
const htmlToZipFile = async () => {
return zipper.zip(directory, function(error, zipped) {
if (!error) {
zipped.compress();
zipped.save("./package.zip", function(error) {
if (!error) {
console.log("Saved successfully!");
}
});
} else {
console.log(error);
}
});
};

Try using the following
while (stack.length > 0) {
isFunctionRunning = true
var a = stack.shift()
var urls = a[0]
var collection_urls = urls.split(",");
var to_email = a[1]
rimraf(directory, function () {
console.log("done");
});
fs.mkdirSync(directory);
for (url of collection_urls) {
await some_function_to_download(url); // 5 sec per download
}
try {
const result = await html_to_zip_file() // takes 5 sec to zip
const sendMailResult = await transporter.sendMail(set_mail_options(to_email))
} catch(e)
{
console.log(e)
}
console.log("reached")
}
Since html_to_zip_file() and sendMail function are independent
we can use
const result = await Promise.all([html_to_zip_file(),transporter.sendMail(set_mail_options(to_email))]);

Sending multiple files to google bucket with multer - why are my files empty?

I previously had a single file upload set up and working properly. Now I need to make it handle multiple files.
Here is my code right now:
const multer = require('multer')
const { Storage } = require('#google-cloud/storage')
const storage = new Storage()
const m = multer({ storage: multer.memoryStorage() })
module.exports = app => {
app.use('/', router)
router.post(
'/reader-:shortId/file-upload',
passport.authenticate('jwt', { session: false }),
m.array('files'),
async function (req, res) {
const bucketName = req.params.shortId.toLowerCase()
await storage.createBucket(bucketName)
bucket = storage.bucket(bucketName)
let promises = []
req.files.forEach((file) => {
const blob = bucket.file(file.originalname)
const newPromise = new Promise((resolve, reject) => {
blob.createWriteStream({
metadata: { contentType: file.mimetype }
}).on('finish', async response => {
await blob.makePublic()
resolve(response)
}).on('error', err => {
reject('upload error: ', err)
}).end()
})
promises.push(newPromise)
})
Promise.all(promises).then((response) => {
// the response I get here is [undefined, undefined]
res.status(200).send(response)
}).catch((err) => {
res.status(400).send(err.message)
});
})
}
req.files does give me an array of files, with a buffer and a size that makes sense.
The promises all resolve.
But once I check the files in the google bucket, they have the right name but don't have any content (and size of 0)
As I said before, it was working when I was doing it with one file (using m.single('file')
I don't want to use the bucket as the destination with multer setup because I also have to change the file name before uploading to google bucket.
edit: this is the code example given by google cloud documentations for single file uploads (https://cloud.google.com/nodejs/getting-started/using-cloud-storage):
function sendUploadToGCS (req, res, next) {
if (!req.file) {
return next();
}
const gcsname = Date.now() + req.file.originalname;
const file = bucket.file(gcsname);
const stream = file.createWriteStream({
metadata: {
contentType: req.file.mimetype
},
resumable: false
});
stream.on('error', (err) => {
req.file.cloudStorageError = err;
next(err);
});
stream.on('finish', () => {
req.file.cloudStorageObject = gcsname;
file.makePublic().then(() => {
req.file.cloudStoragePublicUrl = getPublicUrl(gcsname);
next();
});
});
stream.end(req.file.buffer);
}
I originally had something like that working, but I just don't understand where it is getting the file buffer data from. That is probably where things are different with multiple files.

I know its too late, but someone might looking an answer for uploading multiple files on Google Cloud Storage.
Dependencies:
Express
Google Cloud Library
Multer
Body Parser
This is the controller code.
exports.post_image_upload = async (req, res) => {
/** Check if file exist */
if (!req.files) {
res.status(400).send('No file uploaded.');
return;
}
let PublicUrls = []
req.files.forEach((file) => {
const blob = bucket.file(file.fieldname + '-' + Date.now() + path.extname(file.originalname))
const blobStream = blob.createWriteStream({
metadata: { contentType: file.mimetype }
})
blobStream.on('finish', ()=> {
blob.makePublic()
})
blobStream.on('error', err => {
//Put your error message here
})
blobStream.end(file.buffer)
const Url = `https://storage.googleapis.com/${bucket.name}/${blob.name}`
PublicUrls.push(Url)
})
res.send(PublicUrls)
}
Good Luck

Ok, turns out I had to change
.end()
to
.end(file.buffer)

Marie Pelletier, I think your approach is 100% right. I modified a little your code trying to avoid the async response:
let promises = []
req.files.forEach((file) => {
const blob = bucket.file(file.originalname)
const newPromise = new Promise((resolve, reject) => {
blob.createWriteStream({
metadata: { contentType: file.mimetype },
resumable: false //Good for small files
}).on('finish', () => {
const Url = `https://storage.googleapis.com/${bucket.name}/${blob.name}`;
resolve({ name: file.originalname, url: Url });
}).on('error', err => {
reject('upload error: ', err);
}).end(file.buffer);
})
promises.push(newPromise);
})
Promise.all(promises).then((response) => {
res.status(200).send(response)
}).catch((err) => {
res.status(400).send(err.message)
});
This way, I didn't get 'undefined' anymore.