We Keep Coding

Node.js AWS Lambda putObject doesn't seem to finish

I have a Lambda function that is meant to download a directory of files from s3, convert them, delete the old files, and upload the new output files back to s3. The output for each file will be at least one file and a folder.
Everything seems to be working as intended, except for the upload. No errors are thrown, it just ends without putting.
I'm a novice, so feel free to point out I've done it all wrong.
exports.handler = async ({ dirName }) => {
// const jsonIn = JSON.parse(event.body);
// const dirName = jsonIn.dirName;
const localDir = `/tmp/${dirName}`;
const params = {
Bucket: 'to-pdf-test',
Delimiter: '/',
Prefix: dirName + '/',
StartAfter: dirName + '/'
};
var s3List;
var localList = [];
execSync(`mkdir ${localDir}`);
try {
s3List = await s3.listObjectsV2(params).promise();
} catch (e) {
throw e;
}
await Promise.all(
s3List.Contents.map(async (file) => {
let f = await getFiles(file);
localList.push(f);
})
).then(res => {console.log('Get Successful' + res) } )
.catch(err => {console.log('error' + err) } );
await Promise.all(
localList.map(async (file) => {
convertFile(file);
})
).then(res => {console.log('Convert Successful' + res) } )
.catch(err => {console.log('error' + err) } );
dirSync(localDir, async (filePath, stat) => {
let bucketPath = filePath.substring(5);
let uploadParams = { Bucket: 'to-pdf-test',
Key: `${bucketPath}`,
Body: fs.readFileSync(filePath) };
console.log('DS fPath ' + filePath);
console.log('DS bPath ' + bucketPath);
console.log(uploadParams.Body);
try {
let res = await s3.putObject(uploadParams).promise();
console.log('Upload Complete', res);
} catch (e) {
console.log('Error', e);
}
});
};
async function getFiles(file) {
let filePath = `/tmp/${file.Key}`;
let fileParams = {
Bucket: 'to-pdf-test',
Key: file.Key
};
try {
const { Body: inputFileBuffer } = await s3.getObject(fileParams).promise();
fs.writeFileSync(filePath, inputFileBuffer);
} catch (e) {
throw (e);
}
return filePath;
}
function convertFile(file) {
const noPath = getFilename(file);
const fPath = getFilePath(file);
if (path.extname(noPath) === '.msg') {
execSync(`cd ${fPath} && ${command} ${noPath}`);
} else {
console.log(`${noPath} not run. Not .msg`);
}
fs.unlinkSync(file);
}
function getFilename(fullPath) {
return fullPath.replace(/^.*[\\\/]/, '');
}
function getFilePath(fullPath) {
return fullPath.substring(fullPath.lastIndexOf('/'), 0);
}
function dirSync(dirPath, callback) {
fs.readdirSync(dirPath).forEach((name) => {
var filePath = path.join(dirPath, name);
var stat = fs.statSync(filePath);
if (stat.isDirectory()) {
dirSync(filePath, callback);
} else {
callback(filePath, stat);
}
});
}
I had the upload working in a previous version of this function, so thanks to this post for when it was working.

My solution for the moment - Read the local directory separately, push the paths of the files to localList then .map the array with all the paths to upload them.
localList = [];
//read dir and push to localList array
await dirSync(localDir, (filePath, stat) => {
localList.push(filePath);
});
console.log(localList);
await Promise.all(
localList.map( async (file) => {
let bucketPath = file.substring(5);
let uploadParams = {
Bucket: 'to-pdf-test',
Key: bucketPath,
Body: fs.readFileSync(file) };
console.log('Uploading', file);
await s3.putObject(uploadParams).promise()
.then((res) => {console.log('Upload Successful', bucketPath) } )
.catch((err) => {console.log('error' + err) } );
})
);
If there is better (or proper) way to do this, someone let me know :)

Node.js Google Drive Client - Error downloading file: response.data.on is not a function

I am using the Node.js Google Drive client trying to download certain files from a gdrive. When using the example provided in their GitHub I get a Uncaught (in promise) TypeError: res.data.on is not a function error. The file is still created locally, but it's just an empty file from createWriteStream().
When I log the res variable I get: ReadableStream {locked: false}.
I'm pretty new to streams so this is quite a bit over my head.
Here is my code. You'll notice it's almost exactly what the example they provide looks like.
syncFileFromDrive(fileId, filePath) {
filePath.replace(userDataPath, '');
filePath = `${userDataPath}/${filePath}`;
filePath.replaceAll('//', '/');
logger.info(`Sync file from drive: Syncing file to path: ${filePath}`);
logger.info(`Sync file from drive: File id: ${fileId}`)
const dest = fs.createWriteStream(filePath);
let progress = 0;
this.drive.files.get({fileId, alt: 'media'}, {responseType: 'stream'}).then(res => {
console.log(res)
console.log(res.data)
res.data
.on('end', () => {
console.log('Done downloading file.');
folderStructure.buildFileMenu()
resolve(dest)
})
.on('error', err => {
console.error('Error downloading file.');
reject(err);
})
.on('data', d => {
progress += d.length;
if (process.stdout.isTTY) {
process.stdout.clearLine();
process.stdout.cursorTo(0);
process.stdout.write(`Downloaded ${progress} bytes`);
}
})
.pipe(dest);
});
}
Edit: I should add that this is for an Electron application. So while Node is supported, I'm not sure if that may affect the way I can use streams.

This feels like it's a bit of a work around, and I am open to any suggestions, but this was able to solve the issue I was having.
syncFileFromDrive(fileId, filePath) {
filePath.replace(userDataPath, '');
filePath = `${userDataPath}/${filePath}`;
filePath.replaceAll('//', '/');
logger.info(`Sync file from drive: Syncing file to path: ${filePath}`);
logger.info(`Sync file from drive: File id: ${fileId}`)
this.drive.files
.get({ fileId, alt: "media"}, {responseType: 'stream'})
.then((res) => {
const dest = fs.createWriteStream(filePath);
const decoder = new TextDecoder("utf-8");
const reader = res.data.getReader()
reader.read().then(function processText({ done, value }) {
if (done) {
console.log("Stream complete");
return;
}
dest.write(decoder.decode(value))
// Read some more, and call this function again
return reader.read().then(processText);
});
})
}

Please take a look at my implementation, which I used to downloading the file
import { google } from 'googleapis';
const getOauth2Client = () => new google.auth.OAuth2(
process.env.GOOGLE_DRIVE_CLIENT_ID,
process.env.GOOGLE_DRIVE_CLIENT_SECRET,
process.env.GOOGLE_DRIVE_REDIRECT_URL
);
const downloadFile = ({ id, access_token, path }) => {
return new Promise((resolve, reject) => {
const dest = fs.createWriteStream(path);
const oauth2Client = getOauth2Client();
oauth2Client.setCredentials({ access_token });
const drive = google.drive({
version: 'v3',
auth: oauth2Client
});
drive.files.get(
{ fileId: id, alt: 'media' }, { responseType: 'stream' },
(err, res) => {
if (err) reject(err);
res.data
.on('end', () => {
console.log('Done');
})
.on('error', _e => {
console.log('Error', _e);
if (_e) reject(_e);
})
.pipe(dest);
dest.on('finish', () => {
console.log('Download finished');
resolve(true);
});
}
);
});
};

This is because in the renderer process, Google's gaxios modules uses the fetch API instead of Node's http. Fetch API returns a ReadableStream unlike http which returns a Node.js Readable. Currently there's no way to change the default adapter. You can use this quick workaround the convert it.
// Transforms a web ReadableStream to Node.js Readable
function toNodeReadable(webStream) {
const reader = webStream.getReader();
const rs = new Readable();
rs._read = async () => {
const result = await reader.read();
if (!result.done) {
rs.push(Buffer.from(result.value));
} else {
rs.push(null);
}
};
return rs;
}
Usage with your code:
syncFileFromDrive(fileId, filePath) {
filePath.replace(userDataPath, '');
filePath = `${userDataPath}/${filePath}`;
filePath.replaceAll('//', '/');
logger.info(`Sync file from drive: Syncing file to path: ${filePath}`);
logger.info(`Sync file from drive: File id: ${fileId}`)
const dest = fs.createWriteStream(filePath);
let progress = 0;
this.drive.files.get({fileId, alt: 'media'}, {responseType: 'stream'}).then(res => {
console.log(res)
console.log(res.data)
toNodeReadable(res.data)
.on('end', () => {
console.log('Done downloading file.');
folderStructure.buildFileMenu()
resolve(dest)
})
.on('error', err => {
console.error('Error downloading file.');
reject(err);
})
.on('data', d => {
progress += d.length;
if (process.stdout.isTTY) {
process.stdout.clearLine();
process.stdout.cursorTo(0);
process.stdout.write(`Downloaded ${progress} bytes`);
}
})
.pipe(dest);
});
}

Firebase function storage POST image issue

I have encountered a problem when following a maximilian schwarzmüller course, which has otherwise been great: https://www.youtube.com/watch?v=qZ1EFnFOGvE
The image logs in the Firebase console as uploaded, recognises the type of file/size etc. But continually loads and never displays the image. I use a post request in POSTMAN to upload the image.
When I upload manually to firebase on their UI, everything works fine.
My code:
const functions = require("firebase-functions");
const os = require("os");
const path = require("path");
const spawn = require("child-process-promise").spawn;
const cors = require("cors")({ origin: true });
const Busboy = require("busboy");
const fs = require("fs");
const gcconfig = {
projectId: "REDACTED",
keyFilename: "REDACTED"
};
const gcs = require("#google-cloud/storage")(gcconfig);
//
exports.onFileChange = functions.storage.object().onFinalize(event => {
const object = event.data;
const bucket = object.bucket;
const contentType = object.contentType;
const filePath = object.name;
console.log("File change detected, function execution started");
if (object.resourceState === "not_exists") {
console.log("We deleted a file, exit...");
return;
}
if (path.basename(filePath).startsWith("resized-")) {
console.log("We already renamed that file!");
return;
}
const destBucket = gcs.bucket(bucket);
const tmpFilePath = path.join(os.tmpdir(), path.basename(filePath));
const metadata = { contentType: contentType };
return destBucket
.file(filePath)
.download({
destination: tmpFilePath
})
.then(() => {
return spawn("convert", [tmpFilePath, "-resize", "500x500", tmpFilePath]);
})
.then(() => {
return destBucket.upload(tmpFilePath, {
destination: "resized-" + path.basename(filePath),
metadata: metadata
});
});
});
exports.uploadFile = functions.https.onRequest((req, res) => {
cors(req, res, () => {
if (req.method !== "POST") {
return res.status(500).json({
message: "Not allowed"
});
}
const busboy = new Busboy({ headers: req.headers });
let uploadData = null;
busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {
const filepath = path.join(os.tmpdir(), filename);
uploadData = { file: filepath, type: mimetype };
file.pipe(fs.createWriteStream(filepath));
});
busboy.on("finish", () => {
const bucket = gcs.bucket("REDACTED");
bucket
.upload(uploadData.file, {
uploadType: "media",
metadata: {
metadata: {
contentType: uploadData.type
}
}
})
.then(() => {
res.status(200).json({
message: "It worked!"
});
})
.catch(err => {
res.status(500).json({
error: err
});
});
});
busboy.end(req.rawBody);
});
});
My security rules:
rules_version = '2';
service firebase.storage {
match /b/{bucket}/o {
match /{allPaths=**} {
allow read, write:if true;
}
}
}

Sending multiple files to google bucket with multer - why are my files empty?

I previously had a single file upload set up and working properly. Now I need to make it handle multiple files.
Here is my code right now:
const multer = require('multer')
const { Storage } = require('#google-cloud/storage')
const storage = new Storage()
const m = multer({ storage: multer.memoryStorage() })
module.exports = app => {
app.use('/', router)
router.post(
'/reader-:shortId/file-upload',
passport.authenticate('jwt', { session: false }),
m.array('files'),
async function (req, res) {
const bucketName = req.params.shortId.toLowerCase()
await storage.createBucket(bucketName)
bucket = storage.bucket(bucketName)
let promises = []
req.files.forEach((file) => {
const blob = bucket.file(file.originalname)
const newPromise = new Promise((resolve, reject) => {
blob.createWriteStream({
metadata: { contentType: file.mimetype }
}).on('finish', async response => {
await blob.makePublic()
resolve(response)
}).on('error', err => {
reject('upload error: ', err)
}).end()
})
promises.push(newPromise)
})
Promise.all(promises).then((response) => {
// the response I get here is [undefined, undefined]
res.status(200).send(response)
}).catch((err) => {
res.status(400).send(err.message)
});
})
}
req.files does give me an array of files, with a buffer and a size that makes sense.
The promises all resolve.
But once I check the files in the google bucket, they have the right name but don't have any content (and size of 0)
As I said before, it was working when I was doing it with one file (using m.single('file')
I don't want to use the bucket as the destination with multer setup because I also have to change the file name before uploading to google bucket.
edit: this is the code example given by google cloud documentations for single file uploads (https://cloud.google.com/nodejs/getting-started/using-cloud-storage):
function sendUploadToGCS (req, res, next) {
if (!req.file) {
return next();
}
const gcsname = Date.now() + req.file.originalname;
const file = bucket.file(gcsname);
const stream = file.createWriteStream({
metadata: {
contentType: req.file.mimetype
},
resumable: false
});
stream.on('error', (err) => {
req.file.cloudStorageError = err;
next(err);
});
stream.on('finish', () => {
req.file.cloudStorageObject = gcsname;
file.makePublic().then(() => {
req.file.cloudStoragePublicUrl = getPublicUrl(gcsname);
next();
});
});
stream.end(req.file.buffer);
}
I originally had something like that working, but I just don't understand where it is getting the file buffer data from. That is probably where things are different with multiple files.

I know its too late, but someone might looking an answer for uploading multiple files on Google Cloud Storage.
Dependencies:
Express
Google Cloud Library
Multer
Body Parser
This is the controller code.
exports.post_image_upload = async (req, res) => {
/** Check if file exist */
if (!req.files) {
res.status(400).send('No file uploaded.');
return;
}
let PublicUrls = []
req.files.forEach((file) => {
const blob = bucket.file(file.fieldname + '-' + Date.now() + path.extname(file.originalname))
const blobStream = blob.createWriteStream({
metadata: { contentType: file.mimetype }
})
blobStream.on('finish', ()=> {
blob.makePublic()
})
blobStream.on('error', err => {
//Put your error message here
})
blobStream.end(file.buffer)
const Url = `https://storage.googleapis.com/${bucket.name}/${blob.name}`
PublicUrls.push(Url)
})
res.send(PublicUrls)
}
Good Luck

Ok, turns out I had to change
.end()
to
.end(file.buffer)

Marie Pelletier, I think your approach is 100% right. I modified a little your code trying to avoid the async response:
let promises = []
req.files.forEach((file) => {
const blob = bucket.file(file.originalname)
const newPromise = new Promise((resolve, reject) => {
blob.createWriteStream({
metadata: { contentType: file.mimetype },
resumable: false //Good for small files
}).on('finish', () => {
const Url = `https://storage.googleapis.com/${bucket.name}/${blob.name}`;
resolve({ name: file.originalname, url: Url });
}).on('error', err => {
reject('upload error: ', err);
}).end(file.buffer);
})
promises.push(newPromise);
})
Promise.all(promises).then((response) => {
res.status(200).send(response)
}).catch((err) => {
res.status(400).send(err.message)
});
This way, I didn't get 'undefined' anymore.

How to receive multiple requests with Express.JS

I'm writing an Angular 6 + Express.JS app and now I stuck with the following problem: when there are some multiple requests made at the same time, sometimes (especially when there are more than 4 requests) all of them response with 404 or even get cancelled. Is there any problem with the way I handle requests in Express or I should add some tweaks for concurrent requests?
Requests:
let requests = [];
files.forEach((file) => {
if (file.type.toLowerCase().includes('zip')) {
requests.push(this.imagesService.uploadArchive(file).pipe(first()));
} else {
requests.push(this.imagesService.saveImage(file).pipe(first()));
}
});
forkJoin(requests).subscribe(
(res) => res.forEach(response => {
this.onSave.emit(response);
}),
(error) => {
console.error(error);
},
() => {
this.close.emit();
}
);
Express handling routes:
router.post('/images',
formidable({
encoding: 'utf-8',
uploadDir: path.resolve(__dirname, '..', '..', 'uploads'),
multiples: true,
keepExtensions: true
}),
(req, res, next) => {
const image = req.fields;
const data = req.files;
image.path = data.image.path;
const file = fs.createReadStream(image.path);
saveImage(image).then(
result => {
if (result) {
res.status(200).send(result);
} else {
console.error("Cannot save image");
res.status(400).send("Cannot save image");
}
}).catch(e => console.error(e.stack));
});
Responses:
UPDATE
router.post('/archives',
formidable({
encoding: 'utf-8',
uploadDir: path.resolve(__dirname, '..', '..', 'uploads'),
multiples: true,
keepExtensions: true
}),
(req, res, next) => {
const data = req.files;
let promises = [];
fs.readFile(data.archive.path, async (err, archive) => {
if (err) throw err;
await extractImagesFromZip(archive, data.archive.path).then((images) =>
images.forEach((image) => {
promises.push(
saveImage(image).then(
result => {
if (result) {
result.path = result.path.split('/').pop();
return result;
} else {
console.error("Cannot save image " + image.name);
fs.unlink(image.path, () => {});
}
}).catch(e => {
fs.unlink(image.path, () => {});
console.error(e.stack)
})
);
})
);
Promise.all(promises)
.then((result) => {
if (result.length > 0) {
res.status(200).send(result)
} else {
res.status(400).send("None images were saved")
}
}).catch((error) => {
console.log(error.stack);
res.status(400).send("None images were saved")
});
});
}
);
export const extractImagesFromZip = (file, link) => {
let promises = [];
var zip = new JSZip();
return zip.loadAsync(file)
.then((archive) => {
Object.values(archive.files).filter(
f =>
['.jpg', '.jpeg', '.png'].some((suffix) => f.name.toLowerCase().endsWith(suffix))
&& ![...f.name.toLowerCase().split('/')].pop().startsWith('.')
&& !f.dir
).forEach(f => promises.push(zip.file(f.name).async('nodebuffer').then((content) => {
const ext = f.name.split('.').pop().toLowerCase();
var dest = path.resolve(__dirname, '..', '..') + '/uploads/upload_'
+ crypto.randomBytes(Math.ceil(1322)).toString('hex').slice(0, 32).toLowerCase()
+ '.' + ext;
return new Promise((res, rej) => {
fs.writeFile(dest, content, (err) => {
if (err) rej(err);
res(new Promise((resolve, reject) => {
fs.readFile(dest, (erro, data) => {
if (erro) reject(erro);
if (data) resolve({
name: f.name,
type: 'image/' + (ext === 'jpg' ? 'jpeg' : ext),
path: dest
});
});
}));
});
});
})));
fs.unlink(link, () => {});
return Promise.all(promises);
});
}
export const saveImage = (image) => {
return database.raw(
"INSERT INTO images (name, type, path) " +
"VALUES (?, ?, ?) " +
"RETURNING name, type, path, id",
[image.name, image.type, image.path]
).then(data => data.rows[0]).catch(e => console.error(e.stack));
};
UPDATE 2
Everything works fine if user and server are on localhost (regardless server running with nginx or without it), but problem appears when server is remote

Such code worked
public async uploadFiles(files: File[]) {
of(files)
.pipe(
concatMap(files =>
files.map(file => {
return this.imagesService
.saveImage(file)
.pipe(
map(),
catchError((error, caught) => {
console.error(error);
return empty();
})
);
})
),
concatAll(),
toArray(),
map(res => console.log)
)
.subscribe();
}