We Keep Coding

Execute function from S3 downloaded JavaScript file without creating a local file on disk

Assume that we download the script below from S3:
const sum_two = (a, b) => {
return a + b;
}
Downloading using this:
async function downloadFileS3(filenameDownload) {
const AWS = require('aws-sdk');
const fs = require('fs');
AWS.config.update(
{
accessKeyId: "XXXXXXXXXXXXXXXXXXXXXXXX",
secretAccessKey: "YYYYYYYYYYYYYYYYYYYYY",
correctClockSkew: true,
}
);
const params = {
Bucket: 'ZZZZZZZZZZZZZZZZZZZZZZZZ',
Key: filenameDownload,
}
const s3 = new AWS.S3();
try {
const data = await s3.getObject(params).promise();
if (data) {
// do something with data.Body
const downloaded = data.Body.toString('utf-8');
// grab sum_two from "downloaded" and use below
.......
.......
.......
}
}
catch (error) {
console.log(error);
}
}
How can we extract sum_two (and use it afterwards) from downloaded WITHOUT creating a local file ?
Meaning I don't want to use the code below !!!
// fs.writeFile(`${filenameDownload}`, downloaded, function (err) {
// if (err) {
// console.log('Failed to create file', err);
// }
// console.log(sum_two(.... , ....));
// });

You can use node.js native vm module for this.
Create a Script from the downloaded code
Run it in the current context
After that the downloaded function will become available for usage:
const vm = require('vm');
// ...
const downloaded = data.Body.toString('utf-8');
const script = new vm.Script(downloaded);
script.runInThisContext()
console.log(sum_two(1,2))

How to extract .zip files to a directory within a lambda function

The issue that I am running into is that when I test the function against a .zip file when the function gets to the fs.createReadStream&Zip the function is not running or returning an error and I would like to get an understanding on what I am doing wrong and how a correct solution would look.
const AWS = require('aws-sdk');
const fs = require('fs');
const mkdirp = require('mkdirp');
const unzipper = require('unzipper');
exports.handler = async (event, context) => {
// Variables for bucket init
let sourceBucket = 'am-doc-mgmt-s3-dev-landing';
let storageBucket = 'am-doc-mgmt-s3-dev';
// Variables for folder init and Buffer config
const localZippedFolder = '/tmp/ZippedStudentData/';
const localUnzippedFolder = '/tmp/UnzippedStudentData/';
const ZipBuffer = Buffer.from(localZippedFolder, 'base64');
const UnzippedBuffer = Buffer.from(localUnzippedFolder, 'base64');
// Inits AWS s3 Bucket and DynamoDB
let s3 = new AWS.S3();
let docClient = new AWS.DynamoDB.DocumentClient({ region: 'us-east-1' });
// Gets the file bucket and file name of the s3 object from context
let fileBucket = event.Records[0].s3.bucket.name;
let fileName = event.Records[0].s3.object.key;
let params = {
Bucket: fileBucket,
Key: fileName
};
// Creates temporary variables
let tempFile = localZippedFolder + fileBucket;
let tempUnzippedFile = localUnzippedFolder + fileBucket;
// Make Directories for Zipped and Unzipped files
try {
const zipDirFolder = await mkdirp(localZippedFolder, { recursive: true })
const unZipDirFolder = await mkdirp(localUnzippedFolder, { recursive: true });
console.log('SUCCESS: unzipped directory created!');
console.log('SUCCESS: zipped directory create!')
// Download files from s3 Bucket
let newFolder = await s3.getObject(params).promise()
.then(data => {
console.log(data);
return data;
});
// Extract files from zipped folder and store them in a local directory
fs.createReadStream(params.Key)
.pipe(unzipper.Extract({path: unZipDirFolder}))
.on('finish', () => {
fs.readdir(unZipDirFolder);
}).on('error', (err) => {
// error handling here
console.log(err);
});
}
catch (error) {
console.log(error);
}
};
error: not getting anything back from the fs.createReadStream function. Its as if it just skips over the function.

It's honestly hard to figure out what problem you're really trying to solve since you just aren't very specific about that. If you want the containing async function to not resolve it's promise until the unzipping is done, you can wrap the stream in a promise like this:
const AWS = require('aws-sdk');
const fs = require('fs');
const mkdirp = require('mkdirp');
const unzipper = require('unzipper');
exports.handler = async (event, context) => {
// Variables for bucket init
let sourceBucket = 'am-doc-mgmt-s3-dev-landing';
let storageBucket = 'am-doc-mgmt-s3-dev';
// Variables for folder init and Buffer config
const localZippedFolder = '/tmp/ZippedStudentData/';
const localUnzippedFolder = '/tmp/UnzippedStudentData/';
const ZipBuffer = Buffer.from(localZippedFolder, 'base64');
const UnzippedBuffer = Buffer.from(localUnzippedFolder, 'base64');
// Inits AWS s3 Bucket and DynamoDB
let s3 = new AWS.S3();
let docClient = new AWS.DynamoDB.DocumentClient({ region: 'us-east-1' });
// Gets the file bucket and file name of the s3 object from context
let fileBucket = event.Records[0].s3.bucket.name;
let fileName = event.Records[0].s3.object.key;
let params = {
Bucket: fileBucket,
Key: fileName
};
// Creates temporary variables
let tempFile = localZippedFolder + fileBucket;
let tempUnzippedFile = localUnzippedFolder + fileBucket;
// Make Directories for Zipped and Unzipped files
try {
const zipDirFolder = await mkdirp(localZippedFolder, { recursive: true })
const unZipDirFolder = await mkdirp(localUnzippedFolder, { recursive: true });
console.log('SUCCESS: unzipped directory created!');
console.log('SUCCESS: zipped directory create!')
// Download files from s3 Bucket
let newFolder = await s3.getObject(params).promise();
await new Promise((resolve, reject) => {
// Extract files from zipped folder and store them in a local directory
fs.createReadStream(params.Key)
.pipe(unzipper.Extract({path: unZipDirFolder}))
.on('finish', resolve);
.on('error', reject);
});
} catch (error) {
console.log(error);
// rethrow error so caller sees the error
throw error;
}
};
And, your caller using this exported function will HAVE to use .then() or await on the returned promise to know when it's done. And, use .catch() or try/catch around await to catch errors.

If someone's open to using Python, they can use a buffer to read and unzip the files. Something like this:
zipped_file = s3_resource.Object(bucket_name=sourcebucketname, key=filekey)
buffer = BytesIO(zipped_file.get()["Body"].read())
zipped = zipfile.ZipFile(buffer)
for file in zipped.namelist():
logger.info(f'current file in zipfile: {file}')
final_file_path = file + '.extension'
with zipped.open(file, "r") as f_in:
content = f_in.read()
destinationbucket.upload_fileobj(io.BytesIO(content),
final_file_path,
ExtraArgs={"ContentType": "text/plain"}
)
There's also a tutorial here: https://betterprogramming.pub/unzip-and-gzip-incoming-s3-files-with-aws-lambda-f7bccf0099c9

how to pipe an archive (zip) to an S3 bucket

I’m a bit confused with how to proceed. I am using Archive ( node js module) as a means to write data to a zip file. Currently, I have my code working when I write to a file (local storage).
var fs = require('fs');
var archiver = require('archiver');
var output = fs.createWriteStream(__dirname + '/example.zip');
var archive = archiver('zip', {
zlib: { level: 9 }
});
archive.pipe(output);
archive.append(mybuffer, {name: ‘msg001.txt’});
I’d like to modify the code so that the archive target file is an AWS S3 bucket. Looking at the code examples, I can specify the bucket name and key (and body) when I create the bucket object as in:
var s3 = new AWS.S3();
var params = {Bucket: 'myBucket', Key: 'myMsgArchive.zip' Body: myStream};
s3.upload( params, function(err,data){
…
});
Or
s3 = new AWS.S3({ parms: {Bucket: ‘myBucket’ Key: ‘myMsgArchive.zip’}});
s3.upload( {Body: myStream})
.send(function(err,data) {
…
});
With regards to my S3 example(s), myStream appears to be a readable stream and I am confused as how to make this work as archive.pipe requires a writeable stream. Is this something where we need to use a pass-through stream? I’ve found an example where someone created a pass-through stream but the example is too terse to gain proper understanding. The specific example I am referring to is:
Pipe a stream to s3.upload()
Any help someone can give me would greatly be appreciated. Thanks.

This could be useful for anyone else wondering how to use pipe.
Since you correctly referenced the example using the pass-through stream, here's my working code:
1 - The routine itself, zipping files with node-archiver
exports.downloadFromS3AndZipToS3 = () => {
// These are my input files I'm willing to read from S3 to ZIP them
const files = [
`${s3Folder}/myFile.pdf`,
`${s3Folder}/anotherFile.xml`
]
// Just in case you like to rename them as they have a different name in the final ZIP
const fileNames = [
'finalPDFName.pdf',
'finalXMLName.xml'
]
// Use promises to get them all
const promises = []
files.map((file) => {
promises.push(s3client.getObject({
Bucket: yourBubucket,
Key: file
}).promise())
})
// Define the ZIP target archive
let archive = archiver('zip', {
zlib: { level: 9 } // Sets the compression level.
})
// Pipe!
archive.pipe(uploadFromStream(s3client, 'someDestinationFolderPathOnS3', 'zipFileName.zip'))
archive.on('warning', function(err) {
if (err.code === 'ENOENT') {
// log warning
} else {
// throw error
throw err;
}
})
// Good practice to catch this error explicitly
archive.on('error', function(err) {
throw err;
})
// The actual archive is populated here
return Promise
.all(promises)
.then((data) => {
data.map((thisFile, index) => {
archive.append(thisFile.Body, { name: fileNames[index] })
})
archive.finalize()
})
}
2 - The helper method
const uploadFromStream = (s3client) => {
const pass = new stream.PassThrough()
const s3params = {
Bucket: yourBucket,
Key: `${someFolder}/${aFilename}`,
Body: pass,
ContentType: 'application/zip'
}
s3client.upload(s3params, (err, data) => {
if (err)
console.log(err)
if (data)
console.log('Success')
})
return pass
}

The following example takes the accepted answer and makes it work with local files as requested.
const archiver = require("archiver")
const fs = require("fs")
const AWS = require("aws-sdk")
const s3 = new AWS.S3()
const stream = require("stream")
const zipAndUpload = async () => {
const files = [`test1.txt`, `test2.txt`]
const fileNames = [`test1target.txt`, `test2target.txt`]
const archive = archiver("zip", {
zlib: { level: 9 } // Sets the compression level.
})
files.map((thisFile, index) => {
archive.append(fs.createReadStream(thisFile), { name: fileNames[index] })
})
const uploadStream = new stream.PassThrough()
archive.pipe(uploadStream)
archive.finalize()
archive.on("warning", function (err) {
if (err.code === "ENOENT") {
console.log(err)
} else {
throw err
}
})
archive.on("error", function (err) {
throw err
})
archive.on("end", function () {
console.log("archive end")
})
await uploadFromStream(uploadStream)
console.log("all done")
}
const uploadFromStream = async pass => {
const s3params = {
Bucket: "bucket-name",
Key: `streamtest.zip`,
Body: pass,
ContentType: "application/zip"
}
return s3.upload(s3params).promise()
}
zipAndUpload()

Error: Cloud Functions for Firebase spawn EACCES (ghostscript)

I tried using Firebase Cloud Functions to create a thumbnail of a PDF file.
After the call of gs I get the following error:
2018-06-12T11:29:08.685Z E makeThumbnail: Error: spawn EACCES
at exports._errnoException (util.js:1020:11)
at ChildProcess.spawn (internal/child_process.js:328:11)
at exports.spawn (child_process.js:370:9)
at Object.exec (/user_code/node_modules/gs/index.js:86:28)
at Promise (/user_code/index.js:95:12)
at mkdirp.then.then (/user_code/index.js:86:12)
2018-06-12T11:29:08.698166767Z D makeThumbnail: Function execution took 780 ms, finished with status: 'error'
Is it necessary to use a component like ghostscript to use a plan other than Spark?
In addition, my code. Maybe I just do not see my problem in the code
const functions = require('firebase-functions');
const mkdirp = require('mkdirp-promise');
const gcs = require('#google-cloud/storage')();
const admin = require('firebase-admin');
const spawn = require('child-process-promise').spawn;
const path = require('path');
const os = require('os');
const fs = require('fs');
const gs = require('gs');
const THUMB_MAX_HEIGHT = 200;
const THUMB_MAX_WIDTH = 200;
const THUMB_PREFIX = 'thumb_';
const gs_exec_path = path.join(__dirname, './lambda-ghostscript/bin/gs');
try{admin.initializeApp(functions.config().firebase); } catch(e) {}
exports.makeThumbnail = functions.storage.object().onFinalize((object) => {
const filePath = object.name;
const contentType = object.contentType;
const fileDir = path.dirname(filePath);
const fileName = path.basename(filePath);
const thumbFilePath = path.normalize(path.join(fileDir, `${THUMB_PREFIX} ${fileName}`));
const tempLocalFile = path.join(os.tmpdir(), filePath);
const tempLocalDir = path.dirname(tempLocalFile);
const tempLocalThumbFile = path.join(os.tmpdir(), thumbFilePath);
const tmp_dir = os.tmpdir();
if (fileName.startsWith(THUMB_PREFIX)) {
console.log('Is thumbnail');
return null;
}
const bucket = gcs.bucket(object.bucket);
const file = bucket.file(filePath);
const thumbFile = bucket.file(thumbFilePath);
const metadata = {
contentType: contentType,
};
return mkdirp(tmp_dir).then(() => {
console.log("Dir Created");
console.log(tempLocalFile);
return file.download({destination: tempLocalFile});
}).then(() => {
console.log("File downloaded");
if(!contentType.startsWith("image/")){
return new Promise((resolve, reject) => {
const pg= 1;
gs().batch().nopause()
.option(`-dFirstPage=${pg}`)
.option(`-dLastPage=${pg}`)
.executablePath(gs_exec_path)
.device('png16m')
.output(tempLocalThumbFile+".png")
.input(tempLocalFile)
.exec(err => err ? reject(err) : resolve());
});
}
else
{
var args = [ tempLocalFile, '-thumbnail', `${THUMB_MAX_WIDTH}x${THUMB_MAX_HEIGHT}>`, tempLocalThumbFile ];
return spawn('convert', args, {capture: ['stdout', 'stderr']});
}
}).then(() => {
return bucket.upload(tempLocalThumbFile, { destination: thumbFilePath });
}).then(() => {
fs.unlinkSync(tempLocalFile);
fs.unlinkSync(tempLocalThumbFile);
return result[0];
});
});

After hours of scratching my head and running same code over and over again pointlessly, I've finally found the problem!
The executable path that you've defined is not correct. It should be 'gs'.
Here's a complete gs() call sample:
gs()
.batch()
.option('-dFirstPage=2')
.option('-dLastPage=2')
.nopause()
.res(90)
.executablePath('gs')
.device('jpeg')
.output(tempNewPath2)
.input(tempFilePath)
.exec((err, stdout, stderr) => {
if (!err) {
console.log('gs executed w/o error');
console.log('stdout', stdout);
console.log('stderr', stderr);
resolve();
} else {
console.log('gs error:', err);
reject(err);
}
});
For more help, you can go through a sample repo that I created for this issue
https://github.com/krharsh17/ghostscript-firebase-sample

Watching a directory with nodejs - Not registering files uploaded by ftp

I'm trying to watch for any newly added files to an ftp server, which has the directory mapped to a drive on the server that's running the node application. The problem is that it doesn't register any events for files added through ftp; when files are modified or created through the node application they are picked up fine.
I'm currently using chokidar to watch the directory and log any events with the simple code below:
const watcher = chokidar.watch('./myDir', {
persistent: true,
awaitWriteFinish: {
stabilityThreshold: 2000,
pollInterval: 100
}
});
watcher
.on('add', path => console.log(`File ${path} has been added`))
.on('change', path => console.log(`File ${path} has been changed`));
I've added the awaitWriteFinish option to try to see if it will register when the file is completed from the ftp transfer, but with no joy.
Any suggestions?

You can watch a directory using the native module fs:
const fs = require('fs');
const folderPath = './test';
const pollInterval = 300;
let folderItems = {};
setInterval(() => {
fs.readdirSync(folderPath)
.forEach((file) => {
let path = `${folderPath}/${file}`;
let lastModification = fs.statSync(path).mtimeMs;
if (!folderItems[file]) {
folderItems[file] = lastModification;
console.log(`File ${path} has been added`);
} else if (folderItems[file] !== lastModification) {
folderItems[file] = lastModification;
console.log(`File ${path} has been changed`);
}
});
}, pollInterval);
But the above example will not watch the files in subfolders. Another approach to watch all subfolders, is to use unix find through the node child_process.exec function.
const fs = require('fs');
const {execSync} = require('child_process');
const folderPath = './test';
const pollInterval = 500;
let folderItems = {};
setInterval(() => {
let fileList = execSync(`find ${folderPath}`).toString().split('\n');
for (let file of fileList) {
if (file.length < 1) continue;
let lastModification = fs.statSync(file).mtimeMs;
if (!folderItems[file]) {
folderItems[file] = lastModification;
console.log(`File ${file} has been added`);
} else if (folderItems[file] !== lastModification) {
folderItems[file] = lastModification;
console.log(`File ${file} has been changed`);
}
}
}, pollInterval);