I'm using Node 12.x version to write my Lambda function. Here is the Parsing error that I am getting. What could be the reason?
Update
const im = require("imagemagick");
const fs = require("fs");
const os = require("os");
const uuidv4 = require("uuid/v4");
const {promisify} = require("util");
const AWS = require('aws-sdk');
const resizeAsync = promisify(im.resize)
const readFileAsync = promisify(fs.readFile)
const unlinkAsync = promisify(fs.unlink)
AWS.config.update({region: 'ap-south-1'})
const s3 = new AWS.S3();
exports.handler = async (event) => {
let filesProcessed = event.Records.map((record) => {
let bucket = record.s3.bucket.name;
let filename = record.s3.object.key;
//Fetch filename from S3
var params = {
Bucket: bucket,
Key: filename
};
//let inputData = await s3.getObject(params).promise()
let inputData = await s3.getObject(params).promise();
//Resize the file
let tempFile = os.tmpdir() + '/' + uuidv4() + '.jpg';
let resizeArgs = {
srcData: inputData.Body,
dstPath: tempFile,
width: 150
};
await resizeAsync(resizeArgs)
//Read the resized File
let resizedData = await readFileAsync(tempFile)
//Upload the resized file to S3
let targetFilename = filename.substring(0, filename.lastIndexOf('.') + '-small.jpg')
var params = {
Bucket: bucket + '-dest',
Key: targetFilename,
Body: new Buffer(resizedData),
ContentType: 'image/jpeg'
}
await s3.putObject(params).promise();
return await unlinkAsync(tempFile)
})
await Promise.all(filesProcessed)
return "done"
}
Here is the same code. I am getting Unexpected token S3 error when hovering the red mark (shown in the image)
What you can do is, declare inputData as below and initialize it with the response from the getObject.
let inputData;
var params = {
Bucket: "examplebucket",
Key: "HappyFace.jpg"
};
s3.getObject(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else inputData = data; // successful response
});
For more, you can refer here
Related
I want a make an API that will take a file or folder path from the user and upload it to AWS s3 I made progress but
when the user gives a file path it's searching the file path in the server, not in the user's pc
I know I made a mistake but I don't know how to connect API from the users pc and get access to system files
here is code for the post route
router.post("/create/:id", auth, async (req, res) => {
try {
let form = new multiparty.Form();
form.parse(req, async (err, fields, files) => {
console.log(fields);
console.log(files);
//check if user has access to project
const user_id = req.userId;
const project_id = req.params.id;
const user_access = await check_user_access_project(user_id, project_id);
const user = await User.findById(user_id);
const project = await Project.findById(project_id);
if (user_access === 1) {
//create version
const version = new Version({
project_id: project_id,
user_id: user_id,
versionName: fields.versionName[0],
version_description: fields.versionDescription[0],
version_file: [],
});
const version_data = await version.save();
console.log(version_data);
let version_id = version_data._id;
//sync folders to s3
const version_folder_path = fields.files_path[0];
let key = `${user.firstName}_${user_id}/${project.projectName}/${fields.versionName[0]}`;
const version_folder_list = await sync_folders(
version_folder_path,
key
);
console.log("version folder list", version_folder_list);
//update version with version folders
await Version.findByIdAndUpdate(
version_id,
{
$set: {
version_file: version_folder_list,
},
},
{ new: true }
);
//wait for version update
await version.save();
//send response
res.json({
success: true,
version: version_data,
});
} else {
res.status(401).json({
success: false,
message: "User does not have access to project",
});
}
});
} catch (error) {
res.status(400).json({ message: error.message });
}
});
here is the folder sync code
const sync_folders = async (folder_path, key) => {
function getFiles(dir, files_) {
files_ = files_ || [];
var files = fs.readdirSync(dir);
for (var i in files) {
var name = dir + "/" + files[i];
if (fs.statSync(name).isDirectory()) {
getFiles(name, files_);
} else {
files_.push(name);
}
}
return files_;
}
const files = getFiles(folder_path);
console.log(files);
const fileData = [];
for (let i = 0; i < files.length; i++) {
const file = files[i];
console.log(file);
const fileName = file.split("/").pop();
const fileType = file.split(".").pop();
const fileSize = fs.statSync(file).size;
const filePath = file;
const fileBuffer = fs.readFileSync(filePath);
//folder is last part of folder path (e.g. /folder1/folder2/folder3)
const folder = folder_path.split("/").pop();
console.log("folder: " + folder);
//split filepath
const filePath_ = filePath.split(folder).pop();
let filekey = key + "/" + folder + filePath_;
console.log("filekey: " + filekey);
const params = {
Bucket: bucket,
Key: filekey,
Body: fileBuffer,
ContentType: fileType,
ContentLength: fileSize,
};
const data = await s3.upload(params).promise();
console.log(data);
fileData.push(data);
}
console.log("file data", fileData);
console.log("files uploaded");
return fileData;
};
if some buddy can help me pls I need your help
You need to post the item in a form rather than just putting the directory path of user in and then upload the result to your s3 bucket.
This might be a good start if you're new to it:
https://www.w3schools.com/nodejs/nodejs_uploadfiles.asp
The script used when trying to get contents from the csv stored in the s3 bucket
const mysql = require("mysql");
const fs = require("fs");
const { google } = require("googleapis");
const AWS = require("aws-sdk");
const client = new AWS.SecretsManager({ region: "eu-west-1" });
const analyticsreporting = google.analyticsreporting("v4");
const csv = require('ya-csv')
const fastCsv = require('fast-csv')
const s3 = new AWS.S3();
const getParams = {
Bucket: 'data',
Key: 'athena_test/nameplate.csv'
};
exports.handler = async (context, event) => {
const data = await s3.getObject(getParams, function (err, data){
if(err){console.log("ERROR: ",err)}
else {return data}
})
console.log(data.Body)
}
the console log returns undefined rather than the contents of the csv
Hey you can try this one:-
const csv = require('#fast-csv/parse');
const s3Stream = await s3.getObject(params).createReadStream();
const data = await returnDataFromCSV();
console.log(data.Body);
const returnDataFromCSV =()=> {
let promiseData = new Promise((resolve, reject) => {
const parser = csv
.parseStream(csvFile, { headers: true })
.on("data", (data) => {
console.log('Parsed Data:-', data);
})
.on("end", ()=> {
resolve("CSV finished here");
})
.on("error",()=> {
reject("if failed");
});
});
try {
return await promiseData;
} catch (error) {
console.log("Get Error: ", error);
return error;
}
}
CreateStream: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Request.html#createReadStream-property
Here is my code. the variable file in the code
console.log(JSON.stringify(file));
is undefined.
const functions = require('firebase-functions');
const admin = require('firebase-admin');
const Buffer = require('buffer').Buffer;
const stream = require('stream');
exports.updateUserInfo = functions.https.onRequest(function(req, res){
const uid = req.params.uid;
const name = req.query.name;
const gender = req.query.gender;
const birthday = req.query.birthday;
const base64Image = req.query.profileImage;
let bufferStream = new stream.PassThrough();
bufferStream.end(new Buffer.from(base64Image, 'base64'));
let bucket = admin.storage().bucket();
let file = bucket.file('user_images/' + uid + '/example.jpg');
bufferStream.pipe(file.createWriteStream({
metadata: {
contentType: 'image/jpeg'
}
}))
.on('error', error => {
console.log("error", error);
})
.on('finish', (file) => {
console.log(JSON.stringify(file));
});
})
Stream event finish informs that operation is done, and passes no arguments. See https://nodejs.org/api/stream.html#stream_event_finish.
If you want use file object after operation is done:
let file = bucket.file('user_images/' + uid + '/example.jpg');
bufferStream.pipe(...)
.on('finish', () => {
console.log(file);
});
I'm trying to get an s3.getObject() running inside an async getInitialProps() function in a nextJS project, but I can't for the love of it figure out how to get the results prepped to they can be returned as an object (which is needed for getInitialProps() and nextJS' SSR to work properly).
Here is the code:
static async getInitialProps({ query }) {
const AWS = require('aws-sdk');
const s3 = new AWS.S3({
credentials: {
accessKeyId: KEY
secretAccessKey: KEY
}
});
// The id from the route (e.g. /img/abc123987)
let filename = query.id;
const params = {
Bucket: BUCKETNAME
Key: KEYDEFAULTS + '/' + filename
};
const res = await s3.getObject(params, (err, data) => {
if (err) throw err;
let imgData = 'data:image/jpeg;base64,' + data.Body.toString('base64');
return imgData;
});
return ...
}
The idea is to fetch an image from S3 and return it as base64 code (just to clear things up).
From your code, s3.getObject, works with callback. you need to wait for the callback to be called.
You can achieve it by converting this callback into a promise.
static async getInitialProps({ query }) {
const AWS = require('aws-sdk');
const s3 = new AWS.S3({
credentials: {
accessKeyId: KEY
secretAccessKey: KEY
}
});
// The id from the route (e.g. /img/abc123987)
let filename = query.id;
const params = {
Bucket: BUCKETNAME
Key: KEYDEFAULTS + '/' + filename
};
const res = await new Promise((resolve, reject) => {
s3.getObject(params, (err, data) => {
if (err) reject(err);
let imgData = 'data:image/jpeg;base64,' + data.Body.toString('base64');
resolve(imgData);
});
});
return ...
}
I am trying to download a zip file in my MERN application. I am getting the file in the response, how ever the client does not download the actual file. I am using archiver to zip files then return them in a fetch call.
Archive Service:
const archiver = require('archiver')
const zip = archiver('zip')
const path = require('path')
const fs = require('fs')
const appDir = path.dirname(require.main.filename)
exports.FileArchiver = function (feed, res) {
// const app = this.app;
const uploadsDir = path.join(appDir, '/uploads/');
const templatesDir = path.join(appDir, '/templates/');
const feedArray = feed.feed.data;
const extensions = [".jpg", ".png", ".svg"];
const feedArrayString = JSON.stringify(feedArray);
const feedArrayObject = JSON.parse(feedArrayString);
let imageArray = [];
let templateType = 'b'; //test
// grab image names from object
feedArrayObject.forEach(function(x){iterate(x)});
// remove duplicates
imageArray = uniq_fast(imageArray);
// zip images
for (let i = 0; i < imageArray.length; i++) {
console.log(imageArray[i])
const filePath = path.join(uploadsDir, imageArray[i]);
zip.append(fs.createReadStream(filePath), { name: 'images/'+imageArray[i] });
}
res.attachment(feed.name + '.zip');
zip.pipe(res);
zip.append(feedArrayString, { name: 'feed.json' })
zip.directory(templatesDir + '/' + templateType, false);
zip.on('error', (err) => { throw err; });
zip.on('warning', (err) => {
if (err.code === 'ENOENT') {
console.log('ENOENT for archive')
} else {
throw err;
}
});
zip.finalize();
return this;
}
Client side fetch:
export const downloadData = (url, _id, name, type) => {
return fetch(url, {method: 'GET'})
.then((res) => {
console.log(res);
return res;
})
}
Client side Headers attached:
content-disposition: attachment; filename="a_gpo.zip"
content-type: application/zip
The network request response returns 200 status and I can also see the attachment response in client contains zip file jargon. However the client does not return the actual file download.
Try to redirect the browser location to the URL
export const downloadData = (url) => {
window.location = url;
}