We Keep Coding

Node.js AWS Lambda putObject doesn't seem to finish

I have a Lambda function that is meant to download a directory of files from s3, convert them, delete the old files, and upload the new output files back to s3. The output for each file will be at least one file and a folder.
Everything seems to be working as intended, except for the upload. No errors are thrown, it just ends without putting.
I'm a novice, so feel free to point out I've done it all wrong.
exports.handler = async ({ dirName }) => {
// const jsonIn = JSON.parse(event.body);
// const dirName = jsonIn.dirName;
const localDir = `/tmp/${dirName}`;
const params = {
Bucket: 'to-pdf-test',
Delimiter: '/',
Prefix: dirName + '/',
StartAfter: dirName + '/'
};
var s3List;
var localList = [];
execSync(`mkdir ${localDir}`);
try {
s3List = await s3.listObjectsV2(params).promise();
} catch (e) {
throw e;
}
await Promise.all(
s3List.Contents.map(async (file) => {
let f = await getFiles(file);
localList.push(f);
})
).then(res => {console.log('Get Successful' + res) } )
.catch(err => {console.log('error' + err) } );
await Promise.all(
localList.map(async (file) => {
convertFile(file);
})
).then(res => {console.log('Convert Successful' + res) } )
.catch(err => {console.log('error' + err) } );
dirSync(localDir, async (filePath, stat) => {
let bucketPath = filePath.substring(5);
let uploadParams = { Bucket: 'to-pdf-test',
Key: `${bucketPath}`,
Body: fs.readFileSync(filePath) };
console.log('DS fPath ' + filePath);
console.log('DS bPath ' + bucketPath);
console.log(uploadParams.Body);
try {
let res = await s3.putObject(uploadParams).promise();
console.log('Upload Complete', res);
} catch (e) {
console.log('Error', e);
}
});
};
async function getFiles(file) {
let filePath = `/tmp/${file.Key}`;
let fileParams = {
Bucket: 'to-pdf-test',
Key: file.Key
};
try {
const { Body: inputFileBuffer } = await s3.getObject(fileParams).promise();
fs.writeFileSync(filePath, inputFileBuffer);
} catch (e) {
throw (e);
}
return filePath;
}
function convertFile(file) {
const noPath = getFilename(file);
const fPath = getFilePath(file);
if (path.extname(noPath) === '.msg') {
execSync(`cd ${fPath} && ${command} ${noPath}`);
} else {
console.log(`${noPath} not run. Not .msg`);
}
fs.unlinkSync(file);
}
function getFilename(fullPath) {
return fullPath.replace(/^.*[\\\/]/, '');
}
function getFilePath(fullPath) {
return fullPath.substring(fullPath.lastIndexOf('/'), 0);
}
function dirSync(dirPath, callback) {
fs.readdirSync(dirPath).forEach((name) => {
var filePath = path.join(dirPath, name);
var stat = fs.statSync(filePath);
if (stat.isDirectory()) {
dirSync(filePath, callback);
} else {
callback(filePath, stat);
}
});
}
I had the upload working in a previous version of this function, so thanks to this post for when it was working.

My solution for the moment - Read the local directory separately, push the paths of the files to localList then .map the array with all the paths to upload them.
localList = [];
//read dir and push to localList array
await dirSync(localDir, (filePath, stat) => {
localList.push(filePath);
});
console.log(localList);
await Promise.all(
localList.map( async (file) => {
let bucketPath = file.substring(5);
let uploadParams = {
Bucket: 'to-pdf-test',
Key: bucketPath,
Body: fs.readFileSync(file) };
console.log('Uploading', file);
await s3.putObject(uploadParams).promise()
.then((res) => {console.log('Upload Successful', bucketPath) } )
.catch((err) => {console.log('error' + err) } );
})
);
If there is better (or proper) way to do this, someone let me know :)

Extract matching row by comparing two CSV file in NodeJs

The scenario is I have two large CSV files csv1.csv and csv2.csv. In both the files, there is an email column and I have to read csv1.csv row by row and check if the email exists in csv2.csv and if matches write the row of csv2.csv in csv3.csv. I have tried read stream as well but it is not working as expected. Any guidance or help is appreciated.
Thanks to all in advance.
Following are the CSV files
csv1.csv
email,header1,header2
test1#example.com,test1,test1
test2#example.com,test2,test2
test3#example.com,test3,test3
test4#example.com,test4,test4
test5#example.com,test5,test5
csv2.csv
email,header1,header2
test4#example.com,test4,test4
test5#example.com,test5,test5
test6#example.com,test6,test6
test7#example.com,test7,test7
test8#example.com,test8,test8
Following is the code that I tried
const fs = require('fs');
const csv = require('fast-csv')
class CsvHelper {
static write(filestream, rows, options) {
return new Promise((res, rej) => {
csv.writeToStream(filestream, rows, options)
.on('error', err => rej(err))
.on('finish', () => res());
});
}
constructor(opts) {
this.headers = opts.headers;
this.path = opts.path;
this.writeOpts = {
headers: this.headers,
includeEndRowDelimeter: true
};
}
create(rows) {
return CsvHelper.write(fs.createWriteStream(this.path, { flags: 'a' }), rows, { ...this.writeOpts });
}
append(rows) {
return CsvHelper.write(fs.createWriteStream(this.path, { flags: 'a' }), rows, {
...this.writeOpts,
writeHeaders: false,
});
}
}
class Helper {
async matchCsv (outerRow) {
try {
const filePath2 = "csv2.csv";
const filePath3 = "csv3.csv";
let row = [];
const csvFile = new CsvHelper({
path: filePath3,
headers: ["Email", "Active"]
});
return new Promise((resolve, reject) => {
fs.createReadStream(filePath2)
.on("error", err => {
reject(err);
})
.pipe(csv.parse({headers: true}))
.on("error", err => {
reject(err);
})
.on("data", async innerRow => {
if(outerRow["email"] === innerRow["email"]) {
console.log("====================");
console.log("match found");
console.log(innerRow);
console.log("====================");
row.push([innerRow["email"], "yes"]);
console.log("row: ", row);
}
})
.on("finish", async() => {
if (!fs.existsSync(filePath3)) {
await csvFile.create(row).then(() => {
resolve("Done from matchCsv");
})
} else {
await csvFile.append(row).then(() => {
resolve("Done from matchCsv");
})
}
})
});
} catch (err) {
throw(err);
}
}
async generateCsv () {
try {
const filePath1 = "csv1.csv";
return new Promise((resolve, reject) => {
fs.createReadStream(filePath1)
.on("error", err => {
reject(err);
})
.pipe(csv.parse({headers: true}))
.on("error", err => {
reject(err);
})
.on("data", async outerRow => {
const result = await this.matchCsv(outerRow);
console.log("result: ", result);
})
.on("finish", () => {
resolve("Generated csv3.csv file.");
});
});
} catch (err) {
throw(err);
}
}
}
async function main() {
const helper = new Helper();
const result = await helper.generateCsv()
console.log(result);
}
main();

So the question is a little confusing, but I think I know what you want. Here's what I would do to check if the email exists. It will add all the rows to an array, cycle through them, then if the email address matches the email you're looking for, it will do something else... I think you said you wanted to write to a csv file again with the row, but that should be simple enough.
const csv = require('csv-parser');
const fs = require('fs');
const filepath = "./example_data.csv";
const emailAdd = "myemail#email.com";
var rowsArr = [];
fs.createReadStream(filepath)
.on('error', () => {
// handle error
})
.pipe(csv())
.on('data', (row) => {
rowsArr.push(row);
})
.on('end', () => {
for (var i = 0; i <= rowsArr.length; i++) {
if (rowsArr[i].emailAddress == emailAdd) {
//do something
}
}
})

undefined response on async action

I'm trying to launch an exe file from an Electron app with React/Redux.
From the component i'm doing dispatch(launch(titleId, titleName))
The problem is i'm getting path undefined when i'm waiting for readFolders() async.
Any idea what i'm doing wrong and what should i change in my aproach?
Thanks in advance!
launch.js
export const launch = async (titleId, titleName) => {
const path = await readFolders(titleId);
console.log('path:', path) //undefined
execFile(path, (err, data) => {
if (err) {
console.log('err', err);
} else if (data) {
console.log('data:', data)
} else {
console.log('success');
}
});
return {
type: 'LAUNCH',
};
};
readFolders.js
import fs from 'fs';
import { homedir } from 'os';
const fsPromises = fs.promises;
const isExeFile = file => file.match(/.*\.exe$/i);
export const readFolders = async titleId => {
const userDir = homedir();
const folderPath = `${userDir}/downloads`;
const fullPath = `${folderPath}/${titleId}`;
try {
const contents = await fsPromises.readdir(fullPath);
contents.forEach(async item => {
if (isExeFile(item)) {
console.log('isExeFile');
return `${fullPath}/${item}`;
}
try {
const nestedFolder = await fsPromises.readdir(`${fullPath}/${item}`);
nestedFolder.forEach(nestedItem => {
if (isExeFile(nestedItem)) {
return `${fullPath}/${item}/${nestedItem}`;
}
return null;
});
} catch (err) {
console.log('err:', err);
}
});
} catch (err) {
console.log('err main:', err);
}
};
Edit:
I also tried this way and now const path = await readFolders(titleId); returns the correct result, but this way eslint is complaining (https://eslint.org/docs/rules/no-async-promise-executor) and it doesn't feel like a good solution.
return new Promise(async (resolve, reject) => {
try {
const contents = await fsPromises.readdir(fullPath);
contents.forEach(async item => {
if (isExeFile(item)) {
console.log(`${fullPath}/${item}`);
return resolve(`${fullPath}/${item}`);
}
try {
const nestedFolder = await fsPromises.readdir(`${fullPath}/${item}`);
nestedFolder.forEach(nestedItem => {
if (isExeFile(nestedItem)) {
console.log(`${fullPath}/${item}/${nestedItem}`);
return resolve(`${fullPath}/${item}/${nestedItem}`);
}
return null;
});
} catch (err) {
console.log('err:', err);
reject(err);
}
});
} catch (err) {
console.log('err main:', err);
reject(err);
}
});

Missing return at the end. When you return in forEach, It returns from callback anonymous function only. return ${fullPath}/${item}/${nestedItem};
For more you can read my blog on it:
https://medium.com/#deepak_v/weird-part-how-to-break-the-loop-in-javascript-8bba3e658267
Updated code:
export const readFolders = async (titleId) => {
const userDir = homedir();
const folderPath = `${userDir}/downloads`;
const fullPath = `${folderPath}/${titleId}`;
try {
const contents = await fsPromises.readdir(fullPath);
let path = "";
contents.some(async (item) => {
if (isExeFile(item)) {
console.log("isExeFile");
path = `${fullPath}/${item}`;
return path;
}
try {
const nestedFolder = await fsPromises.readdir(`${fullPath}/${item}`);
const found = nestedFolder.some((nestedItem) => {
if (isExeFile(nestedItem)) {
path = `${fullPath}/${item}/${nestedItem}`;
return path;
}
return false;
});
if (found) return path;
else return false;
} catch (err) {}
});
return path;
} catch (err) {
console.log("err main:", err);
}
};

Method does not await for fetch to complete

I am using Node Ytdl in order to obtain metadata of videos. I have a 'provider' function which I want to return the metadata object of a video.
const getMetaD = async (mediaID) => {
let obj = {};
await ytdl.getInfo(mediaID, {}, async (err, info) => {
if (err) console.log(err);
let format = await ytdl.chooseFormat(info.formats, { quality: 'highest' });
if (format) {
console.log('Format found!', format);
obj = { url: format.url, thumbnail: info.thumbnail, title: info.title };
}
});
return obj;
}
However, my getMetaD function never returns the object assigned in the if(format) block. It will always return the obj = {}. What am I doing wrong? Why doesn't my code await for the results to be assigned to obj before returning it?

You are using await on a callback. It only works on promises. You can achieve the same thing by doing this though.
const getMetaD = (mediaID) => {
return new Promise((resolve, reject) => {
let obj = {};
ytdl.getInfo(mediaID, {}, async (err, info) => {
if (err) console.log(err);
let format = await ytdl.chooseFormat(info.formats, { quality: 'highest' });
if (format) {
console.log('Format found!', format);
obj = { url: format.url, thumbnail: info.thumbnail, title: info.title };
}
resolve(obj);
});
})
}

ytdl.getInfo will return a Promise if you don't pass in a callback
So, you would rewrite your code as follows (no error checking here, but then you don't do any in your code either)
const getMetaD = async (mediaID) => {
let obj = {};
const info = await ytdl.getInfo(mediaID, {});
const format = await ytdl.chooseFormat(info.formats, { quality: 'highest' });
if (format) {
console.log('Format found!', format);
obj = { url: format.url, thumbnail: info.thumbnail, title: info.title };
}
return obj;
}

How to receive multiple requests with Express.JS

I'm writing an Angular 6 + Express.JS app and now I stuck with the following problem: when there are some multiple requests made at the same time, sometimes (especially when there are more than 4 requests) all of them response with 404 or even get cancelled. Is there any problem with the way I handle requests in Express or I should add some tweaks for concurrent requests?
Requests:
let requests = [];
files.forEach((file) => {
if (file.type.toLowerCase().includes('zip')) {
requests.push(this.imagesService.uploadArchive(file).pipe(first()));
} else {
requests.push(this.imagesService.saveImage(file).pipe(first()));
}
});
forkJoin(requests).subscribe(
(res) => res.forEach(response => {
this.onSave.emit(response);
}),
(error) => {
console.error(error);
},
() => {
this.close.emit();
}
);
Express handling routes:
router.post('/images',
formidable({
encoding: 'utf-8',
uploadDir: path.resolve(__dirname, '..', '..', 'uploads'),
multiples: true,
keepExtensions: true
}),
(req, res, next) => {
const image = req.fields;
const data = req.files;
image.path = data.image.path;
const file = fs.createReadStream(image.path);
saveImage(image).then(
result => {
if (result) {
res.status(200).send(result);
} else {
console.error("Cannot save image");
res.status(400).send("Cannot save image");
}
}).catch(e => console.error(e.stack));
});
Responses:
UPDATE
router.post('/archives',
formidable({
encoding: 'utf-8',
uploadDir: path.resolve(__dirname, '..', '..', 'uploads'),
multiples: true,
keepExtensions: true
}),
(req, res, next) => {
const data = req.files;
let promises = [];
fs.readFile(data.archive.path, async (err, archive) => {
if (err) throw err;
await extractImagesFromZip(archive, data.archive.path).then((images) =>
images.forEach((image) => {
promises.push(
saveImage(image).then(
result => {
if (result) {
result.path = result.path.split('/').pop();
return result;
} else {
console.error("Cannot save image " + image.name);
fs.unlink(image.path, () => {});
}
}).catch(e => {
fs.unlink(image.path, () => {});
console.error(e.stack)
})
);
})
);
Promise.all(promises)
.then((result) => {
if (result.length > 0) {
res.status(200).send(result)
} else {
res.status(400).send("None images were saved")
}
}).catch((error) => {
console.log(error.stack);
res.status(400).send("None images were saved")
});
});
}
);
export const extractImagesFromZip = (file, link) => {
let promises = [];
var zip = new JSZip();
return zip.loadAsync(file)
.then((archive) => {
Object.values(archive.files).filter(
f =>
['.jpg', '.jpeg', '.png'].some((suffix) => f.name.toLowerCase().endsWith(suffix))
&& ![...f.name.toLowerCase().split('/')].pop().startsWith('.')
&& !f.dir
).forEach(f => promises.push(zip.file(f.name).async('nodebuffer').then((content) => {
const ext = f.name.split('.').pop().toLowerCase();
var dest = path.resolve(__dirname, '..', '..') + '/uploads/upload_'
+ crypto.randomBytes(Math.ceil(1322)).toString('hex').slice(0, 32).toLowerCase()
+ '.' + ext;
return new Promise((res, rej) => {
fs.writeFile(dest, content, (err) => {
if (err) rej(err);
res(new Promise((resolve, reject) => {
fs.readFile(dest, (erro, data) => {
if (erro) reject(erro);
if (data) resolve({
name: f.name,
type: 'image/' + (ext === 'jpg' ? 'jpeg' : ext),
path: dest
});
});
}));
});
});
})));
fs.unlink(link, () => {});
return Promise.all(promises);
});
}
export const saveImage = (image) => {
return database.raw(
"INSERT INTO images (name, type, path) " +
"VALUES (?, ?, ?) " +
"RETURNING name, type, path, id",
[image.name, image.type, image.path]
).then(data => data.rows[0]).catch(e => console.error(e.stack));
};
UPDATE 2
Everything works fine if user and server are on localhost (regardless server running with nginx or without it), but problem appears when server is remote

Such code worked
public async uploadFiles(files: File[]) {
of(files)
.pipe(
concatMap(files =>
files.map(file => {
return this.imagesService
.saveImage(file)
.pipe(
map(),
catchError((error, caught) => {
console.error(error);
return empty();
})
);
})
),
concatAll(),
toArray(),
map(res => console.log)
)
.subscribe();
}