I have a code that fetches CSV files from an SFTP and parses them, The following function doesn't reject when there is an error (permission denied) when opening the file
const getCSV = (fileName) => {
const results = []
return new Promise((resolve, reject) => {
if (!fileName) {
resolve(results)
}
sftp
.createReadStream(`${directoryToFetch}/${fileName}`)
.on('error', (e) => {
console.error(`Failed parsing CSV ${e}, ${directoryToFetch}/${fileName}`)
return reject(new Error(`Failed parsing CSV ${e}: ${directoryToFetch}/${fileName}`))
})
.pipe(csv({ skipLines: 1, separator: '\t' }))
.on('data', (data) => results.push(data))
.on('end', () => {
return resolve(results)
})
.on('error', () => {
return reject('Failed parsing CSV')
})
})
}
the function does get to the .on('error') event and it executes the reject but in the for loop that awaits on the results from the function I don't get the .catch triggered by the rejection of the promise
const filesList = await getRelevantFileList()
const processedFiles = []
for (const file of filesList) {
try {
const { name } = file
let dataFromFile = await getCSV(name)
const dataToInsert = dataFromFile.filter((entry) => entry.SharesOutstanding > 0)
dataFromFile = []
processedFiles.push(file)
} catch (error) {
console.error(`${error} Unable to fetch ${file}`)
}
}
One issue I can see here is that
if (!fileName) {
resolve(results)
}
Is missing a return, so it should be
if (!fileName) {
return resolve(results)
}
This means that if the fileName is missing you will both an error and a resolve
Related
I am trying to parse data from a .csv file, and save it to an array for later use.
I understand the concept of promises, but I have no idea what am I missing in my code that I cannot resolve the Promise and get the value (the string in the .csv file). It while I can view all the data inside the promise (.on('data')) from debugging mode, I just can't save it in order to use it later in my 'try&catch'.
const fs = require("fs");
const csv = require("csv-parser");
const { resolve } = require("path");
async function readCSV(filepath) {
return new Promise(async (resolve, reject) => {
await fs
.createReadStream(filepath)
.pipe(csv())
.on("data", (data) => {
results.push(data);
})
.on("error", (error) => reject(results))
.on("end", () => {
resolve(results);
});
});
}
const results = [];
const csvFilePath =
"/languages.csv";
try {
const languages = readCSV(csvFilePath).then((res) => {
return res;
});
console.log(languages);
} catch (e) {
console.log(e);
}
and the output on the console is:
>Promise {<pending>}
No debugger available, can not send 'variables'
** That's from the debugging mode when I pause inside the promise:
https://i.stack.imgur.com/H9nHi.png
You can't try catch a returned promise without the await keyword in an async function.
If you're returning a promise, you need to use the .catch method on the promise.
Also, when you're logging languages you're doing so before the promise resolves because you're not using the await keyword.
I'm sure the promise resolves. Instead, log res inside the .then method.
const fs = require("fs");
const csv = require("csv-parser");
const results = [];
function readCSV(filepath) {
return new Promise((resolve, reject) => {
fs
.createReadStream(filepath)
.pipe(csv())
.on("data", (data) => {
results.push(data);
})
.on("error", (error) => reject(results))
.on("end", () => {
resolve(results);
});
});
}
const csvFilePath = "./languages.csv";
(async () => {
const output = await readCSV(csvFilePath);
console.log(output)
})();
I am new to node and stuck with this issue. Here' the file:
I am running 'startProcess' function and I want to run 'downloadFiles' and wait until it's completed and save the files before executing any code after it.
This code always ends up running 'runVideoUploadEngine' even before the download has been completed?
const downloadAndSaveFiles = async ({ url, dir }) => {
try {
https.get(url, (res) => {
// File will be stored at this path
console.log('dir: ', dir);
var filePath = fs.createWriteStream(dir);
res.pipe(filePath);
filePath.on('finish', () => {
filePath.close();
console.log('Download Completed');
});
});
return true;
} catch (e) {
console.log(e);
throw e;
}
};
const downloadFiles = async ({ data }) => {
try {
mediaUrl = data.mediaUrl;
thumbnailUrl = data.thumbnailUrl;
const mediaExt = path.extname(mediaUrl);
const thumbExt = path.extname(thumbnailUrl);
mediaDir = `${__dirname}/temp/${'media'}${mediaExt}`;
thumbDir = `${__dirname}/temp/${'thumb'}${thumbExt}`;
await downloadAndSaveFiles({ url: mediaUrl, dir: mediaDir });
await downloadAndSaveFiles({ url: thumbnailUrl, dir: thumbDir });
return { mediaDir, thumbDir };
} catch (e) {
console.log(e);
throw e;
}
};
module.exports = {
startProcess: async ({ message }) => {
//check if message is proper
data = JSON.parse(message.Body);
//download video and thumbnail and store in temp.
console.log('starting download..');
const { mediaDir, thumbDir } = await downloadFiles({ data });
console.log('dir:- ', mediaDir, thumbDir);
pageAccessToken =
'myRandomToken';
_pageId = 'myRandomPageID';
console.log('running engine');
await runVideoUploadEngine({ pageAccessToken, _pageId, mediaDir, thumbDir });
//start videoUploadEngine
//on success: delete video/thumbnail
},
};
What am I doing wrong?
downloadAndSaveFiles returns a promise (because the function is async) but that promise doesn't "wait" for https.get or fs.createWriteStream to finish, and therefore none of the code that calls downloadAndSaveFiles can properly "wait".
If you interact with callback APIs you cannot really use async/await. You have to create the promise manually. For example:
const downloadAndSaveFiles = ({ url, dir }) => {
return new Promise((resolve, reject) => {
// TODO: Error handling
https.get(url, (res) => {
// File will be stored at this path
console.log('dir: ', dir);
var filePath = fs.createWriteStream(dir);
filePath.on('finish', () => {
filePath.close();
console.log('Download Completed');
resolve(); // resolve promise once everything is done
});
res.pipe(filePath);
});
});
};
This is a promisified spawn function:
async function aspawn(cmd, args){
return new Promise((resolve, reject) => {
const proc = spawn(cmd, args);
proc.stderr.on('data', data => {
console.error('err', data.toString());
});
proc.stdout.on('data', data => {
console.error('stdout', data.toString());
});
proc.on('close', code => {
console.error('closed with code', code);
resolve();
});
});
}
I was wondering if it's possible to make it less indented
Using async iterator and once event emitter feature you could write them like this:
const { spawn } = require('child_process')
const { once } = require('events')
aspawn1('cat', ['README.md'])
.then(() => aspawn1('cat', ['FOO.md'])) // error stream
.then(() => aspawn2('cat', ['README.md']))
async function aspawn1 (cmd, args) {
try {
const proc = spawn(cmd, args)
// in any case you can add events to `proc`
// consume the stream
for await (const chunk of proc.stdout) {
console.log('>>> ' + chunk.length)
}
for await (const chunk of proc.stderr) {
console.log('err >>> ' + chunk.length)
}
// the stream is ended and the spawn aswell
} catch (err) {
// if you need to retrun always a positive promise
console.log('error happened', err)
}
}
// Since node: v11.13.0, v10.16.0 you may write that function like this to have a strict "fire and forget" spawn:
function aspawn2 (cmd, args) {
return once(spawn(cmd, args), 'close')
}
I've got a series of promises.
I never get the console log "Processed folders" printed out. Execution seems to stop once it hits the first await Promise.all call.
Not entirely sure where I've missed up?
const subfolders = [];
const exportFolder = () => {
// Other stuff happening here
const subfolder = {};
subfolder.items = [];
subfolder.items.push({ name: 'item 2.1' });
const folder = {};
folder.items = [];
folder.items.push({ name: 'item 1' });
folder.items.push({ name: 'item 2', isFolder: true, items: subfolder.items });
console.log('Folder:', folder);
console.log('Started');
exportFolderToCsv(folder).then(response => console.log('Finished', response));
};
const exportFolderToCsv = async folder => {
console.log('Processing folders');
let promises = [];
for (const folderItem of folder.items) {
if (folderItem.isFolder && folderItem.items.length > 0) {
subfolders.push(folderItem);
return;
}
promises.push(processFolderItem(folderItem));
}
await Promise.all(promises).then(response => console.log('Processed folders:', response));
if (subfolders.length > 0) {
console.log('Processing subfolders');
promises = [];
for (const folderItem of subfolders.items) {
promises.push(processFolderItem(folderItem));
}
await Promise.all(promises).then(response => console.log('Processed subfolders:', response));
}
console.log('Finished');
};
const processFolderItem = folderItem => new Promise(resolve => {
console.log('Processing folder item');
// To stuff here with folderItem, get Doc Chars, process row and resolve
getCharacters(folderItem)
.then(response => {
console.log('Processed folder item characters list:', response);
createCSVRow(folderItem, response)
.then(response => {
console.log('Processed CSV row:', response);
resolve(response);
})
});
});
const getCharacters = folderItem => new Promise(resolve => {
console.log('Processing folder item characters list');
// To stuff here with folderItem and then resolve
const characters = 'Foobar characters';
resolve(characters);
});
const createCSVRow = (folderItem, characters) => new Promise(resolve => {
console.log('Processing CSV row');
// To stuff here with folderItem and characters and then resolve
const csvRow = 'Foobar row';
resolve(csvRow);
});
exportFolder();
The function is returned before any Promise call is executed, it should be continue. So please resolve it first.
const exportFolderToCsv = async (folder) => {
// ...
for (const folderItem of folder.items) {
if (folderItem.isFolder && folderItem.items.length > 0) {
subfolders.push(folderItem)
return // here lies the problem, it should be `continue` instead
}
promises.push(processFolderItem(folderItem))
}
// ...
}
The Promise.all method, will catch errors if at least one Promise in chain throws an error or reject it.
It seems that one of your promises got an exception.
You can debug it simply adding the catch like so:
Promise.all(PromiseList).then(()=>console.log("all works done") ).catch(errors=>console.log("something wrong",errors))
If you are using await there is no point in chaining.
let response = await Promise.all(promises);
console.log('Processed subfolders', response)
Your promises will encounter errors, and you need to consider the failed situations.
At the same time, Promise.all will stop if either one promise failed, so maybe your Promise.all failed.
try {
Promise.all(promises).then(response => console.log('Processed subfolders', response));
} catch (err) {
console.log(err);
}
// or to check your promise result
const folderResult = await Promise.all(promises);
to check the exact result
I created multiples functions under a directory called data and fill them with some random data returned by a function called generatedRandomData.
To create multiple files I wrote these functions:
const createFile = (fileName, data) => {
if (fs.existsSync(fileName)) throw new Error('Filename already exists');
fs.writeFile(fileName, data, {
encoding: 'utf8',
flag: 'w',
}, (error) => {
if (error) return error;
console.log('File created successfully');
return null;
});
};
const createFiles = (dirPath, sizeList) => {
if (sizeList && !sizeList.length) throw new Error('The list of size should not be empty');
const fileCreationPromises = sizeList.map(async (size) => {
const data = generateRandomData(size);
const fileName = resolve(dirPath, `./data_${size}.txt`);
await createFile(fileName, data);
});
return Promise.all(fileCreationPromises);
};
Then I call the function generateData in order to generate random data and call the functions described above then create the files:
const generateData = async (dirPath, sizeList) => {
if (!dirPath) throw new Error('No directory path was provied');
if (!sizeList || (sizeList && !sizeList.length)) throw new Error('Size list should not be empty');
await createFiles(dirPath, sizeList);
};
I call another function called execute which reads data from those file in order to continue the treatment:
const execute = async (func, dirPath, label) => {
const files = fs.readdirSync(dirPath);
const result = [];
if (files && files.length) {
for (const file of files) {
const filename = resolve(dirPath, `./${file}`);
const parsedData = readDataFromFile(filename);
const data = parsedData.split(',').map((d) => Number(d));
const { length } = data;
result.push({
label: length,
value: getExecutionTime(func, data),
});
}
}
await createFile(resolve(dirPath, `./${label}`), result);
};
Finally, I call the function initialize:
const { resolve } = require('path');
const fs = require('fs');
const { generateData, sizeList, execute } = require('../utils/helpers');
const { underscorePartial } = require('../main/partial');
const dirPath = resolve(__dirname, '../data');
const initialize = () => {
if (!fs.existsSync(dirPath)) {
fs.mkdir(dirPath, async (error) => {
if (error) throw error;
await generateData(dirPath, sizeList);
await execute(underscorePartial, dirPath, 'uExecutionTime.txt');
});
}
};
try {
initialize();
} catch (error) {
console.log(error);
}
However I realized that uExecutionTime.txt to be created in the final step contains undefined due to the function readDataFromFile which returns undefined.
I guess the readDataFromFile starts reading from files before the creation of data finished.Any suggestions to fix my code or are there anything missed or wrong in the code?
The problem is your createFile function. You care awaiting it while it doesn't return promise. It is a callback style. It should be wrapped in promise.
const createFile = (fileName, data) => {
if (fs.existsSync(fileName)) throw new Error('Filename already exists');
return new Promise((resolve, reject) => {
fs.writeFile(fileName, data, {
encoding: 'utf8',
flag: 'w',
}, (error) => {
if (error) reject(error);
console.log('File created successfully');
resolve(null);
});
});
};
Hope this resolves the issue.