I am trying to download txt and mp3 files and use the content of them in another node module.
How can I create a global variable with the piped contents from the downloaded txt (and MP3 at a later stage) file to use outside of the FTP function?
async function example() {
var finalData = '';
const client = new ftp.Client()
client.ftp.verbose = true
try {
await client.access({
host: "XXXX",
user: "XXXX",
password: "XXXX",
})
await client.upload(fs.createReadStream("README.txt"), myFileNameWithExtension)
//let writeStream = fs.createWriteStream('/tmp/' + myFileNameWithExtension);
//await client.download(writeStream, myFileNameWithExtension)
finalData = await (() => {
return new Promise((resolve, reject) => {
writeStream
.on('finish', () => {
// Create a global variable to be used outside of the FTP function scope to pipe the txt content into another node mogule
})
.on('error', (err) => {
console.log(err);
reject(err);
})
})
})();
}
catch (err) {
console.log(err)
}
client.close();
return finalData;
}
No, don't create any global variables. Just resolve the promise with the data:
var finalData = await new Promise((resolve, reject) => {
writeStream.on('finish', () => {
resolve(); // pass a value
}).on('error', (err) => {
reject(err);
});
});
The finalData will become whatever value you pass into resolve(…) - I don't know what result you want to pass there. In the end, just return the data from your example function (as you already do), so that the caller will be able to use it after waiting for the returned promise.
Related
I'm working on a microcontroller that would either take docx files or html strings in input and would transform it into a singular pdf file and return its link as an ouput.
My code looks like this so far:
// 'files' is an array of uploaded docx files.
const uploaded = files.map((file) => {
return new Promise((resolve, reject) => {
pump(
file.toBuffer(),
fs.createWriteStream(join(__dirname, 'files', file.filename))
.on('finish', resolve)
)
})
})
Promise.all(uploaded)
// Is triggered
.then(async () => await convertFiles())
// Is not triggered
.then(async () => {
// concatStoreFiles() is an external function because I need it somewhere else too
test = await concatStoreFiles(join(__dirname, 'files'))
console.log({test})
res.send(test)
})
const convertFiles = () => {
return new Promise((resolve, reject) => {
const cmd = `soffice --headless --convert-to pdf --outdir ${join(__dirname, 'files')} ${join(__dirname, 'files', '*.*')}`
exec(cmd, (error, stdout, stderror) => {
if (error) console.warn(error)
resolve(stdout ?? stderror)
})
})
}
concatStoreFile.js
module.exports = async function concatFiles (dirPath, outPath) {
return new Promise ((resolve, reject) => {
const existingFiles = []
fs.readdir(dirPath, (e, files) => {
files.forEach((file) => {
// is added to the files list only if finishing with ".pdf"
if (/[\d\w_-]+.pdf/.matches(file)) {
existingFiles.push(file)
}
});
resolve(existingFiles)
})
})
}
I'm working with Insomnia for my development / test process, and it tells me that I get an empty response. However, I'm supposed to get an array of pdf files existing in a specific directory. I'm not even getting console.log({test}), so I don't think my second then() is triggered.
I'm really rusty with async / await and Promise syntaxes, what should I do in this situation?
Thank you in advance
The #fastify/multipart's toBuffer() API returns a Promise, not a buffer. Checkout this article
So you need to write something like:
const uploaded = files.map(processFile)
async function processFile (file) {
const buffer = await file.toBuffer()
const storedFileName = join(__dirname, 'files', file.filename)
const writeStream = fs.createWriteStream(storedFileName)
return new Promise((resolve, reject) => {
pump(buffer, writeStream, (err) => {
if(err) { return reject(err) }
resolve(storedFileName)
})
}
}
Moreover, to improve the code, I returned the storedFileName instead of recalculating it.
You can convert this:
.then(async () => await convertFiles())
to this:
.then(() => convertFiles())
Mixing async/await and promise then/catch leads to hidden bugs hard to find
This question already has answers here:
How do I convert an existing callback API to promises?
(24 answers)
Closed 2 years ago.
I have the following code on a server:
let tmpFileName;
// GET
app.get('/clicked', async (req, res) => {
let nullOutput = writeTmpFile("hello, world!");
await deleteTmpFile();
console.log("Hurray, finished!");
res.send({result:nullOutput});
})
function writeTmpFile(content){
tmpFileName = "tmp" + Math.random().toString() + "tsl";
return new Promise(resolve => {
fs.writeFile(tmpFileName, content, function (err) {
if (err) throw err;
console.log('Temp file creation successful.');
});
})
}
function deleteTmpFile(spec){
return new Promise(resolve => {
fs.unlink(tmpFileName, function (err) {
if (err) throw err;
console.log('Temp file deletion successful.');
});
})
}
However, in my console output, I only get
Temp file creation successful.
Temp file deletion successful.
However, if I delete await deleteTempFile(), then Hurray, finished! shows up on the console.
And more generally, how do I debug these patterns of problems?
Why is this happening?
I have rewritten your code, to showcase how to use promises.
Promise callback gets two functions as arguments: resolve and reject.
You should call resolve when operation finishes with success, and reject when it fails.
// I moved `tmpFileName` variable from here into the request handler,
// because it was "global" and would be shared between requests.
app.get('/clicked', async (req, res) => {
let tmpFileName = "tmp" + Math.random().toString() + "tsl"
let writingResult = await writeTmpFile(tmpFileName, "hello, world!")
let deletionResult = await deleteTmpFile(tmpFileName)
res.send({ writingResult, deletionResult })
console.log("Hurray, finished!")
})
function writeTmpFile (filename, content) {
return new Promise((resolve, reject) => {
fs.writeFile(filename, content, function (err) {
// on error reject promise with value of your choice
if (err) reject(err)
// on success resolve promise with value of your choice
resolve('Temp file creation successful.')
})
})
}
function deleteTmpFile (filename) {
return new Promise((resolve, reject) => {
fs.unlink(filename, function (err) {
if (err) reject(err)
resolve('Temp file deletion successful.')
})
})
}
For working with the file you can use writeFileSync instead writeFile. (Reference).
For multiple Promise you can use the Promise.all method.
const promise1 = Promise.resolve(3);
const promise2 = 42;
const promise3 = new Promise((resolve, reject) => {
setTimeout(resolve, 100, 'foo');
});
Promise.all([promise1, promise2, promise3]).then((values) => {
console.log(values);
});
from MDN
I am trying to retrieve multiple files from S3 using a readstream, and insert them into a single file locally.
Below, the 'output' variable is the single writestream I wish to append to using the downloaded S3 file data.
I am looping through days where the nextDay variable is used for the S3 key. The fileservice.s3Handler.getS3Obj returns an S3 object which allows a readstream for a single file and appending to the output file.
However, no other files are being read and are not showing the console either using the on('data', ()) method.
I tried to wrap the readstream in a promise to try to wait until the read was finished but it is running the same error.
More recently I keep get this error: "ERR_STREAM_WRITE_AFTER_END"
Not sure what is going wrong here.
async fetchCSV(req, res) {
const output = fs.createWriteStream(outputPathWithFile, {
'flags': 'a'});
let nextDay = startDate;
while (nextDay !== endDate) {
const s3path = path.join(`${req.params.stationId}`, `${nextDay}.csv`);
const file = await this.fileService.s3Handler.getS3Obj(s3path);
await this.completePipe(file, output);
nextDay = await getTomorrow(nextDay);
}
}
completePipe(file, output) {
return new Promise((resolve) => {
file.createReadStream().on('finish', () => {
resolve();
}).on('error', (err) => {
resolve();
}).on('data', (data) => {
console.log(data.toString());
}).pipe(output);
})
}
}
getS3Obj(file) {
return new Promise(async (resolve) => {
const getParams = {
Bucket: this.bucket,
Key: file
};
resolve(this.s3.getObject(getParams, (err) => {
if (err) {
console.log('Error in getS3 object')
}
}));
})
}
Please help me?
Solved it.
Did a couple things:
Added a tag to the pipe method.
stream.pipe(output, {end: false})
Instead of creating a new function for the promise I just put this code in instead:
await new Promise((resolve) => {
stream.once('finish', () => {
resolve();
});
});
But the tag was what made it work, the promise was just a tidy up.
Yay.
I have 3 layer callbacks like this :
app.post('/', (req, res) => {
var filename = `outputs/${Date.now()}_output.json`;
let trainInput = req.files.trainInput;
let trainOutput = req.files.trainInput;
let testInput = req.files.trainInput;
//first
trainInput.mv(`inputs/${req.body.caseName}/train_input.csv`, function (err) {
if (err) return res.status(500).send(err);
//second
trainOutput.mv(`inputs/${req.body.caseName}/train_output.csv`, function (err) {
if (err) return res.status(500).send(err);
//third
testInput.mv(`inputs/${req.body.caseName}/test_input.csv`, function (err) {
if (err) return res.status(500).send(err);
res.send('success');
});
});
});
});
In this case, there are only 3 file uploads. In another case, I have more than 10 file uploads, and it makes 10 layer callbacks. I know it because of JavaScript asynchronous.
Is there any way, with this case, to make a beautiful code? This is because when it 10 layer callbacks, the code looks horizontally weird.
Thanks
You can use the following code to make you code look better and avoid callback hell
app.post('/', async (req, res) => {
var filename = `outputs/${Date.now()}_output.json`;
let trainInput = req.files.trainInput;
let trainOutput = req.files.trainInput;
let testInput = req.files.trainInput;
try {
var result1 = await trainInput.mv(`inputs/${req.body.caseName}/train_input.csv`);
var result2 = await trainInput.mv(`inputs/${req.body.caseName}/train_output.csv`);
var result2 = await testInput.mv(`inputs/${req.body.caseName}/test_input.csv`);
res.send('success');
}
catch (error) {
res.status(500).send(error);
}
});
You can make the functions return a Promise
I advice to make one function because you do the same thing 3 times. In this case I called the function 'save' but you can call it what ever you want. The first parameter is the file end the second the output filename.
function save(file, output) = return new Promise((resolve, reject) => {
file.mv(`inputs/${req.body.caseName}/${output}`, err =>
if (err) return reject(err)
resolve()
})
Promise.all([
save(req.files.trainInput, 'train_input.csv'),
save(req.files.trainInput, 'train_output.csv'),
save(req.files.trainInput, 'test_input.csv')
])
.then(_ => res.send(200))
.catch(err => res.send(400);
What version of Node you using? If async/await is available that cleans it up a bunch.
const moveCsv = (file, dest) => {
return new Promise((resolve, reject) => {
//third
file.mv(dest, function (err) {
if (err) reject(err);
resolve();
});
})
}
app.post('/', async(req, res) => {
try {
var filename = `outputs/${Date.now()}_output.json`;
const {
trainInput,
trainOutput,
testInput
} = req.files;
const prefix = `inputs/${req.body.caseName}`;
await moveCsv(trainInput, `${prefix}/train_input.csv`);
await moveCsv(trainOutput, `${prefix}/train_output.csv`);
await moveCsv(testInput, `${prefix}/test_input.csv`);
res.send('success');
} catch(err) {
res.status(500).send(err);
}
});
I'm also assuming here that your trainInput, trainOutput, testOutput weren't all meant to be req.files.trainInput.
Just be careful since the synchronous nature of the await calls are thread blocking. If that writer function takes ages you could also looking at putting those calls onto a worker thread. Won't really matter if your requests to that server endpoint are fast and non-frequent.
You can add RXJS to your project and use Observables.forkJoin()
Solution with Observables(assuming that trainInput.mv() returns Observable):
/* Without a selector */
var source = Rx.Observable.forkJoin(
trainInput.mv(`inputs/${req.body.caseName}/train_input.csv`),
trainInput.mv(`inputs/${req.body.caseName}/train_output.csv`),
trainInput.mv(`inputs/${req.body.caseName}/test_input.csv`)
);
var subscription = source.subscribe(
function (x) {
// On success callback
console.log('Success: %s', x);
},
function (err) {
// Error callback
console.log('Error');
},
function () {
// Completed - runs always
console.log('Completed');
});
// => Success: [result_1, result_2, result_3] or Error
// => Completed
I am using a single Node module basic-ftp to download a txt file in AWS Lambda and place it in the /tmp/ directory within the Lambda function.
I then want to work with the txt file and its contents outside of the FTP function.
I am using Async and Promises and have got a bit lost with the code. The current error returned in AWS Lambda is
module initialization error: ReferenceError
await finalData = (() => {
^^^^^^^^^^^^^^^
resulting from this line await finalData = (() => {
Can anyone help fix this and help me access the finalData outside of the FTP function?
var fs = require('fs');
var ftp = require("basic-ftp");
var path = require('path');
exports.handler = async (event, context, callback) => {
var fullPath = event.line_items[0].meta_data[2].value.tmp_name; // File path on Linux server -------
var myFileNameWithExtension = path.basename(fullPath); // Uploaded filename with the file extension eg. filename.txt
// FTP Function - Download from FTP and write to /tmp/ within AWS Lambda function
example()
//example().then(finalData=> callback(finalData))
async function example() {
var finalData = '';
const client = new ftp.Client()
client.ftp.verbose = true
try {
await client.access({
host: "XXXX",
user: "XXXX",
password: "XXXX",
})
let writeStream = fs.createWriteStream('/tmp/' + myFileNameWithExtension);
await client.download(writeStream, myFileNameWithExtension)
await finalData = (() => {
return new Promise((resolve, reject) => {
writeStream
.on('finish', () => {
fs.readFile("/tmp/" + myFileNameWithExtension, function (err, data) {
if (err) {
reject(err)
} else {
console.log('Contents of AWS Lambda /tmp/ directory', data);
resolve(data);
}
});
})
.on('error', (err) => {
console.log(err);
reject(err);
})
})
})();
}
catch (err) {
console.log(err)
}
client.close();
return finalData;
}
// Output contents of downloaded txt file into console and use in later code outside of the FTP function
console.log("Raw text:\n" + finalData.Body.toString('ascii'));
};
finalData is only defined within example which returns it, but you're not assigning this to anything. Combined with Luca Kiebel's comment, try adding
const finalData = await example();
then log that out.
Because finalData is defined within the function example, it is only available within that function and any function defined within that function.
You Don't Know JS explains this better than I can