In the 1st function 'get_files()' I can log the file_list variable, it is correct here, however when I log it again in my 2nd function 'get_diffs()' it is undefined..
// Get files
async function get_files() {
await fs.readdir(dirPath, function (err, files) {
(async () => {
if (await err) {
console.log("Error getting directory information.", err)
} else {
var file_list = []; // Reset
await files.forEach(function (file) {
file_list.push(file);
});
console.log('1st Call = ' + file_list); // Correct
return await file_list;
}
})();
});
}
// Get Diffs
async function get_diffs() {
const file_list = await get_files();
console.log('2nd Call = ' + file_list); // Undefined
const dates = await get_dates();
return await files.filter(x => !dates.includes(x));
}
You have misunderstood async/await. Learn the basics here
function get_files() {
return new Promise((resolve, reject) => {
fs.readdir(dirPath, function (err, files) {
if (err) {
reject(err);
} else {
var file_list = []; // Reset
files.forEach(function (file) {
file_list.push(file);
});
console.log('1st Call = ' + file_list); // Correct
resolve(file_list);
}
});
})
}
fs.readdir does not return a promise. Use the promise based function fs.promise.readdir instead.
async function get_diffs() {
const file_list = await fs.promise.readdir(dirPath);
// ...
}
So you don't really need the other function. It had many problems anyway. await makes not much sense when used with an expression that is not a promise. All the places where you have used await in get_files, the expression that follows it does not represent a promise.
Related
I am trying to call some function using a single express router , I want to call them in order, meaning that I don't want getLaps() function to execute before get streams function has done all the work , so I tried to use some solutions I found on the internet but it didn't work, the second function doesn't execute. Please help.
Here is my code :
router.get("/", async (req, res,done) => {
res.status(201).send('created user')
return getLaps(function () {
getStreams(function () {
});
});
// await getStreams();
// await getLaps();
// console.log("hey")
});
Here is the get laps function :
function getLaps(req) {
const access_token = '75f2d92fdc445033312854d775e039b6c5bf04e7';
//for test 3756582581,
const idL = [5567017025, 5566531480];
const stravaClient = StravaClientService.getClient(access_token);
const activityService = StravaActivityService(stravaClient);
var params = {
TableName: "run-id",
Key: {
"id": "15428785",
}
};
console.log("cool laps")
docClient.get(params, async function (err, data) {
if (err) {
console.log("Error", err);
} else {
}
idL.map((id, index) => setTimeout(() => activityService.listLaps(id), (5 + index) * 60)
)
//data.Item.json
});
}
and the streams function :
function getStreams(req) {
const idS = [
5567017025, 5566531480
];
const stravaClient = StravaClientService.getClient(access_token);
const activityService = StravaActivityService(stravaClient);
var params = {
TableName: "run-id",
Key: {
"id": "15428785",
}
};
console.log("cool streams")
docClient.get(params, async function (err, data) {
if (err) {
console.log("Error", err);
} else {
idS.map((id, index) => setTimeout(() => activityService.streamActivity(id), (5 + index) * 60))
console.log("got the streams")
}
});
}
in your getStream and getLaps function return promises instead of other object/Stuff like
async function getStream(){
return new Promise(async (resolve, reject){
//Do something
//where you want to return something just call resolve function like
resolve()
//if you want some output of getStream() just pass it to resolve function
//const result = 'I'm result'
resolve(result)
})
}
do same thing with the laps function and in your router call them with await keyword
I am building a web scraper to get all of user's submissions on codeforces.
I don't know much about async, await, promises.
I have used axios (promise based) to request codeforces and cheerio to parse HTML .
app.post("/", (req, res) => {
const usernameorhandle = req.body.userName;
getstatus(usernameorhandle).then ( ()=> {
var output = fs.createWriteStream(__dirname + '/Data/solutions.zip');
var archive = archiver('zip', {
zlib: { level: 9 } // Sets the compression level.
});
output.on('close', function() {
console.log(archive.pointer() + ' total bytes');
console.log('archiver has been finalized and the output file descriptor has closed.');
});
output.on('end', function() {
console.log('Data has been drained');
});
res.attachment(__dirname + "/Data/Problems", 'Codeforces-Solutions');
archive.pipe(res);
archive.directory(__dirname + "/Data/Problems", 'Codeforces-Solutions');
archive.finalize();
}) })
I am using to accept post request.
I am putting all the solutions into a folder and creating zip folder and then send to res.
Below is my getstatus function.
async function getstatus(handle){
return new Promise(async (resolve, reject)=> {
console.log("HELLLLLLLOOOOOOOO");
await axios.get("https://codeforces.com/api/user.status?handle=" + handle + "&from=1")
.then(response => {
if(response.data.status === 'OK'){
let results = response.data.result;
console.log("AAAAAAAAAAAAAAAAAAAAAAAa");
scrape(results).then( () =>{
console.log("DONE");
resolve();
})
.catch(err => console.log(err));
// resolve();
}
else console.log(submissions.comment);
})
})
}
I use scrape function to obtain HTML data and put to folder named Problems.
async function scrape (results){
console.log("inside scrape");
// console.log("HELLO");
return new Promise( async (resolve, reject) => {
await results.forEach(async (result)=> {
if(result.verdict === 'OK'){
await axios.get("https://codeforces.com/contest/" + result.contestId + "/submission/" + result.id)
.then(solutionPage => {
const $ = cheerio.load(solutionPage.data);
const path = "/home/srujan/Desktop/crawlerapp/Data/Problems/" + result.problem.name + ".cpp";
fs.writeFile(path, $('#program-source-text').text(), function(err){
if(err){
console.log(err);
}
else{
console.log("Saved file");
}
})
})
.catch( error => {
console.log("HTML PARSE ERROR" + error);
})
}
})
console.log("hey");
resolve();
})
The problem is I am getting
HELLLLLLLOOOOOOOO
AAAAAAAAAAAAAAAAAAAAAAAa
inside scrape
hey
DONE
saved file
saved file
...
Browser downloads after DONE and then files are saved.
I am new to js and don't know why I am getting this.
PS : I know this is very long question. I tried reading a lot about this. Didn't understand properly how to do that. I copy pasted some code which I didn't understand like how to zip a folder.
forEach(callback) executes callback. If callback returns a promise (ie, it's an async function), the promise won't be resolved before calling the callback on the next element of the array.
So, basically, you can't use async functions inside forEach... But you can use for-loops or Promise.all instead!
Also, fs.writeFile works with sync + callback, but there exists a fs.promise.writeFile that uses promises instead.
Here's a scrape function that should work better:
async function scrape(results) {
for (const result of results) {
if(result.verdict === 'OK') {
const solutionPage = await axios.get("https://codeforces.com/contest/" + result.contestId + "/submission/" + result.id);
const $ = cheerio.load(solutionPage.data);
const path = "/home/srujan/Desktop/crawlerapp/Data/Problems/" + result.problem.name + ".cpp";
try {
await fs.promises.writeFile(path, $('#program-source-text').text());
} catch(err) { console.log(err) }
}
}
}
The problem is to use result.forEach
Try to use a simple for(let i = 0; i < result.length; i++) without async.
If that doesn't work, try to return anything inside the then.
This is how I would construct getstatus function with await async
async function getstatus(handle) {
const response = await axios.get("https://codeforces.com/api/user.status?handle=" + handle + "&from=1")
if(response.data.status === 'OK') {
let results = response.data.result;
try {
await scrape(results);
console.log("DONE");
}
catch(error) {
}
}
}
and scrape function accordingly...
const fs = require('fs').promises;
async function scrape (results) {
results.forEach(async (result)=> {
if(result.verdict === 'OK') {
const solutionPage = await axios.get("https://codeforces.com/contest/" + result.contestId + "/submission/" + result.id)
const $ = cheerio.load(solutionPage.data);
const path = "/home/srujan/Desktop/crawlerapp/Data/Problems/" + result.problem.name + ".cpp";
try {
await fs.writeFile(path, $('#program-source-text').text())
console.log("Saved file");
}
catch(error) {
}
}
}
}
async function count() {
let nedb = <<functional nedb database>>
let count = 0
await iterate(nedb, {}, g=>{
count++
})
console.log(count)
iterate(xx, query, callback) {
return new Promise((res, rej)=> {
pkgs.find(query).exec((err, res)=>{
if(err!==null) {
this.err(err)
} else {
res.forEach((pkg)=>{
callback(pkg)
})
}
})
res()
})
I would like to write something, after the iteration is over, is there way to do it in some normal way?
You need to put the res()olve call inside the find callback. Immediately fulfilling the promise to undefined won't help.
But you really shouldn't pass an iteration callback at all. Instead, write
async function count() {
let nedb = …;
let count = 0;
const pkgs = await query(nedb, {});
pkgs.forEach(pkg => count++);
// or maybe better just:
const count = pkgs.length;
console.log(count);
}
function query(pkgs, query) {
return new Promise((resolve, reject)=> {
pkgs.find(query).exec((error, result)=>{
if (error !== null) reject(error);
else resolve(result);
});
});
}
I have 3 layer callbacks like this :
app.post('/', (req, res) => {
var filename = `outputs/${Date.now()}_output.json`;
let trainInput = req.files.trainInput;
let trainOutput = req.files.trainInput;
let testInput = req.files.trainInput;
//first
trainInput.mv(`inputs/${req.body.caseName}/train_input.csv`, function (err) {
if (err) return res.status(500).send(err);
//second
trainOutput.mv(`inputs/${req.body.caseName}/train_output.csv`, function (err) {
if (err) return res.status(500).send(err);
//third
testInput.mv(`inputs/${req.body.caseName}/test_input.csv`, function (err) {
if (err) return res.status(500).send(err);
res.send('success');
});
});
});
});
In this case, there are only 3 file uploads. In another case, I have more than 10 file uploads, and it makes 10 layer callbacks. I know it because of JavaScript asynchronous.
Is there any way, with this case, to make a beautiful code? This is because when it 10 layer callbacks, the code looks horizontally weird.
Thanks
You can use the following code to make you code look better and avoid callback hell
app.post('/', async (req, res) => {
var filename = `outputs/${Date.now()}_output.json`;
let trainInput = req.files.trainInput;
let trainOutput = req.files.trainInput;
let testInput = req.files.trainInput;
try {
var result1 = await trainInput.mv(`inputs/${req.body.caseName}/train_input.csv`);
var result2 = await trainInput.mv(`inputs/${req.body.caseName}/train_output.csv`);
var result2 = await testInput.mv(`inputs/${req.body.caseName}/test_input.csv`);
res.send('success');
}
catch (error) {
res.status(500).send(error);
}
});
You can make the functions return a Promise
I advice to make one function because you do the same thing 3 times. In this case I called the function 'save' but you can call it what ever you want. The first parameter is the file end the second the output filename.
function save(file, output) = return new Promise((resolve, reject) => {
file.mv(`inputs/${req.body.caseName}/${output}`, err =>
if (err) return reject(err)
resolve()
})
Promise.all([
save(req.files.trainInput, 'train_input.csv'),
save(req.files.trainInput, 'train_output.csv'),
save(req.files.trainInput, 'test_input.csv')
])
.then(_ => res.send(200))
.catch(err => res.send(400);
What version of Node you using? If async/await is available that cleans it up a bunch.
const moveCsv = (file, dest) => {
return new Promise((resolve, reject) => {
//third
file.mv(dest, function (err) {
if (err) reject(err);
resolve();
});
})
}
app.post('/', async(req, res) => {
try {
var filename = `outputs/${Date.now()}_output.json`;
const {
trainInput,
trainOutput,
testInput
} = req.files;
const prefix = `inputs/${req.body.caseName}`;
await moveCsv(trainInput, `${prefix}/train_input.csv`);
await moveCsv(trainOutput, `${prefix}/train_output.csv`);
await moveCsv(testInput, `${prefix}/test_input.csv`);
res.send('success');
} catch(err) {
res.status(500).send(err);
}
});
I'm also assuming here that your trainInput, trainOutput, testOutput weren't all meant to be req.files.trainInput.
Just be careful since the synchronous nature of the await calls are thread blocking. If that writer function takes ages you could also looking at putting those calls onto a worker thread. Won't really matter if your requests to that server endpoint are fast and non-frequent.
You can add RXJS to your project and use Observables.forkJoin()
Solution with Observables(assuming that trainInput.mv() returns Observable):
/* Without a selector */
var source = Rx.Observable.forkJoin(
trainInput.mv(`inputs/${req.body.caseName}/train_input.csv`),
trainInput.mv(`inputs/${req.body.caseName}/train_output.csv`),
trainInput.mv(`inputs/${req.body.caseName}/test_input.csv`)
);
var subscription = source.subscribe(
function (x) {
// On success callback
console.log('Success: %s', x);
},
function (err) {
// Error callback
console.log('Error');
},
function () {
// Completed - runs always
console.log('Completed');
});
// => Success: [result_1, result_2, result_3] or Error
// => Completed
I'm using through2 to generate multiple files from a Gulp stream. I'm using NodeJS 10.6.0 so thought I'd make full use of async/await, but am not fully understanding the mechanics yet. Currently the through2 done() callback is being fired before all files have been written.
Here's what I have (simplified) - note that I'm not returning the stream at the end as there is no need to.
async function createDirectory(pathDir) {
return new Promise((resolve, reject) => {
mkdirp(pathDir, (err) => {
if (err) reject(err);
else resolve();
});
});
}
async function writeFile(outputFilePath, outputFileContent) {
return new Promise((resolve, reject) => {
fs.writeFile(outputFilePath, outputFileContent, (err) => {
if (err) reject(err);
else resolve();
});
});
}
async function doWriteFile(outputFolderPath, outputFilePath, outputContent) {
await createDirectory(outputFolderPath);
await writeFile(outputFilePath, outputContent, outputContent);
}
async function doGenerateVariant(data, variantArr) {
for (const variant of variantArr) {
/* Do a load of stuff */
const variantOutputFolderPath = blah;
const variantOutputFilePath = blah;
const variantOutputContent = blah;
await doWriteFile(variantOutputFolderPath, variantOutputFilePath, variantOutputContent);
}
}
const generateVariant = () => {
return through.obj((file, enc, done) => {
const data = JSON.parse(file.contents.toString());
*/ Do a load of stuff */
const { variant } = data;
const variantArr = Object.values(variant);
doGenerateVariant(data, variantArr);
return done();
});
};
This doesn't work as done() gets returned before all files have been written. I'm guessing I'm missing a return or two but nothing I do seems to be working.
If I pass done() into doGenerateVariant and call it after doWriteFile everything works as expected but I know this isn't correct.
You need to wait for doGenerateVariant to do its job before calling done. Remember async function always returns a Promise. So you could do it this way
const generateVariant = () => {
return through.obj((file, enc, done) => {
const data = JSON.parse(file.contents.toString());
*/ Do a load of stuff */
const { variant } = data;
const variantArr = Object.values(variant);
doGenerateVariant(data, variantArr).then(() => done());
});
};
or using async/await
const generateVariant = () => {
return through.obj(async (file, enc, done) => {
const data = JSON.parse(file.contents.toString());
*/ Do a load of stuff */
const { variant } = data;
const variantArr = Object.values(variant);
await doGenerateVariant(data, variantArr);
done();
});
};