I am using the google translate api to translate data from a json file to french locale and then write it back to a file. I am using a recursive function to iterate over the json file since it is deeply nested. However the execution is not waiting till the translation is completed before it writes to the file. I have tried using callback and promise approaches but i couldn't get it right.
Just for it to work as I required an output as an emergency I have set a timout before the write method is called. It work but I would like to learn the appropriate/correct approach to implement this.
const fs = require('fs')
const {Translate} = require('#google-cloud/translate').v2
require('dotenv').config()
process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0
const credentials = JSON.parse(process.env.credentials)
const translate = new Translate({
credentials,
projectId: credentials.project_id,
})
let data = {}
// writeJSONTofile should be executed only after readJSONFile execution is completed
//read file
const readJSONFile = () => {
try {
data = JSON.parse(fs.readFileSync('...\\locale\\en.json'))
iterateAndTranslate(data)
setTimeout(() => {
writeJSONToFile()
}, 25000)
} catch (error) {
console.log(error)
}
}
// iterate, translate, reassign
const iterateAndTranslate = async (data) => {
for(key in data) {
if (typeof data[key] === 'object' && data[key] !== null) {
iterateAndTranslate(data[key])
} else{
data[key] = await translateText(data[key], 'fr')
}
}
}
//translate method
const translateText = async (text, targetLanguage) => {
try {
let [response] = await translate.translate(text, targetLanguage)
return response
} catch (error) {
console.log(error)
return 0
}
}
const writeJSONToFile = async () => {
var outputFileName = 'C:\\test\\test.json'
await fs.writeFileSync(outputFileName, JSON.stringify(data,null,4), (err) => {
if(err) {
console.log(err)
} else {
console.log('Done!')
}
})
}
// start from here
readJSONFile()
You have a few issues with your code.
Your functions use a global variable and mutate it instead of getting input and returning output.
timeout will cause unexpected behavior in your case.
you are using var
you have redundant async-await on the writeJSONToFile function
See my view of point about the possible solution.
const fs = require("fs");
const { Translate } = require("#google-cloud/translate").v2;
require("dotenv").config();
process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = 0;
const credentials = JSON.parse(process.env.credentials);
const translate = new Translate({
credentials,
projectId: credentials.project_id,
});
// writeJSONTofile should be executed only after readJSONFile execution is completed
//read file
const readJSONFile = async () => {
try {
const data = JSON.parse(fs.readFileSync("...\\locale\\en.json"));
return iterateAndTranslate(data);
} catch (error) {
console.log(error);
}
return {};
};
// iterate, translate, reassign
const iterateAndTranslate = async (data) => {
for (let key in data) {
if (typeof data[key] === "object" && data[key] !== null) {
await iterateAndTranslate(data[key]);
} else {
data[key] = await translateText(data[key], "fr");
}
}
return data;
};
//translate method
const translateText = async (text, targetLanguage) => {
try {
let [response] = await translate.translate(text, targetLanguage);
return response;
} catch (error) {
console.log(error);
}
return null;
};
const writeJSONToFile = (data) => {
let outputFileName = "C:\\test\\test.json";
fs.writeFileSync(outputFileName, JSON.stringify(data, null, 4), (err) => {
if (err) {
console.log(err);
} else {
console.log("Done!");
}
});
};
// start from here
const run = async () => {
const data = await readJSONFile();
writeJSONToFile(data);
};
run();
See more:
why not using global varible
why not using var
Related
I'm studying the node.js module async,I want to find out if there is a way to change the async.retry method to retry even on successfull operations but stop based on some condition or response let's say its an api call.
According to its docs ,the function will continue trying the task on failures until it succeeds.if it succeeds it will only run only that time But how can i make it work the same on successfull operations and make it stop on some condition ?
const async = require('async');
const axios = require('axios');
const api = async () => {
const uri = 'https://jsonplaceholder.typicode.com/todos/1';
try {
const results = await axios.get(uri);
return results.data;
} catch (error) {
throw error;
}
};
const retryPolicy = async (apiMethod) => {
async.retry({ times: 3, interval: 200 }, apiMethod, function (err, result) {
// should retry untill the condition is met
if (result.data.userId == 5) {
// stop retring
}
});
};
retryPolicy(api);
Yes, You can just throw a custom error if condition is not met. Would be something like that:
const async = require('async');
const axios = require('axios');
const api = async () => {
const uri = 'https://jsonplaceholder.typicode.com/todos/1';
try {
const results = await axios.get(uri);
if(typeof result.data.userId != 'undefined' && result.data.userId == 5){ // change this condition to fit your needs
return results.data;
}else{
throw {name : "BadDataError", message : "I don't like the data I got"};
}
} catch (error) {
throw error;
}
};
I don't think this is possible.
On the async.retry documentation you can find this description:
Attempts to get a successful response from task no more than times
times before returning an error. If the task is successful, the
callback will be passed the result of the successful task. If all
attempts fail, the callback will be passed the error and result (if
any) of the final attempt.
However, using the delay function given here, you can do what you want another way:
const async = require('async');
const axios = require('axios');
const delay = (t, val) => {
return new Promise((resolve) => {
setTimeout(() => { resolve(val) }, t);
});
}
const api = async () => {
const uri = 'https://jsonplaceholder.typicode.com/todos/1';
try {
const results = await axios.get(uri);
return results.data;
} catch (error) {
throw error;
}
};
const retryPolicy = async (apiMethod) => {
const times = 3
const interval = 200
let data
for (count = 0; count < 3; count++) {
try {
data = await apiMethod()
catch(e) {
console.log(e)
await delay(interval)
continue
}
if (data.userId === 5) {
break;
}
await delay(interval)
}
// do something
};
retryPolicy(api);
I'm trying to remove the then statements from the following piece of code and then replace all the catches with try/catch statements. I'm having some issues knowing what to do with the then statements.
export class WelcomePageContribution implements IWorkbenchContribution {
constructor(
#IInstantiationService instantiationService: IInstantiationService,
#IConfigurationService configurationService: IConfigurationService,
#IEditorService editorService: IEditorService,
#IBackupFileService backupFileService: IBackupFileService,
#IFileService fileService: IFileService,
#IWorkspaceContextService contextService: IWorkspaceContextService,
#ILifecycleService lifecycleService: ILifecycleService,
#ICommandService private readonly commandService: ICommandService,
) {
const enabled = isWelcomePageEnabled(configurationService, contextService);
if (enabled && lifecycleService.startupKind !== StartupKind.ReloadedWindow) {
backupFileService.hasBackups().then(hasBackups => {
const activeEditor = editorService.activeEditor;
if (!activeEditor && !hasBackups) {
const openWithReadme = configurationService.getValue(configurationKey) === 'readme';
if (openWithReadme) {
return Promise.all(contextService.getWorkspace().folders.map(folder => {
const folderUri = folder.uri;
return fileService.resolve(folderUri)
.then(folder => {
const files = folder.children ? folder.children.map(child => child.name) : [];
const file = arrays.find(files.sort(), file => strings.startsWith(file.toLowerCase(), 'readme'));
if (file) {
return joinPath(folderUri, file);
}
return undefined;
}, onUnexpectedError);
})).then(arrays.coalesce)
.then<any>(readmes => {
if (!editorService.activeEditor) {
if (readmes.length) {
const isMarkDown = (readme: URI) => strings.endsWith(readme.path.toLowerCase(), '.md');
return Promise.all([
this.commandService.executeCommand('markdown.showPreview', null, readmes.filter(isMarkDown), { locked: true }),
editorService.openEditors(readmes.filter(readme => !isMarkDown(readme))
.map(readme => ({ resource: readme }))),
]);
} else {
return instantiationService.createInstance(WelcomePage).openEditor();
}
}
return undefined;
});
} else {
return instantiationService.createInstance(WelcomePage).openEditor();
}
}
return undefined;
}).then(undefined, onUnexpectedError);
}
}
}
so that the entire thing reads more like this..
const enabled = await isWelcomePageEnabled(configurationService, contextService);
if (enabled && lifecycleService.startupKind !== StartupKind.ReloadedWindow) {
const hasBackups = await backupFileService.hasBackups();
const activeEditor = editorService.activeEditor;
if (!activeEditor && !hasBackups) {
const openWithReadme = configurationService.getValue(configurationKey) === 'readme';
if (openWithReadme) {
...
It looks like you're on the right track with your second code block. then is called on a promise, so instead of using then you would await the function then was called on, save it to a variable, and then move the code that was in the callback to then below the await at the same indentation level. Whenever you await, you can wrap it in a try/catch and put what would have been in the catch callback inside of the catch block.
So for example
fetchData().then(data => {
console.log(data)
}).catch(err => {
console.error(err)
})
becomes
try {
const data = await fetchData()
console.log(data)
}
catch (err) {
console.error(err)
}
The complication in your example is that the code is in a class constructor, and those can't be async.
I created multiples functions under a directory called data and fill them with some random data returned by a function called generatedRandomData.
To create multiple files I wrote these functions:
const createFile = (fileName, data) => {
if (fs.existsSync(fileName)) throw new Error('Filename already exists');
fs.writeFile(fileName, data, {
encoding: 'utf8',
flag: 'w',
}, (error) => {
if (error) return error;
console.log('File created successfully');
return null;
});
};
const createFiles = (dirPath, sizeList) => {
if (sizeList && !sizeList.length) throw new Error('The list of size should not be empty');
const fileCreationPromises = sizeList.map(async (size) => {
const data = generateRandomData(size);
const fileName = resolve(dirPath, `./data_${size}.txt`);
await createFile(fileName, data);
});
return Promise.all(fileCreationPromises);
};
Then I call the function generateData in order to generate random data and call the functions described above then create the files:
const generateData = async (dirPath, sizeList) => {
if (!dirPath) throw new Error('No directory path was provied');
if (!sizeList || (sizeList && !sizeList.length)) throw new Error('Size list should not be empty');
await createFiles(dirPath, sizeList);
};
I call another function called execute which reads data from those file in order to continue the treatment:
const execute = async (func, dirPath, label) => {
const files = fs.readdirSync(dirPath);
const result = [];
if (files && files.length) {
for (const file of files) {
const filename = resolve(dirPath, `./${file}`);
const parsedData = readDataFromFile(filename);
const data = parsedData.split(',').map((d) => Number(d));
const { length } = data;
result.push({
label: length,
value: getExecutionTime(func, data),
});
}
}
await createFile(resolve(dirPath, `./${label}`), result);
};
Finally, I call the function initialize:
const { resolve } = require('path');
const fs = require('fs');
const { generateData, sizeList, execute } = require('../utils/helpers');
const { underscorePartial } = require('../main/partial');
const dirPath = resolve(__dirname, '../data');
const initialize = () => {
if (!fs.existsSync(dirPath)) {
fs.mkdir(dirPath, async (error) => {
if (error) throw error;
await generateData(dirPath, sizeList);
await execute(underscorePartial, dirPath, 'uExecutionTime.txt');
});
}
};
try {
initialize();
} catch (error) {
console.log(error);
}
However I realized that uExecutionTime.txt to be created in the final step contains undefined due to the function readDataFromFile which returns undefined.
I guess the readDataFromFile starts reading from files before the creation of data finished.Any suggestions to fix my code or are there anything missed or wrong in the code?
The problem is your createFile function. You care awaiting it while it doesn't return promise. It is a callback style. It should be wrapped in promise.
const createFile = (fileName, data) => {
if (fs.existsSync(fileName)) throw new Error('Filename already exists');
return new Promise((resolve, reject) => {
fs.writeFile(fileName, data, {
encoding: 'utf8',
flag: 'w',
}, (error) => {
if (error) reject(error);
console.log('File created successfully');
resolve(null);
});
});
};
Hope this resolves the issue.
I have a function
getAllSymbolsConfig = async reels => {
const symbols = new Set();
for (const column of reels) {
for (const symbol of column.symbols) {
symbols.add(symbol);
}
}
const coords = {};
for (const symbol of symbols) {
const response = await fetch(
`${this.props.gameDir}icons/${symbol}.json`
);
const json = await response.json();
coords[symbol] =
json.frames.find(frame => frame.filename.includes("active_0000")) ||
json.frames.find(frame => frame.filename.includes("active")) ||
json.frames[0];
coords[symbol].meta = json.meta;
}
return coords;
};
where it tries to fetch data from url ${this.props.gameDir}icons/${symbol}.json
However, in some situations ${symbol} variable is not available and I need to replace it with hardcoded url icon.json:
${this.props.gameDir}icons/icon.json
How would i retry the second fallback url if the first one fails?
Updated Try this:
function fetchData(url) {
await fetch(url).then((response) => {
return response;
})
}
let response = null;
response = fetchData(`${this.props.gameDir}icons/${symbol}.json`)
if (response.status == 400 ) {
response = fetchData(hardCodedUrl)
}
if (!!response) {
response.then((returnedResponse) => {
// Your response to manipulate
}).catch((error) => {
// Your error is here!
console.log(error)
});
}
I have 3 functions and each of them return a promise. How can I assign the response from each promise to a defined object?
This is my code:
const runResponse = {
webAudits: [],
webJourneys: [],
appJourneys: []
};
webAuditsFailures(req)
.then(
appJourneysFailures(req)
)
.then(
webJourneysFailures(req)
).then(
res.status(201).json({ reports: runResponse })
);
This is what I've tried:
webAuditsFailures(req)
.then(
(response) => {
runResponse.webAudits = response
},
appJourneysFailures(req)
)
.then(
(response) => {
runResponse.appJourneys = response
},
webJourneysFailures(req)
).then(
(response) => {
runResponse.webJourneys = response
},
res.status(201).json({ reports: runResponse })
);
But it doesn't works as expected because the webAuditsFailures is called again even if it didn't end and I don't understand why...
These are other failed attempts to fix this:
Using await
const webAudits = await webAuditsFailures(req);
const appJourneys = await appJourneysFailures(req);
const webJourneys = await webJourneysFailures(req);
runResponse.webAudits = webAudits;
runResponse.webJourneys = webJourneys;
runResponse.appJourneys = appJourneys;
The same thing happens with this:
const webAudits = await webAuditsFailures(req);
runResponse.webAudits = webAudits;
Using the co module:
co(function* () {
var runResponse = yield {
webAudits: webAuditsFailures(req),
webJourneys: appJourneysFailures(req),
appJourneys: webJourneysFailures(req)
};
res.status(201).json({ reports: runResponse });
});
Using Promise.all:
Promise.all([webAuditsFailures(req), appJourneysFailures(req),
webJourneysFailures(req)])
.then(function(allData) {
res.status(201).json({ reports: allData });
});
This is the webAuditsFailures function, which sequentially calls another functions that return a promise
export default async (req) => {
const report = req.body.webAudits;
const def = deferred();
if(report.length > 0) {
var reportList = [];
for(const [reportIndex, item] of report.entries()) {
for(const [runIndex, run] of item.runs.entries()) {
const result = await waComplianceBusiness(req, run.id);
var failureList = [];
if(result.data.overviews) {
const compliance = result.data.overviews[0].compliance;
if(compliance) {
for(const [index, rule] of compliance.entries()) {
const response = await waRuleOverview(req, run.id, rule.id);
const failedConditions = response.data.failedConditions;
const ruleName = response.data.ruleName;
if(response.data.pagesFailed > 0) {
for(const [condIndex, condition] of failedConditions.entries()) {
const request = {
itemId: condition.conditionResult.id,
itemType: condition.conditionResult.idType,
parentId: condition.conditionResult.parentId,
parentType: condition.conditionResult.parentType
}
const body = {
runId: run.id,
ruleId: rule.id,
payload: request
}
waConditionOverview(req, body).done(response => {
// do stuff here
});
}
}
}
if(failureList.length > 0) {
item.runs[runIndex].failures = failureList;
}
}
}
}
}
def.resolve(report);
return def.promise
}
else {
return [];
}
}
This is the problem:
waConditionOverview(req, body).done(response => {
// do stuff here
});
You're performing an async action but not waiting for the result. Don't use the deferred model - use util.promisify for callbacks.
In addition, I warmly recommend not mutating the request/resposne like this but storing the information in objects and returning those.
Here is how you'd write the code:
export default async (req) => {
const report = req.body.webAudits;
if(report.length === 0) return;
const runs = Array.from(report.entries(), ([i, item]) => item.runs.entries());
for (const [_, run] of runs) {
const result = await waComplianceBusiness(req, run.id);
var failureList = [];
if (!result.data.overviews) {
continue;
}
const compliance = result.data.overviews[0].compliance;
if(!compliance) {
continue;
}
for(const [_, rule] of compliance.entries()) {
const response = await waRuleOverview(req, run.id, rule.id);
const { failedConditions, ruleName} = response.data;
if(failureList.length > 0) {
item.runs[runIndex].failures = failureList;
}
if(response.data.pagesFailed === 0) continue;
for(const [_, condition] of failedConditions.entries()) {
const request = {
itemId: condition.conditionResult.id,
itemType: condition.conditionResult.idType,
parentId: condition.conditionResult.parentId,
parentType: condition.conditionResult.parentType
}
const body = { runId: run.id, ruleId: rule.id, payload: request}
const reponse = await waConditionOverview(req, body);
// do stuff here
// update response
// update report, so next time we try it's updated and doesn't return empty;
}
}
}
return report;
}
In a promise chain, the current .then() should return a promise. The result of this promise will be passed to the next .then():
webAuditsFailures(req)
.then((response) => {
runResponse.webAudits = response;
return appJourneysFailures(req); // return a promise
})
.then((response) => { // response contains the result of the promise
runResponse.appJourneys = response;
return webJourneysFailures(req);
})
.then((response) => {
runResponse.webJourneys = response;
res.status(201).json({ reports: runResponse });
});
Depending on what .json() in the last .then() does, you should return that as well if there are other .then()s in the promise chain.