My objective is to read data from two files and compare the data. My input files are result3.json and result4.json.The data in these files are coma separated.
result3.json
[
"temp1.txt",
"temp2.txt",
]
node:
function readFromFile(file) {
var fileNames = [];
//setTimeout(function() {
fs.readFile(file,function(err, data){
if (err) {
return console.log(err);
}
var temp = JSON.parse(data.toString().split(","));
// console.log(temp.length);
for (let index = 0; index < temp.length; index++) {
//console.log(temp[index]);
fileNames.push(temp[index]);
//console.log(fileNames[index]);
}
Done(); // to block the async call
});
//},3000);
//console.log(fileNames.length);
return fileNames;
}
var baseListOfFiles = readFromFile('./output/result3.json'); // Assume this is the base file
var currentListOfFiles = readFromFile('./output/result4.json'); // Assume this is the current file
function Done(){
//console.log('Out baseListOfFiles + ' + baseListOfFiles.length);
for (let index = 0; index < baseListOfFiles.length; index++) {
console.log("[baseListOfFiles] " + baseListOfFiles[index]);
}
//console.log('Out currentListOfFiles+ ' + currentListOfFiles.length);
for (let index = 0; index < currentListOfFiles.length; index++) {
console.log("[currentListOfFiles] " + currentListOfFiles[index]);
}
}
Above is my code. It seems to be async call, so it always return 0 fileNames.
Is there any way to control it?
Here's example code using Promises:
const fs = require('fs');
function readFromFile(file) {
return new Promise((resolve, reject) => {
fs.readFile(file, function (err, data) {
if (err) {
console.log(err);
reject(err);
}
else {
resolve(JSON.parse(data));
}
});
});
}
const promises = [
readFromFile('./output/result3.json'),
readFromFile('./output/result4.json')
];
Promise.all(promises).then(result => {
console.log(result);
baseListOfFiles = result[0];
currentListOfFiles = result[1];
// do more stuff
});
First, an array promises is built; each Promise reads the file, then calls resolve with the result.
This array is passed to Promise.all(), which then calls the callback, passing the array of results in the same order.
You're right, readFile is async. What you're looking for is readFileSync: https://nodejs.org/api/fs.html#fs_fs_readfilesync_path_options
With that can can do:
const data = fs.readFileSync(file);
//do something with data
There are a few ways to 'promisify' readFile if you like, the options are discussed here: Using filesystem in node.js with async / await
Related
tldr at the bottom:
I don't really know how to explain my problem so I start with an example.
I have this async function (in reactJS but I think this is a JS related issue).
onUploadDrop = async (e, folderId) => {
e.preventDefault();
// check if the user uploaded files or folders
var uploadedItems = e.dataTransfer.items;
let files = [];
for (var i = 0; i < uploadedItems.length; i++) {
let item = uploadedItems[i].webkitGetAsEntry();
if (item.isDirectory) {
alert("is directory")
} else {
var file = await this.getFileByWebkitEntry(item);
files.push(file);
}
console.log(i);
}
// do something with files[]
}
This function is calling another async function:
getFileByWebkitEntry = async (item) => {
return new Promise(resolve => {
item.file(function (file) {
resolve(file);
}, function (err) {
console.log(err);
resolve("");
});
});
}
I'm looping through e.datatransfer.files which are basically some uploaded files or folders. Unfortunately this for-loop gets only executed once.
I did some debugging and found out that if I place a console.log before and after this line: var file = await ... This comes out:
tldr: After the await statement uploadedItems is empty thus ending the loop. Why is this happening?
I solved this by not using async - await but Promises instead.
It looks like this:
onUploadDrop = (e, folderId) => {
e.preventDefault();
// check if the user uploaded files or folders
var uploadedItems = e.dataTransfer.items;
let promises = [];
for (var i = 0; i < uploadedItems.length; i++) {
let item = uploadedItems[i].webkitGetAsEntry();
if (item.isDirectory) {
alert("is directory")
} else {
promises.push(this.getFileByWebkitEntry(item));
}
console.log(i);
}
Promise.all(promises).then(result => {
// do something with result (result = files)
});
The return Promise.all([photoArray]) returns an empty array, seemingly not waiting for the callFB to return its promise that then pushes into the array.
I am not sure what I am doing wrong but am relatively new to Promises with for loops and Ifs.
I am not sure exactly if I am using the correct number of Promises but I seem to not be able to get the 3rd tier Promise.all to wait for the for loop to actually finish (in this scenario, the for loop has to look through many item so this is causing an issue where it is not triggering callFeedback for all the items it should before context.done() gets called.
I have tried using Q.all also for the Promise.all([photoArray]) but have been unable to get that working.
module.exports = function (context, myBlob) {
var res = myBlob
var promiseResolved = checkPhoto(res,context);
var promiseResolved2 = checkVideo(res,context);
Promise.all([promiseResolved, promiseResolved2]).then(function(results){
context.log(results[0], results[1]);
// context.done();
});
});
};
};
function checkPhoto(res, context){
return new Promise((resolve, reject) => {
if (res.photos.length > 0) {
var photoArray = [];
for (var j = 0; j < res.photos.length; j++) {
if (res.photos[j].feedbackId !== null){
var feedbackId = res.photos[j].feedbackId;
var callFB = callFeedback(context, feedbackId);
Promise.all([callFB]).then(function(results){
photoArray.push(results[0]);
});
} else {
photoArray.push("Photo " + j + " has no feedback");
}
}
return Promise.all([photoArray]).then(function(results){
context.log("end results: " + results);
resolve(photoArray);
});
} else {
resolve('No photos');
}
})
}
function checkVideo(res, context){
return new Promise((resolve, reject) => {
same as checkPhoto
})
}
function callFeedback(context, feedbackId) {
return new Promise((resolve, reject) => {
var requestUrl = url.parse( URL );
var requestBody = {
"id": feedbackId
};
// send message to httptrigger to message bot
var body = JSON.stringify( requestBody );
const requestOptions = {
standard
};
var request = https.request(requestOptions, function(res) {
var data ="";
res.on('data', function (chunk) {
data += chunk
// context.log('Data: ' + data)
});
res.on('end', function () {
resolve("callFeedback: " + true);
})
}).on('error', function(error) {
});
request.write(body);
request.end();
})
}
The code suffers from promise construction antipattern. If there's already a promise (Promise.all(...)), there is never a need to create a new one.
Wrong behaviour is caused by that Promise.all(...).then(...) promise isn't chained. Errors aren't handled and photoArray.push(results[0]) causes race conditions because it is evaluated later than Promise.all([photoArray])....
In case things should be processed in parallel:
function checkPhoto(res, context){
if (res.photos.length > 0) {
var photoArray = [];
for (var j = 0; j < res.photos.length; j++) {
if (res.photos[j].feedbackId !== null){
var feedbackId = res.photos[j].feedbackId;
var callFB = callFeedback(context, feedbackId);
// likely no need to wait for callFB result
// and no need for Promise.all
photoArray.push(callFB);
} else {
photoArray.push("Photo " + j + " has no feedback");
}
}
return Promise.all(photoArray); // not [photoArray]
} else {
return 'No photos';
};
}
callFB promises don't depend on each other and thus can safely be resolved concurrently. This allows to process requests faster.
Promise.all serves a good purpose only if it's used to resolve promises in parallel, while the original code tried to resolve the results (results[0]).
In case things should be processed in series the function benefits from async..await:
async function checkPhoto(res, context){
if (res.photos.length > 0) {
var photoArray = [];
for (var j = 0; j < res.photos.length; j++) {
if (res.photos[j].feedbackId !== null){
var feedbackId = res.photos[j].feedbackId;
const callFBResult = await callFeedback(context, feedbackId);
// no need for Promise.all
photoArray.push(callFBResult);
} else {
photoArray.push("Photo " + j + " has no feedback");
}
}
return photoArray; // no need for Promise.all, the array contains results
} else {
return 'No photos';
};
}
Add try..catch to taste.
I am trying to iterate through the JSON files generated by the protractor tests. I pull all the file names into an array and call a method that opens and parses through the each file, post the results to the database and pass back a passed/failed flag.
I have tried all the examples here
Make angular.forEach wait for promise after going to next object and still get the same results.
The method is actually called, but the results are not posted to the db. I have tested the parser.parseResults on an individual file and it successfully posted to the db, so it has to have something to do with the promise not resolving correctly.
Is it not possible to do something like this in the jasmine/protractor framework? Or do I have something wrong in the code?
I have included the code for my latest attempt.
Thank You
Christine
matches.reduce(function (p, val) {
console.log('val', val);
return p.then(function () {
return parser.parseResults(val);
});
}, Promise.resolve()).then(function (finalResult) {
console.log('finalResult = ', finalResult);
}, function (err) {
console.log('error in reduce',err);
});
parser.parseResults code
protractorParser.prototype.parseResults = function (fileName) {
return new Promise((resolve, reject) => {
console.log('In parseresults', fileName);
json.readFile(fileName, function (err, obj) {
try {
if (err != null) {
console.log('error reading file',err);
reject(err);
}
console.log('obj - ',obj);
var results = [];
var Passed = 0;
var Message = '';
var Stack = '';
for (var suite in obj) {
var specs = obj[suite].specs;
console.log('spec - ', specs);
if (specs.length > 0) {
for (var i = 0; i < specs.length; i++) {
var assert = specs[i];
var tcR = new RegExp(/TC[\d]+/);
var tc = assert.description.match(tcR);
if (!assert.failedExpectations.length) {
Passed = 1;
}
else {
assert.failedExpectations.forEach((expectation) => {
Message = expectation.message;
Stack = expectation.stack.split('\n')[1].trim();
})
Passed = 0;
}
if (tc != null) {
utility.TestDataManager.insertAutomationResults(tc[0], assert.description, Passed, process.env.testBuild,
'P', Message, Stack, 0, moment().utcOffset(config.get('settings.timeOffset')).format('YYYY-MM-DDTHH:mm:ss'), '')
.then(function (resp) {
resolve(Passed);
}, (err) => {
console.log('Posting to Database failed ', err);
reject(err);
});
} else {
console.log('no test case found for test: ' + assert.description + ' -- skipping');
reject(err);
}
}
}
}
}
catch (err) {
console.log('rejecting opening file');
reject(err);
}
});
})
}
If there is not exactly one suite in the obj, with exactly one spec, then your promise is either resolved not at all or multiple times.
Avoid wrapping too many things in the new Promise constructor - always promisify on the smallest possible level, and use promise chaining afterwards.
protractorParser.prototype.parseResults = function (fileName) {
return new Promise((resolve, reject) => {
console.log('In parseresults', fileName);
json.readFile(fileName, function (err, obj) {
if (err != null) {
console.log('error reading file', err);
reject(err);
} else {
resolve(obj);
}
});
}).then(function(obj) {
console.log('obj - ',obj);
var results = [];
for (var suite in obj) {
var specs = obj[suite].specs;
console.log('spec - ', specs);
for (let i = 0; i < specs.length; i++) {
const assert = specs[i];
const tcR = /TC[\d]+/;
const tc = assert.description.match(tcR);
let Passed = 1;
let Message = '';
let Stack = '';
if (assert.failedExpectations.length) {
const expectation = assert.failedExpectations[assert.failedExpectations.length-1];
Passed = 0;
Message = expectation.message;
Stack = expectation.stack.split('\n')[1].trim();
}
if (tc != null) {
const time = moment().utcOffset(config.get('settings.timeOffset')).format('YYYY-MM-DDTHH:mm:ss');
const promise = utility.TestDataManager.insertAutomationResults(tc[0], assert.description, Passed, process.env.testBuild, 'P', Message, Stack, 0, time, '');
results.push(promise.catch(err => {
console.log('Posting to Database failed ', err);
throw err;
}));
} else {
console.log('no test case found for test: ' + assert.description + ' -- skipping');
// I don't think you want to `throw err` here, right?
}
}
}
return Promise.all(results);
});
};
New to Node.js here. I'm looking for the correct way to make N asynchronous API calls from within another function, and combining their results to use further downstream. In my case, N would be reasonably small and blocking for their execution not too bad.
In synchronous execution the implementation for combine() below should work.
If I only needed the results from one API call it would be straightforward to implement the following logic in a callback function supplied to callAPI(). Where I stumble is when I need all the results combined before before executing foo(total, [...args]).
I looked into async.whilst but wasn't able to get that to work. I'm skeptical if that actually is the correct fit to my needs. I've also looked into Promises which seems to be the correct lead but it would be nice to get reassurances before crawling into that cavernous rabbit hole. Be it that Promises is the correct way, which module is the standard to use in Node.js projects?
var http = require('http');
function callAPI(id) {
var options = {
host: 'example.com',
path: '/q/result/'.concat(id)
}
var req = http.get(options, (res) => {
var body = [];
res.on('data', (chunk) => {
body.push(chunk);
}).on('end', () => {
body = Buffer.concat(body).toString();
return body;
}).on('error', (err) => {
console.error(err);
});
});
}
function combine(inputs) {
var total = 0;
for (i=0; i < inputs.length; i++) {
total += callAPI(inputs[i]['id']);
};
console.log(total);
// call some function, foo(total, [...args])
}
Edit 1:
I attempted to follow samanime's answer below and modify the API call to return a Promise. See:
function callAPI(id) {
return Promise((resolve, reject) => {
var options = {
host: 'example.com',
path: '/q/result/'.concat(id)
}
var req = http.get(options, (res) => {
var body = [];
res.on('data', (chunk) => {
body.push(chunk);
}).on('end', () => {
body = Buffer.concat(body).toString();
resolve(body);
}).on('error', (err) => {
reject(err);
});
});
});
}
function combine(inputs) {
var combined = [];
for (i=0; i < inputs.length; i++) {
total += callAPI(inputs[i]['id']);
.then(result => {
combined.push(result);
});
};
var total = combined.reduce((a, b) => a + b, 0);
console.log(total);
// call some function, foo(total, [...args])
}
This seems to get me halfway there. If I console.log(combined) inside the then() block I can see the list building up with results from the API calls. However, I still can't access the complete combined at the "end" of the for loop. Can I attach a callback to something to run after the full list has been built? Is there a better way?
Edit 2 (My solution - per Patrick Roberts suggestion)
function callAPI(id) {
return Promise((resolve, reject) => {
var options = {
host: 'example.com',
path: '/q/result/'.concat(id)
}
var req = http.get(options, (res) => {
var body = [];
res.on('data', (chunk) => {
body.push(chunk);
}).on('end', () => {
body = parseInt(Buffer.concat(body));
resolve(body);
}).on('error', (err) => {
reject(err);
});
});
});
}
function combine(inputs) {
var combined = [];
Promise.all(inputs.map(input => callAPI(input.id)))
.then((combined) => {
var total = combined.reduce((a, b) => a + b, 0);
// foo(total, [...args])
});
};
It sounds like you can just chain together a bunch of promises, passing the data along.
Basically something like:
const combined = [];
asyncOne()
.then(result => { combined.push(result); return asyncTwo())
.then(result => { combined.push(result); return asyncThree())
// and so on
As long as each function returns a promise, you'll be all set.
If you want to run them in parallel, use Promise.all(), which will do the same thing for you:
Promise.all([asyncOne(), asyncTwo(), asyncThree() /* , etc */])
.then(combined => /* combined is an array with the results of each */)
This is by far the preferred pattern for this sort of thing.
Your edit is looking a lot better, but try this:
function callAPI(id) {
return Promise((resolve, reject) => {
var options = {
host: 'example.com',
path: '/q/result/' + id
}
http.get(options, (res) => {
var body = [];
res.on('data', (chunk) => {
body.push(chunk);
}).on('end', () => {
body = Buffer.concat(body).toString();
resolve(body);
}).on('error', reject);
});
});
}
function combine(inputs) {
Promise.all(inputs.map(input => callAPI(input.id))).then((combined) => {
// completed array of bodies
console.log(combined);
// foo(combined.length, [...args]);
}).catch((error) => {
console.log(error);
});
}
I would add a counter that keeps track of remaining API calls. Whenever an API call finishes, decrement and if its 0, you're done.
const numCalls = 10;
let remaining = numCalls;
let data = [];
function getRandomInt(min, max) {
min = Math.ceil(min);
max = Math.floor(max);
return Math.floor(Math.random() * (max - min)) + min;
}
function ajax() {
// Simulate ajax with a setTimeout for random amount of time.
setTimeout(() => {
// This is the callback when calling http.get
data.push(getRandomInt(0, 10)); // Some data from server
if (--remaining <= 0) {
// Am I the last call? Use data.
console.log(data);
console.log(data.length);
}
}, getRandomInt(1000, 3000));
}
for (let i = 0; i < numCalls; i++) {
ajax();
}
Here are the cloud functions namely 'batchReq1' and batchPromises.
In any case, if I know the exact number of promises pushed (Consider the size of results to be '2' in function batchPromises(results)) and executed through when(), I can handle the success response by passing that number of result parameters (In the below example request1, request2) in successCallBack of .then().
If I have to process the number of promises pushed to .when() dynamically, then, how can we handle this in SuccessCallBack? Unlike earlier scenario, we can't expect fixed number of results in the then method (batchPromises(results).then(function (result1, result2) {....)
batchReq1
Parse.Cloud.define("batchReq1", function (request, response) {
var results = request.params.imageArray;
batchPromises(results).then(function (result1, result2) {
console.log("Final Result:: Inside Success");
console.log("Final Result:: Inside Success result 1::::"+result1);
console.log("Final Result:: Inside Success result 2::::"+result2);
response.success();
}
// batchPromises(results).then(function (arraySuccess) {
//
// console.log("Final Result:: Inside Success");
// console.log("Final Result:: Inside Success:: Length:: "+arraySuccess.length);
// //Fetch all responses from promises and display
// var _ = require('underscore.js');
// _.each(arraySuccess, function (result) {
//
// console.log("Final Result:: " + result)
//
// });
//
//
// response.success();
//
// }
, function (error) {
console.log("Final Result:: Inside Error");
response.error(error);
});
});
batchPromises
function batchPromises(results) {
var promise = Parse.Promise.as();
var promises = [];
var increment = 0;
var isFromParallelExecution = false;
var _ = require('underscore.js');
_.each(results, function (result) {
var tempPromise = Parse.Promise.as("Promise Resolved ");
promises.push(tempPromise);
}
promise = promise.then(function () {
return Parse.Promise.when(promises);
});
}
increment++;
});
return promise;
}
this is how i handle this...
Parse.Cloud.define("xxx", function(request, response)
{
var channels = ["channel1", "channel2", "channel3", "channel4", "channel5"];
var queries = new Array();
for (var i = 0; i < channels.length; i++)
{
var query = new Parse.Query(channels[i]);
queries.push(query.find());
}
Parse.Promise.when(queries).then(function()
{
var msg = "";
for (var j = 0; j < arguments.length; j++)
{
for (var k = 0; k < arguments[j].length; k++)
{
var object = arguments[j][k];
msg = msg + " " + object.get('somefield');
}
}
response.success(msg);
});
});
Either you just use the arguments object to loop over the results, or you build your arraySuccess without when - it doesn't make much sense here anyway as you batch the requests (executing them sequentially), instead of executing them in parallel:
function batchPromises(tasks) {
var _ = require('underscore.js');
_.reduce(tasks, function (promise, task) {
return promise.then(function(resultArr) {
var tempPromise = Parse.Promise.as("Promise Resolved for "+taks);
return tempPromise.then(function(taskResult) {
resultArr.push(taskResult);
return resultArr;
});
});
}, Parse.Promise.as([]));
}
If you actually wanted to execute them in parallel, use a simple
function batchPromises(tasks) {
var _ = require('underscore.js');
return Parse.Promise.when(_.map(tasks, function (task) {
return Parse.Promise.as("Promise Resolved for "+taks);
}).then(function() {
return [].slice.call(arguments);
});
}