We Keep Coding

nodejs filtering an array of objects where the filtering is partially done in an async function

I've read many similar questions and have tried a bunch of code. Unfortunately, I'm not getting my code to run :-(
So, the situation is as follows: In a route of a node.js server, I have to respond with a filtered array of Objects. Unfortunately, whatever I do, I always get an empty array [] back. The filter is a bit tricky in my opinion, as it consists of a string comparison AND an async call to a library function. With the console output, I can clearly see that the correct element is found, but at the same time I see that I've already received the object...
Here is some code that exemplifies my challenge:
let testArray = [
{
id: 'stringId1',
data: {
someDoc: {
moreContent: 'Some random content',
type: 'noInterest'
}
}
},
{
id: 'stringId2',
data: {
someDoc: {
moreContent: 'Some random content',
type: 'ofInterest'
}
}
},
{
id: 'stringId3',
data: {
someDoc: {
moreContent: 'Some random content',
type: 'ofInterest'
}
}
}
]
// code from a library. Can't take an influence in it.
async function booleanWhenGood(id) {
if (id in some Object) {
return { myBoolean: true };
} else {
return { myBoolean: false };
}
}
// Should return only elements with type 'ofInterest' and that the function booleanWhenGood is true
router.get('/', function(res,req) {
tryOne(testArray).then(tryOneResult =>{
console.log('tryOneResult', tryOneResult);
});
tryTwo(testArray).then(tryTwoResult => {
console.log("tryTwoResult ", tryTwoResult);
});
result = [];
for (const [idx, item] of testArray.entries() ) {
console.log(idx);
if (item.data.someDoc.type === "ofInterest") {
smt.find(item.id).then(element => {
if(element.found) {
result.push(item.id);
console.log("ID is true: ", item.id);
}
});
}
if (idx === testArray.length-1) {
// Always returns []
console.log(result);
res.send(result);
}
}
})
// A helper function I wrote that I use in the things I've tried
async function myComputeBoolean(inputId, inputBoolean) {
let result = await booleanWhenGood(inputId)
if (result.myBoolean) {
console.log("ID is true: ", inputId);
}
return (result.myBoolean && inputBoolean);
}
// A few things I've tried so far:
async function tryOne(myArray) {
let myTmpArray = []
Promise.all(myArray.filter(item => {
console.log("item ", item.id);
myComputeBoolean(item.id, item.data.someDoc.type === "ofInterest")
.then(myBResult => {
console.log("boolean result", myBResult)
if (myBResult) {
tmpjsdlf.push(item.id);
return true;
}
})
})).then(returnOfPromise => {
// Always returns [];
console.log("returnOfPromise", myTmpArray);
});
// Always returns []
return(myTmpArray);
}
async function tryTwo(myArray) {
let myTmpArray = [];
myArray.forEach(item => {
console.log("item ", item.id);
myCompuBoolean(item.id, item.data.someDoc.type === "ofInterest")
.then(myBResult => {
console.log("boolean result", myBResult)
if (myBResult) {
myTmpArray.push(item.did);
}
})
});
Promise.all(myTmpArray).then(promiseResult => {
return myTmpArray;
});
}
Asynchronous programming is really tough for me in this situation... Can you help me get it running?

I didn't inspect your attempts that closely, but I believe you are experiencing some race conditions (you print return and print the array before the promises resolve).
However you can alwayd use a regular for loop to filter iterables. Like this:
let testArray = [
{
id: 'stringId1',
data: {
someDoc: {
moreContent: 'Some random content',
type: 'noInterest'
}
}
},
{
id: 'stringId2',
data: {
someDoc: {
moreContent: 'Some random content',
type: 'ofInterest'
}
}
},
{
id: 'stringId3',
data: {
someDoc: {
moreContent: 'Some random content',
type: 'ofInterest'
}
}
}
]
async function booleanWhenGood(id) {
if (id in { 'stringId1': 1, 'stringId2': 1 }) { // mock object
return { myBoolean: true };
} else {
return { myBoolean: false };
}
}
async function main() {
let filtered = []
for (item of testArray)
if ((await booleanWhenGood(item.id)).myBoolean && item.data.someDoc.type === 'ofInterest')
filtered.push(item)
console.log('filtered :>> ', filtered);
}
main()

How to make sequential if statements asynchronous in JavaScript?

I have a series of if statements in a loop like this:
for( var i = 0; i < results_list.length; i++){
find = await results_list[i];
//result 1
if (find.Process == "one") {
await stored_proc(38, find.Num, find.Status)
}
//result 2
if(find.Process == "two") {
await stored_proc(37, find.Num, find.Status)
}
//result 3
if(find.Process == "three") {
await stored_proc(39, find.Num, find.Status)
}
}
My issue is that it runs all of these synchronously causing my stored procedure to trip over itself. How can I ensure each if statement waits for the previous one to complete before running?
It is also important to know that each if statement is not always run, for instance on one run of my code //result 2 may run and //result 1 and //result 3 may not run.
Sometimes they might all run and sometimes none will run at all.
Thanks for any help!
EDIT: Here is my stored procedure function
async function stored_proc(opID, num, stat){
sql.executeTransaction( connection, {
procedure: "<stored procedure>",
params: {
OpID: {
val: opID,
type: sql.INT
},
num: {
val: num,
type: sql.STRING
},
Pass: {
val: stat,
type: sql.INT
},
ExtraData: {
val: "upload",
type: sql.STRING
}
}
} ).then( async function( data ) {
return data.transaction
.commit()
.then( async function() {
console.log("Updated database...." );
} );
}, function( err ) {
console.log( err );
} );
}
SECOND EDIT: I have looked into this some more and found that the if there is more than one result to upload it will NEVER upload the first sets of results. I have ran some console.log()s through and found it will always get find.Num and find.Status. It will only log Updated database for every result after the first one. I hope this makes sense

In your stored_proc you are not returning the Promise.
Also, promise inside promise is considered anti-pattern you can chain it easily.
async function stored_proc(opID, num, stat) {
return sql.executeTransaction(connection, {
procedure: "<stored procedure>",
params: {
OpID: {
val: opID,
type: sql.INT
},
num: {
val: num,
type: sql.STRING
},
Pass: {
val: stat,
type: sql.INT
},
ExtraData: {
val: "upload",
type: sql.STRING
}
}
})
.then(function (data) {
return data.transaction
.commit()
})
.then(function () {
console.log("Updated database....");
})
.catch((err) => {
console.log(err);
})
}

Testing RxJS using TestScheduler with Promises and angular

I'm having a small issue using RxJS and Angular (not Angular 2) that I'm sure indicates I'm just doing something wrong, but I'm not sure exactly what.
I have a function that creates an rx.Observable stream that I would like to test. A simplified version of the function is below:
ResourceCollection.prototype.rxFetch = function() {
var scheduler = this.injectedScheduler;
var result = functionThatReturnsAnObservable(theseParams).concatMap(function(items) {
var promises = _.map(readFromExternal(items), function(promise) {
// results of this promise should be ignored
return Rx.Observable.fromPromise(promise, scheduler);
});
promises = promises.concat(_.map(items, function(item) {
// callEvent returns EventResult, these values should be passed on
return Rx.Observable.fromPromise(callEvent(item), scheduler);
}));
return promises;
}).concatMap(function(x) { return x; }).filter(function(res) {
return (res instanceOf EventResult);
}).toArray();
return result;
});
My test function looks like this:
describe('query', function() {
var customers;
var scheduler;
beforeEach(function() {
scheduler = new Rx.TestScheduler();
customers = new ResourceCollection({
url: '/api/customers',
keyName: 'CustomerId',
globalActions: {
rxQuery: { method: 'GET', isArray: true }
}
});
$httpBackend.whenGET('/api/customers/rxQuery').
respond(function() {
return [200, [
{ CustomerId: 1, Name: 'Brian', Region: 'North' },
{ CustomerId: 2, Name: 'Ravi', Region: 'East' },
{ CustomerId: 3, Name: 'Ritch', Region: 'East' },
{ CustomerId: 4, Name: 'Jeff', Region: 'West' },
{ CustomerId: 5, Name: 'Brandon', Region: 'West' }
]];
});
});
it('rxFetch customers', function(done) {
var vals;
customers.injectedScheduler = scheduler
var result = customers.rxFetch();
result.subscribe(function(values) {
vals = values;
});
$httpBackend.flush();
// my question is here - what can I do to get rid of this loop?
while (vals == null) {
scheduler.advanceBy(100);
$rootScope.$apply();
}
scheduler.start();
expect(vals.length).toEqual(5);
expect(vals[0]).toBe(customers[0]);
done();
});
});
The issue is a simple one - while the while loop in the test is in there, the test will produce the correct results (which is an array that contains the results of all the callEvent functions). Replace the while loop with a scheduler.scheduleAbsolute (or some other such call) combined with a $rootScope.$apply, and only one of the promises from the callEvent function will complete. Call it twice, and two of them will complete, etc (hence the while loop).
But the while loop is pretty ugly - and I'm sure there has to be an cleaner way to get this test to pass. Many thanks to anyone who can point me in the correct direction.

How can I unit test a complex promise chain?

I have some JavaScript code:
var findLeastUsedPassage;
findLeastUsedPassage = function(StudentId) {
var passageCounts;
passageCounts = [];
return db.Passage.findAll({
where: {
active: true
}
}).each(function(dbPassage) {
var passage;
passage = dbPassage.get();
passage.count = 0;
return passageCounts.push(passage);
}).then(function() {
return db.Workbook.findAll({
where: {
SubjectId: 1,
gradedAt: {
$ne: null
},
StudentId: StudentId
},
include: [
{
model: db.WorkbookQuestion,
include: [db.Question]
}
],
limit: 10,
order: [['gradedAt', 'DESC']]
});
}).each(function(dbWorkbook) {
return Promise.resolve(dbWorkbook.WorkbookQuestions).each(function(dbWorkbookQuestion) {
var passageIndex;
passageIndex = _.findIndex(passageCounts, function(passageCount) {
return passageCount.id === dbWorkbookQuestion.Question.PassageId;
});
if (passageIndex !== -1) {
return passageCounts[passageIndex].count++;
}
});
}).then(function() {
passageCounts = _.sortBy(passageCounts, 'count');
return passageCounts;
});
};
and I want to unit test it (I think). I instrumented mocha to do the testing, but my test doesn't seem all that.. thorough:
describe('Finding the least used Passage', function() {
it('should have a function called findLeastUsedPassage', function() {
return expect(WorkbookLib.findLeastUsedPassage).to.exist;
});
return it('should return the least used passages for a student', function() {
return WorkbookLib.findLeastUsedPassage(10).then(function(passageCounts) {
var passageCountsLength;
passageCountsLength = passageCounts.length;
expect(passageCountsLength).to.equal(74);
expect(passageCounts[0].count).to.be.at.most(passageCounts[1].count);
expect(passageCounts[1].count).to.be.at.most(passageCounts[5].count);
expect(passageCounts[56].count).to.be.at.most(passageCounts[70].count);
return expect(passageCounts[70].count).to.be.at.most(passageCounts[73].count);
});
});
});
What's the right approach to unit testing something like this?

This is a great resource for understanding how to break up your code to able to test it.
Currently, you're code can't be tested well because the logic is all intermingled between multiple database calls, business logic, and glue code. What you need to do is break it all out into multiple named functions that each do one thing, like you do now. Expect that instead of creating the functions in the chain you should create them outside of the chain, then just call them in the promise chain.
var passageCounts = [];
function findAllActivePassages() {
passageCounts = [];
return db.Passage.findAll({
where: {
active: true
}
})
}
function countPassages(dbPassage) {
var passage;
passage = dbPassage.get();
passage.count = 0;
return passageCounts.push(passage);
}
function findAllSubjects(StudentId) {
return db.Workbook.findAll({
where: {
SubjectId: 1,
gradedAt: {
$ne: null
},
StudentId: StudentId
},
include: [
{
model: db.WorkbookQuestion,
include: [db.Question]
}
],
limit: 10,
order: [['gradedAt', 'DESC']]
});
})
// ...
findAllActivePassages()
.each(countPassages)
.then(function() {
return findAllSubjects(studentId)
})
// ...
Now you can test each function individually and in isolation to ensure that they do what you expect

So for starters, you probably want to break up your promise chains to make the discrete units of your code more apparent. I did some quick psuedo javascript (most familliar w/ node so apologies if this doesn't fit vanilla javascript as cleanly).
var p1 = db.Passage.findAll({ where: { active: true }})
var p2 = db.Workbook.findAll({
where: {
SubjectId: 1,
gradedAt: {
$ne: null
},
StudentId: StudentId
},
include: [
{
model: db.WorkbookQuestion,
include: [db.Question]
}
],
limit: 10,
order: [['gradedAt', 'DESC']]
});
Promise.all([p1, p2])
.then(function(results){
var passages = results[0]
var workbooks = results[1];
var passageCounts = {};
passages.foreach(function(passage){
passagecounts[passage.get().id] = 0
});
workbooks.foreach(function(workbook){
workbook.workBookQuestions.foreach(function(question){
return passageCounts[dbWorkbookQuestion.Question.PassageId] += 1;
})
});
return Promise.resolve(passageCounts)
}).then(function(passageCounts){
passageCounts = _.sortBy(passageCounts, 'count'); //this has to change but don't know what underscore offers for sorting an object used as a hashmap
return passageCounts;
});
Now as far as unit testing - you're looking to test discrete units of it so the following use cases seem reasonable:
Do I get any result back when expected?
If i give it specific values are they sorted in the way I expect?
If I have no results for either query does it break? Should it?
It may behoove you to break out the DB calls from the logic and pass the results into a method, makes testing some of the scenarios a bit easier.

why [Circular] is getting added in the hash

here is my code
db.query(str, arr, function selectCb(error, results, fields) {
if (error) {
return proceed(false, {errno:'010',message:error.message}, request);
}
var q = async.queue(function (results, callback) {
// add the gib infor
if (results.refertype=='G') {
var input={};
input.fields="*";
input.gibname=results.refername;
gib.getgibinternal(input, makeCallback(i));
function makeCallback(index) {
return function(gresult) {
results.gib=gresult.data[0];
callback(results);
}
}
// add the user info
} else if(results.refertype=='U') {
var input={};
input.username=results.refername;
input.fields="*";
user.getuserinternal(input, makeCallbackuser(i));
function makeCallbackuser(index) {
return function(gresult) {
results.user=gresult.data[0];
callback(results);
}
}
}
}, results.length);
// assign a callback
q.drain = function() {
return proceed(true, self.results, self.request);
}
self.results=results;
for (var i=0; i<results.length; i++) {
// the first console
console.log(results[i]);
// add some items to the queue
q.push(results[i], function (results) {
results[i]=results;
self.results[i]=results;
//the second console.
console.log(results);
});
}
if (results.length==0) {
return proceed(true, results, request);
}
});
The out put of above code is :
// the first console
{ cardid: 30,
cardtype: 'I',
status: 'A',
refername: 'admin',
refertype: 'U' }
// the second console
{ '1': [Circular],
cardid: 30,
cardtype: 'I',
status: 'A',
refername: 'admin',
refertype: 'U',
user:
{ name: 'admin',
username: 'admin',
deleted: 'N' } }
how this '1': [Circular], get added .?

This bit:
q.push(results[i], function (results) {
is the same as this (with some renaming to make it easier to track the scope):
q.push(self.results[i], function(r) {
r[i] = r; // <------------------- Look at me!
self.results[i] = r;
//the second console.
console.log(r);
});
The self.results[i] change just comes from the self.results=results; right above your for loop. The interesting part is this:
r[i] = r;
If i is 1 you will have added a property named 1 to r whose value is r itself, hence the [Circular]. I would hazard a guess that results.length is 2 and that your function is acting as a closure over i and ending up with using the last value that i had and that's why you get the '1' rather than a '0' property.
I see three main Things that could be causing you trouble:
The classic closure problem with i.
Too many things were called results so it was easy to lose track of which one you were working with.
The circular reference: r[i] = r;.
Another possible source of confusion is that results and self.results are the same object but that might be okay.

Seems like it's a circular reference and console.log is muffling it. Try doing console.log(Object.keys(results['1'])); for more information on what's inside the Object 1