We Keep Coding

Node Child Process (Spawn) is not returning data correctly when using with function

I am trying to generate a new score based on the output of my python script.
The python script returning the data correctly and JS program printing correctly
but the problem is when i return the value and print it, it shows undefined
Function Code -
async function generateCanadaScore(creditscore = 0) {
console.log(creditscore, "creditscore"); //prints 300
let MexicoData = 0;
const python = spawn("python", [
"cp_production.py",
"sample_dill.pkl",
"mexico",
Number(creditscore),
]);
await python.stdout.on("data", function (data) {
console.log("Pipe data from python script ...");
console.log(data.toString()); //prints number
MexicoData = data.toString();
console.log(MexicoData) // prints number
//working fine till here printing MexicoData Correctly (Output from py file) , problem in return
return MexicoData ;
});
python.stderr.on("data", (data) => {
console.log(data); // this function doesn't run
});
// return MexicoData ; already tried by adding return statement here still same error
}
Calling Function Code -
app.listen(3005, async () => {
console.log("server is started");
//function calling
// Only for testing purpose in listen function
let data = await generateCanadaScore(300);
console.log(data, "data"); // undefined
});
I will not be able to share python code it is confidential.

You can't await on the event handler. (It returns undefined, so you're basically doing await Promise.resolve(undefined), which waits for nothing).
You might want to wrap your child process management using new Promise() (which you'll need since child_process is a callback-async API, and you need promise-async APIs):
const {spawn} = require("child_process");
function getChildProcessOutput(program, args = []) {
return new Promise((resolve, reject) => {
let buf = "";
const child = spawn(program, args);
child.stdout.on("data", (data) => {
buf += data;
});
child.on("close", (code) => {
if (code !== 0) {
return reject(`${program} died with ${code}`);
}
resolve(buf);
});
});
}
async function generateCanadaScore(creditscore = 0) {
const output = await getChildProcessOutput("python", [
"cp_production.py",
"sample_dill.pkl",
"mexico",
Number(creditscore),
]);
return output;
}

async api call function seems to work until test

I am trying to get some exchange data and use it for some logic. I thought the most straight forward idea would be to use a function where I could pass the different pairs I may want, there are many per exchange. If I console log inside the function when the data ends everything works find. When i try and get the response when it is ready I get undefined, please see the commented console log and my last line.
I understand that I am missing something with async and the way it works.
const https = require('https');
async function binancePairPrice (binancePair) {
let url = 'https://api.binance.com/api/v3/ticker/bookTicker';
let queryString = '?symbol='
https.get(url+queryString+binancePair,(res) => {
console.log(res.statusCode);
let binanceBookTicker = '';
let binanceBookTickerJson =
res.on('data', data => {
binanceBookTicker += data;
});
res.on('end', () => {
binanceBookTickerJson = JSON.parse(binanceBookTicker);
//console.log(binanceBookTickerJson);
return (binanceBookTickerJson)
});
});
};
binancePairPrice('BTCUSDT').then(console.log);
day one javascript coming from python. This is quite a step from what I am used to writing. Any docs or links would be appreciated too.
thansk

https.get doesn't return something that can be promisified, so you can't await it.
You can do something else which is wrapping the https.get call within a Promise and then await it when calling binancePairPrice.
const https = require('https')
async function binancePairPrice (binancePair) {
let url = 'https://api.binance.com/api/v3/ticker/bookTicker';
let queryString = '?symbol='
return new Promise((resolve) => {
https.get(url, res => {
let binanceBookTicker = '';
let binanceBookTickerJson =
res.on('data', data => {
binanceBookTicker += data;
});
res.on('end', () => {
resolve() = >
{
binanceBookTickerJson = JSON.parse(binanceBookTicker);
return (binanceBookTickerJson)
}
});
})
})
}
(async () => await binancePairPrice('BTCUSDT))()

Your binancePairPrice function doesn't wait for the end callback to happen and therefore just carries on and returns undefined.
You need to return a Promise and then resolve the promise with the value you want to return from the function.
Promise basics
Using the Promise() constructor
async function binancePairPrice (binancePair) {
return new Promise((resolve, reject) => {
let url = 'https://api.binance.com/api/v3/ticker/bookTicker';
let queryString = '?symbol=';
https.get(url+queryString+binancePair,(res) => {
console.log(res.statusCode);
let binanceBookTicker = '';
let binanceBookTickerJson =
res.on('data', data => {
binanceBookTicker += data;
});
res.on('end', () => {
binanceBookTickerJson = JSON.parse(binanceBookTicker);
//console.log(binanceBookTickerJson);
resolve(binanceBookTickerJson)
});
});
});
}

What are different ways to handle asynchronous calls (apis) inside a for loop or array method?

I have been trying to create an api like this where I tried different things like using array methods like map/filter/reduce where either I get pending promises or result returned before execution of api call.
So my doubt is -->
How do I get total number of drawn matches of all pages ( so I need to add data.total from all pages).
How to better understand this behaviour.
async function getNumDraws(year) {
const goals = [...Array(11).keys()];
let result = 0;
console.log(`before loop ${new Date()}`);
for(let goal of goals){
console.log(`in loop before await ${new Date()}`);
await require('https').get(`https://jsonmock.hackerrank.com/api/football_matches?year=${year}&team1goals=${goal}&team2goals=${goal}`,res=>{
let data="";
res.on('data', (chunk) => {
data += chunk;
});
// The whole res has been received. Print out the result.
res.on('end', () => {
data=JSON.parse(data);
console.log(result,data.total)
result= result + data.total;
});
})
console.log(`in loop after await ${new Date()}`);
}
console.log(`after loop ${new Date()}`);
return result;
}
console.log(getNumDraws(2011));

https.get is a callbacked function so await won't work. You should promisify it first like they did in this other SO question;
const https = require("https"); // only require this once
const getJSONAsync = url => new Promise((resolve, reject) => { // define a function getJSONAsync which returns a Promise that wraps the https.get call, getJSONAsync is awaitable
let req = https.get(url, res => { // make the https.get request
if(res.statusCode < 200 || res.statusCode >= 300) { // if the status code is an error one
return reject(new Error('statusCode=' + res.statusCode)); // reject and skip the rest
}
let body = []; // otherwise accumulate..
res.on('data', chunk => body.push(chunk)); // ..the data
res.on('end', () => { // on end
try {
body = JSON.parse(Buffer.concat(body).toString()); // try to JSON.parse the data
} catch(e) {
reject(e); // reject if an error occurs
}
resolve(body); // resolve the parsed json object otherwise
});
});
req.on("error", error => reject(error)); // reject if the request fails too (if something went wrong before the request is sent for example)
});
async function getNumDraws(year) {
let result = 0;
for(let goal = 0; goal < 11; goal++) {
let data = await getJSONAsync(`https://jsonmock.hackerrank.com/api/football_matches?year=${year}&team1goals=${goal}&team2goals=${goal}`);
result += data.total;
}
return result;
}
Note: getJSONAsync is not specific to getNumDraws, you can use it somewhere else if you need it, and since it returns a Promise you can either await it like getNumDraws does or use it with then/catch blocks like so:
getJSONAsync("url")
.then(data => {
// data is the parsed json returned by the request
})
.catch(error => {
// the error message if something fails
})

Object "undefined" in Mocha script. Not sure how to pass on value

I am running a mocha test on a bunch of endpoints and I'm not able to pass on results to the next test case
Failing at line var result = data["results"];
error:data undefined
it('should get series', async () => {
let data = await legacy.getCategories(config[process.env.TEST_ENV].url, config[process.env.TEST_ENV].origin);
console.log(data);
expect(data).to.be.an('array').that.is.not.empty;
done();
});
var unallowedGuids = config[process.env.TEST_ENV].unallowedGuids;
var result = data["results"];
for (i = 0; i < result.length; i++) {
let guid = result[i]["guid"];
let showName = result[i]["title"];
if (!unallowedGuids.includes(guid)){
it('should get colletions guid', async () => {
let data = await legacy.collectionsGuid(config[process.env.TEST_ENV].url, config[process.env.TEST_ENV].origin, guid);
expect(data).to.be.an('array').that.is.not.empty;
done();
});
it('should get series guid', async () => {
let data = await legacy.seriesGuid(config[process.env.TEST_ENV].url, config[process.env.TEST_ENV].origin, guid);
expect(data).to.be.an('array').that.is.not.empty;
done();
});
}
else
console.log("excluded", guid);
}
});```

I think your issue is related to variable scope.
Try modifying your code like so:
let data; // initialize "data" outside of the "it" callback function
it('should get series', async () => {
data = await legacy.getCategories(config[process.env.TEST_ENV].url, config[process.env.TEST_ENV].origin);
console.log(data);
expect(data).to.be.an('array').that.is.not.empty;
done();
});
var unallowedGuids = config[process.env.TEST_ENV].unallowedGuids;
var result = data["results"]; // data will be defined here

For your case, you got undefined because var result = data["results"]; is executed first before your it('should get series', ..) function.
One solution is to restructure the test to make sure the order of execution is correct. We can use before() or beforeEach() For example:
let data;
before(async () => {
data = await legacy.getCategories(config[process.env.TEST_ENV].url, config[process.env.TEST_ENV].origin);
});
it('get collection and series guid', async function() {
var unallowedGuids = config[process.env.TEST_ENV].unallowedGuids;
var result = data["results"]; // it will be executed after data is fetched in `before()`
for (i = 0; i < result.length; i++) {
let guid = result[i]["guid"];
let showName = result[i]["title"];
if (!unallowedGuids.includes(guid)){
let data = await legacy.collectionsGuid(config[process.env.TEST_ENV].url, config[process.env.TEST_ENV].origin, guid);
expect(data).to.be.an('array').that.is.not.empty;
let data = await legacy.seriesGuid(config[process.env.TEST_ENV].url, config[process.env.TEST_ENV].origin, guid);
expect(data).to.be.an('array').that.is.not.empty;
}
else {
console.log("excluded", guid);
}
}
});
Alternatively, you also can put the data fetching in same it() function
// no before()
it('get collection and series guid', async function() {
var unallowedGuids = config[process.env.TEST_ENV].unallowedGuids;
var data = await legacy.getCategories(config[process.env.TEST_ENV].url, config[process.env.TEST_ENV].origin);
var result = data["results"];
....
});
Hope it helps

Make a blocking call to a function in Node.js required in this case?

I am starting to learn node.js. I am stuck with a problem here. I am calling a weather service which returns a JSON(url below).
http://api.wunderground.com/api/Your_key/conditions/q/CA/San_Francisco.json
I want to display the results from the api on an HTML page.
I have written the following code(getInfo module) to retrieve and return the JSON.
var fbResponse;
module.exports = function (url) {
var http=require('http');
http.get(url, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
fbResponse = JSON.parse(body);
console.log("Got response: ", fbResponse.response);
});
}).on('error', function(e) {
console.log("Got error: ", e);
});
return fbResponse;
};
So to use this module I created a test.js file as follows:
var relay = require('./getInfo');
var url = 'http://api.wunderground.com/api/Your_key/conditions/q/CA/San_Francisco.json';
var response;
var x=relay(url);
console.log (x);
Output is as follows:
undefined // console.log(x) from test.js
Got response: { version: '0.1',
termsofService: 'http://www.wunderground.com/weather/api/d/terms.html',
features: { conditions: 1 } }
The console output in the test code runs first with no data in it. The HTTP get competes later and displays the actual output I need.
How can I modify the test code to make a blocking call such that var x in the test code actually have the JSON output instead of undefined?
Can I achieve the desired result without a blocking call to the getInfo module?

As you know, node is asynchronous, so the callback of http.get and res.on('end', .. will fire after relay function is executed and it is returned. So normally you can't return the result from it.
You have a couple of choices:
Pass a callback to relay and use that:
module.exports = function (url, cb) {
http.get(url, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
cb(null, JSON.parse(body));
});
}).on('error', cb);
};
Then use it like this:
var relay = require('./getInfo');
relay(url, function (err, x) {
if (err) {
console.error('oohh i got a error: ', err)
}
console.log('oohh i got a response: ', x)
});
Use promises. This is almost same as passing callbacks. A little less lightweight, but when combining different asynchronous operations, you will understand how awesome they are. For just one asynchronous call there might not be any difference. Here I use q. You can also use bluebird which is way more performant but lacks some of the sugar of q. You can read this article to understand why promises are cleaner than callbacks in some cases.
module.exports = function (url) {
var deferred = Q.defer();
http.get(url, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
deferred.resolve(JSON.parse(body));
});
}).on('error', function(e) {
deferred.reject(e);
});
return deferred.promise;
};
var relay = require('./getInfo');
relay(url).then(function responseHandler(x) {
console.log('my awesome response')
}, function errorHandler(err) {
console.error('got an error', err);
});
Use generators. It is part of Ecmascript 6 specification it only exists in node v0.11.x and later. But it would be almost what you want.
With that past promise example we can do this:
Q.async(function *() {
var relay = require('./getInfo');
var x = yield relay(url);
console.log('my awesome response', x)
});
This is almost what you want. You can also achieve it using the callback solution with co library:
co(function *() {
var relay = require('./getInfo');
var x = yield relay.bind(null, url);
console.log('my awesome response', x);
});
You can also use node-fibers in above example which is almost a similar tool like generators.
If you want to use bleeding edge Javascript, you can use Async/Await instead of generators with promises.

You need to pass in a callback instead:
var http = require('http');
module.exports = function(url, cb) {
http.get(url, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
var resp, err;
try {
resp = JSON.parse(body);
} catch (ex) {
err = ex;
}
cb(err, resp);
});
}).on('error', function(e) {
console.log("Got error: ", e);
cb(e);
});
};
Then use it like:
var relay = require('./getInfo');
var url = 'http://api.wunderground.com/api/Your_key/conditions/q/CA/San_Francisco.json';
var response;
relay(url, function(err, resp) {
if (err) throw err; // TODO: handle better
console.dir(resp);
});

you can take a callback in your module function to return the result.
module.exports = function (url, onsuccess) {
...
res.on('end', function() {
fbResponse = JSON.parse(body);
if(onsuccess){
onsuccess(null, fbResponse);
}
Then in your caller code:
relay(url, function(err, result){
console.log(result);
});
Another option is to use httpsync module which provides synchronous apis for the same functionality that 'http' module provides. But in node js programming, you should always avoid synchronous calls.