I have problems with the synchronous and asynchronous functions on NodeJS, here is my problem,
I have 4 functions, a global variable (users), and a rendering function to have the html page.
the scraper1 and scraper2 function can be performed asynchronously, they scrape a website and fill in the global user variable, and the data_selector1 cannot be done unless scraper1 has done the same for data_selector2 with the scraper2 functions.
1-I need that scraper1 and scraper2 work asynchronously and fill the available users, and only render the HTML page if the 2 scrapers have finished their work.
2-I need an animation to be displayed in the browser during the drilling process, how can I do this
Here is what I tried ..
var express = require('express');
var request = require('request');
var cheerio = require('cheerio');
var fs = require('fs');
var router = express.Router();
/* globale variable to populate */
var users = {
'name':null,
'age':null,
};
//function of scraping link1
function scarper1(callback){
console.log("-------------scraper---------");
var url = 'link1';
request(
{
method: 'GET',
url: 'http://api.myscarperwebservice.com/?url=' + url,
headers: {
Accept: 'application/json',
},
},
function(error, response, body) {
if (error) throw error;
// call the data-selector1 after scraper are finish rendering
data_selector1(body);
}
);
}
//function of scraping link2
function scarper2(callback){
console.log("-------------scraper2---------");
var url = 'link2';
request(
{
method: 'GET',
url: 'http://api.myscarperwebservice.com/?url=' + url,
headers: {
Accept: 'application/json',
},
},
function(error, response, body) {
if (error) throw error;
// call the data-selector2 after scraper are finish rendering
data_selector2(body);
}
);
}
function data_selector1(body)
{
console.log("-------------data-selector---------");
const $ = cheerio.load(body);
$("div[class='.user']").each(function(i,elem){
users['name'] =$(elem).find('span[class=".name]').text();
users['age'] =$(elem).find('span[class=".age]').text();
});
}
function data_selector2(body)
{
console.log("-------------data-selector2---------");
const $ = cheerio.load(body);
$("ul[class='.user']").each(function(i,elem){
users['name'] =$(elem).find('li[class=".name]').text();
users['age'] =$(elem).find('li[class=".age]').text();
});
}
/* GET home page. */
router.post('/recherche', function(req, res, next) {
// i dont know how to make it here to say that scraper1 and scraper2 can be executed async and to render page after that the two scraper are finished
// and while scraper are working to display animation in the client
scarper1(function(results){
console.log(results);res.render('rechercher', { title: 'Express' });
});
});
You can use promise.all() to do that but before you can use it, you need to promisify your functions:
function scarper1(callback) {
return new Promise((resolve, reject) => {
console.log("-------------scraper---------");
var url = 'link1';
request(
{
method: 'GET',
url: 'http://api.myscarperwebservice.com/?url=' + url,
headers: {
Accept: 'application/json',
},
},
function (error, response, body) {
if (error) reject(error);
// call the data-selector1 after scraper are finish rendering
data_selector1(body);
resolve('Done successfully');
}
);
});
}
function scarper2(callback) {
return new Promise((resolve, reject) => {
console.log("-------------scraper2---------");
var url = 'link2';
request(
{
method: 'GET',
url: 'http://api.myscarperwebservice.com/?url=' + url,
headers: {
Accept: 'application/json',
},
},
function (error, response, body) {
if (error) reject(error);
// call the data-selector2 after scraper are finish rendering
data_selector2(body);
resolve('Done successfully');
}
);
});
}
let scarper1 = scarper1(function(results){
console.log(results);res.render('rechercher', { title: 'Express' });
});
let scarper2 = scarper2(function(results){
console.log(results);res.render('rechercher', { title: 'Express' });
});
Promise.all([scarper1, scarper2]).then(function(values) {
console.log(values);
});
For more about promise.all check this docs.
An even better approach is using async.eachLimit() to loop through requests (asynchronously) but first you need to install async package then merge both scraper functions:
const async = require("async");
let urls = [
'link1',
'link2'
]
async.eachLimit(urls, 2, (url) => {
console.log("-------------scraper---------");
request(
{
method: 'GET',
url: 'http://api.myscarperwebservice.com/?url=' + url,
headers: {
Accept: 'application/json',
},
},
function (error, response, body) {
if (error) reject(error);
// call the data-selector1 after scraper are finish rendering
if(url == 'link1')
data_selector1(body);
else
data_selector2(body);
resolve('Done successfully');
}
);
}, (err) => {
console.log("Finished all urls")
});
Related
I have an array of ssn number and I have two api list in which I need to pass ssn number as request json so I need to call both api inside ssn loop so I pass ssn to json request during call both api but code is not work properly both api call at a time simulteniously, Where I need to call both api one by one.
Both API details and code are as follow
My Code:
let ssn = [460458524, 637625452, 453311896, 635285187, 455791630, 642348377, 463590491, 450730278, 641201851, 379965491];
async function getCRCDetails() {
ssn.forEach(function (item) {
if(item){
let CBCOptions = {
'method': 'POST',
'url': 'https://loanboard.houstondirectauto.com/api/Report',
'headers': {
'Content-Type': 'application/json',
'Cookie': 'ci_session=udmojmlc5tfl3epbrmtvgu6nao2f031p'
},
body: JSON.stringify({
"token": loantoken,
"action": "CBCReport",
"variables": {
ssn: item
}
})
}
request(CBCOptions, function (error, response) {
console.log(item);
console.log("CBCOPtion ", CBCOptions);
if (error) throw new Error(error);
result = (JSON.parse(response.body));
console.log("Result =", result);
CRCReport.push(result);
})
let EmployerInfoOptions = {
'method': 'POST',
'url': 'https://loanboard.houstondirectauto.com/api/Report',
'headers': {
'Content-Type': 'application/json',
'Cookie': 'ci_session=udmojmlc5tfl3epbrmtvgu6nao2f031p'
},
body: JSON.stringify({
"token": loantoken,
"action": "getEmployerInfo",
"variables": {
ssn: item
}
})
}
request(EmployerInfoOptions, function (error, response) {
console.log(response.body);
})
}
Here I need to call API request one by one.Anyone Guide me please.
I prefer use async await method for this situation
you need install and require async and request-promise
after that :
const request = require("request-promise");
const async = require("async");
let ssn = [460458524, 637625452, 453311896, 635285187, 455791630, 642348377, 463590491, 450730278, 641201851, 379965491];
async function getCRCDetails() {
//like a forEache
async.eachSeries(ssn, async (item) => {
let CBCOptions = {
method: "POST",
url: "https://loanboard.houstondirectauto.com/api/Report",
headers: {
"Content-Type": "application/json",
Cookie: "ci_session=udmojmlc5tfl3epbrmtvgu6nao2f031p",
},
body: JSON.stringify({
token: loantoken,
action: "CBCReport",
variables: {
ssn: item,
},
}),
};
let EmployerInfoOptions = {
method: "POST",
url: "https://loanboard.houstondirectauto.com/api/Report",
headers: {
"Content-Type": "application/json",
Cookie: "ci_session=udmojmlc5tfl3epbrmtvgu6nao2f031p",
},
body: JSON.stringify({
token: loantoken,
action: "getEmployerInfo",
variables: {
ssn: item,
},
}),
};
try {
let resultCBCOptions = await request(CBCOptions);
let EmployerInfoOptions = await request(EmployerInfoOptions);
console.log(resultCBCOptions)
console.log(EmployerInfoOptions)
//do pushing resultCBCOptions
//do pushing EmployerInfoOptions
} catch (error) {
console.log(error);
}
},
() => {
console.log("finished");
}
);
}
In Node the request methods that you are using are asynchronous. Meaning the runner (server) that runs the code does not wait for the request to finish and just continues to execute the next lines.
One thing that you can do is,
request.post(params).on("response", function(response) {
//....Do stuff with your data you recieve
// Make the next call here
request.post(params).on("response"), function() {
// Here you will have the second call's results
}
})
This ensures that both the API calls happen in order and only after the first one finishes.
Note:
The request library that you are using has been deprecated back in 2020. See https://github.com/request/request Hence, I would suggest you use other libraries like the standard https or http library that is shipped with node or you can use axios.
If you use a forEach loop without awaiting the results, you'll execute them all at the same time. Moreover, request library is kind of old and you need to convert its functions to return a promise.
Here's how I would do it.
const ssn = [1,2,3,4];
function download(item) {
return new Promise(function(resolve, reject) {
let options = {}; // construct your request
request(options, function (error, response) {
if(error) {
return reject(error);
}
resolve(response);
})
}
}
ssn = ssn.map(async function(item) {
let res = await download(item);
// process the result
return res;
});
You can also use the Bluebird library or get another client library such as got or axios.
I have several API Get request at once in nodejs. Each API have new data every couple minutes.
var express = require('express');
var router = express.Router();
var request = require("request");
let value1, value2, bodyData1, bodyData2;
var options = { method: 'GET',
url: 'https://api.example.com/data1',
qs:
{
valueType: 'MAXIMUM'
},
headers:
{
authorization: 'ABC123456',
accept: 'application/json; charset=utf-8' } };
request(options, function (error, response, body) {
if (error) throw new Error(error);
bodyData1 = JSON.parse(body);
value1 = bodyData1.value;
});
var options = { method: 'GET',
url: 'https://api.example.com/data2',
qs:
{
valueType: 'MAXIMUM'
},
headers:
{
authorization: 'ABC123456',
accept: 'application/json; charset=utf-8' } };
request(options, function (error, response, body) {
if (error) throw new Error(error);
bodyData2 = JSON.parse(body);
value2 = bodyData2.value;
});
router.get('/', function(req, res, next) {
res.render('home', {valueA : value1, valueB: value2});
});
module.exports = router;
I want to know if it is possible to combine them into one function?
Any other things I should concern?
It is possible if you have promises which is currently not the case. You have to wrap your request() call in a Promise. You can do it manually with a custom function requestToPromise.
You can then use Promise.all to call multiple promises in parallel.
function requestToPromise(options) {
return new Promise((resolve, reject) => {
request(options, (error, response, body) => {
if (error) return reject(error);
resolve(body);
});
});
}
var optionsRequest1 = {
method: "GET",
url: "https://api.example.com/data1",
qs: {
valueType: "MAXIMUM"
},
headers: {
authorization: "ABC123456",
accept: "application/json; charset=utf-8"
}
};
var optionsRequest2 = {
method: "GET",
url: "https://api.example.com/data2",
qs: {
valueType: "MAXIMUM"
},
headers: {
authorization: "ABC123456",
accept: "application/json; charset=utf-8"
}
};
var requestPromise1 = requestToPromise(optionsRequest1);
var requestPromise2 = requestToPromise(optionsRequest2);
Promise.all([requestPromise1, requestPromise2]).then(results => {
var [resultPromise1, resultPromise2] = results;
}).catch(error => {
//handle error
});
Instead of using the custom function requestToPromise you can also use util.promisify
const util = require('util');
const requestAsync = util.promisify(request);
Promise.all([requestAsync(optionsRequest1), requestAsync(optionsRequest2)]).then(results => {
var [resultPromise1, resultPromise2] = results;
}).catch(error => {
//handle error
});
You can use Redis cache to store data in memory for fast retrieval and fetch from memory very quickly.
Also, after some interval, you can add them to a database through bulk creation. It will decrease your database call.
// Example in sequilize
await db.table_name.bulkcreate([ {0bj1}, {obj2}..,{obj3 } ]);
I'm trying to create a firebase function that makes a HTTP POST request whenever a new document is created.
This is my code:
import * as functions from 'firebase-functions';
const admin = require('firebase-admin');
const request = require("request");
exports.sendMessage = functions.firestore.document('comms/{comms}').onCreate((snap, context) => {
const newValue = snap.data();
if (newValue) {
//const email = newValue.email;
const msg = newValue.msg;
return request({
uri: "url",
method: 'POST',
body: msg,
json: true,
resolveWithFullResponse: true
}).then((response: { statusCode: number; }) => {
if (response.statusCode >= 400) {
throw new Error(`HTTP Error: ${response.statusCode}`);
}
console.log('SUCCESS! Posted', msg);
});
}
return Promise
});
Error received:
TypeError: request(...).then is not a function
at exports.sendMessage.functions.firestore.document.onCreate (/srv/lib/index.js:25:12)
at cloudFunction (/srv/node_modules/firebase-functions/lib/cloud-functions.js:127:23)
at /worker/worker.js:825:24
at
at process._tickDomainCallback (internal/process/next_tick.js:229:7)
request supports callback interfaces natively but does not return a promise, which is what you must do within a Cloud Function.
This is explained in the official Firebase video series here: https://firebase.google.com/docs/functions/video-series/. In particular watch the three videos titled "Learn JavaScript Promises" (Parts 2 & 3 especially focus on background triggered Cloud Functions, but it really worth watching Part 1 before).
You could use request-promise (https://github.com/request/request-promise) and the rp() method which "returns a regular Promises/A+ compliant promise". You would then adapt your code as follows:
import * as functions from 'firebase-functions';
const admin = require('firebase-admin');
const rp = require('request-promise');
exports.sendMessage = functions.firestore.document('comms/{comms}').onCreate((snap, context) => {
const newValue = snap.data();
if (newValue) {
const msg = newValue.msg;
var options = {
method: 'POST',
uri: '....',
body: msg,
json: true // Automatically stringifies the body to JSON
};
return rp(options)
.then(parsedBody => {
// POST succeeded...
console.log('SUCCESS! Posted', msg);
return null;
})
.catch(err => {
// POST failed...
console.log(err);
return null;
});
} else {
return null;
}
});
request module doesn't return a Promise instead try using a callback function for response.
return request({
uri: "url",
method: 'POST',
body: msg,
json: true,
resolveWithFullResponse: true
}, function (error, response, body) {
})
As in the documentation already mention you need to pass the callback to your request
var request = require('request');
request('http://www.google.com', function (error, response, body) {
console.log('error:', error); // Print the error if one occurred
console.log('statusCode:', response && response.statusCode); // Print the response status code if a response was received
console.log('body:', body); // Print the HTML for the Google homepage.
});
If you want to chain your request you can use pipe
request
.get('url/img.png')
.on('response', function(response) {
console.log(response.statusCode) // 200
console.log(response.headers['content-type']) // 'image/png'
})
.pipe(request.put('url'))
If you want to use promise you can use request-promise
var rp = require('request-promise');
rp('http://www.google.com')
.then(function (htmlString) {
// Process html...
})
.catch(function (err) {
// Crawling failed...
});
The request module work on callbacks only, If you want to make Promisify you need to do like this
const request = require('request');
const webService = {};
webService.callApi = (url, bodyObj, method) => {
return new Promise((resolve, reject) => {
const options = {
method: method || 'POST',
url: url,
headers: {
'Content-Type': 'application/json',
},
body: bodyObj,
json: true,
};
// Error Handler
const errorMessge = { code: 500, error: 'INTERNAL_SERVER_ERROR' };
request(options, (error, response, resBody) => {
if (error) {
return reject(errorMessge);
} else if (response.statusCode !== 200) {
return reject(errorMessge);
}
return resolve(resBody);
});
});
};
module.exports = webService;
I am trying to invoke services in node express in a loop. But the problem is that before all the services are invoked, the remaining code is getting executed.
I tried some options with Promise, async/await but they didn't work.
Basically I need to invoke the service in a synchronous way.
I have created 2 mock services in JSON stub. In the first service response, I will get an array of vehicles. Once I got this, I need to update one more value in each array by calling another service.
Here the problem I faced is that the 2nd service is not called synchronously.
const express = require('express');
const request = require("request");
const router = express.Router();
router.get('/make', (req, res) => {
var options = {
headers: {
'Content-Type': 'application/json',
'JsonStub-User-Key': 'ddc159a0-5aa8-4a38-a0f1-913e4d768b56',
'JsonStub-Project-Key': '34ba28a9-471c-435d-ab61-b7732c9583c6'
},
method: "GET",
json: true,
strictSSL : false,
url: `http://jsonstub.com/vehicle/make`
};
request(options, function(error, response, body) {
if (body){
checkModelType(body);
res.status(200).json(body).end();
} else {
console.log("REST Request timeout: ", JSON.stringify(error));
res.status(400).json('Error').end();
}
});
});
function checkModelType(response){
let vehicleList = response.vehicleList;
console.log("--->"+vehicleList.length);
for(var i = 0;i<vehicleList.length;++i){
const modelType = findModel();
vehicleList[i].modelType = modelType;
}
console.log("Updated Vehicle List:"+JSON.stringify(vehicleList));
}
const findModel = () =>{
var options = {
headers: {
'Content-Type': 'application/json',
'JsonStub-User-Key': 'ddc159a0-5aa8-4a38-a0f1-913e4d768b56',
'JsonStub-Project-Key': '34ba28a9-471c-435d-ab61-b7732c9583c6'
},
method: "GET",
json: true,
strictSSL : false,
url: `http://jsonstub.com/vehicle/details`
};
request(options, function(error, response, body) {
if (body){
console.log("Model Type-->"+body.output.modelType);
return body.output.modelType;
} else {
console.log("REST Request timeout: ", JSON.stringify(error));
}
});
}
module.exports = router;
Response :
-----------
PS F:\workSpace_Node\TestApp> node app.js
server running at 9086
--->4
Updated Vehicle List:[{"make":"Audi","model":"A3","vin":"QVFCFQT7894563214"},{"make":"Audi","model":"A4","vin":"ASECFQT7894563214"},{"make":"Audi","model":"Q5","vin":"QWECFQT7894993214"}]
Model Type-->SD
Model Type-->SD
Model Type-->SD
Expected result :
[{"make":"Audi","model":"A3","modelType":"SD", "vin":"QVFCFQT7894563214"},{"make":"Audi","model":"A4","modelType":"SD","vin":"ASECFQT7894563214"}]
You can switch to request-promise library instead of request and then go with some async\await:
const express = require('express');
const request = require('request-promise'); // switched library
const router = express.Router();
router.get('/make', async (req, res) => {
var options = {
headers: {
'Content-Type': 'application/json',
'JsonStub-User-Key': 'ddc159a0-5aa8-4a38-a0f1-913e4d768b56',
'JsonStub-Project-Key': '34ba28a9-471c-435d-ab61-b7732c9583c6'
},
method: "GET",
json: true,
strictSSL : false,
url: `http://jsonstub.com/vehicle/make`
};
const body = await request(options);
if (body) {
await checkModelType(body);
res.status(200).json(body).end();
} else {
console.log("REST Request timeout: ", JSON.stringify(error));
res.status(400).json('Error').end();
}
});
async function checkModelType(response){
let vehicleList = response.vehicleList;
console.log("--->"+vehicleList.length);
for(var i = 0;i<vehicleList.length;++i){
const modelType = await findModel();
vehicleList[i].modelType = modelType;
}
console.log("Updated Vehicle List:"+JSON.stringify(vehicleList));
}
const findModel = async () =>{
var options = {
headers: {
'Content-Type': 'application/json',
'JsonStub-User-Key': 'ddc159a0-5aa8-4a38-a0f1-913e4d768b56',
'JsonStub-Project-Key': '34ba28a9-471c-435d-ab61-b7732c9583c6'
},
method: "GET",
json: true,
strictSSL : false,
url: `http://jsonstub.com/vehicle/details`
};
const body = await request(options);
if (body){
console.log("Model Type-->"+body.output.modelType);
return body.output.modelType;
} else {
console.log("REST Request timeout: ", JSON.stringify(error));
}
}
module.exports = router;
And it will change the order of operations:
--->4
Model Type-->SD
Model Type-->SD
Model Type-->SD
Model Type-->SD
Updated Vehicle List:[{"make":"Audi","model":"A3","vin":"QVFCFQT7894563214","modelType":"SD"},{"make":"Audi","model":"A4","vin":"ASECFQT7894563214","modelType":"SD"},{"make":"Audi","model":"Q7","modelType":"SD"},{"make":"Audi","model":"Q5","vin":"QWECFQT7894993214","modelType":"SD"}]
I am new to the working with TAPE JS for testing. I have it all setup and working, and it works fine with regular tests. But I am trying to test a unique REST API based product that relies on certain calls to have been made before the next call has the information needed to have a successful call.
So here are the first two calls I am trying to get working:
var SessionId;
test('beginIqsSession', function (assert) {
assert.plan(1);
var requestData = {"ProductDataArray":{"Feid":"GIQNY","AltData":"SaneID:null","Debug":"false","PageId":"1.1"}};
request({
url: 'http://192.168.99.100/Iqs/api.php/beginIqsSession',
method: "POST",
json: requestData
}, function(error, response, json){
if(json.responseDataPayload.SessionId)
{
SessionId = json.responseDataPayload.SessionId;
assert.equal(1,1);
}
});
assert.end();
});
test('validateAddress', function (assert) {
assert.plan(2);
console.log("Retrieving validateAddress response");
var requestData = {"SessionId":SessionId,"ValidateAddressDataArray":{"PropertyStreetNumber":"20671","PropertyStreetName":"mountain vista dr","PropertyCity":"anchorage","PropertyState":"AK","PropertyZipCode":"99577"}};
console.log(SessionId);
request({
url: 'http://192.168.99.100/Iqs/api.php/validateAddress',
method: "POST",
json: requestData
}, function (error, response, body) {
if (!error) {
console.log(body);
}
else {
console.log("error: " + error)
}
});
assert.end();
});
So basically in the code above, I am trying to test beginIqsSession, wait for its response, and store a piece of data from that response that future calls require to be sent in.
in validateAddress you'll see I am trying to pass SessionId in which was returned in the previous call, but because this test is being run at the same time as the previous test, this variable is still empty. How can I make the second call, and all future calls, to wait for the previous call to run?
assert.plan apparently doesn't work in this way.
You could use the Promise API
var SessionId;
let p1 = new Promise((resolve, reject) => {
test('beginIqsSession', function (assert) {
assert.plan(1);
var requestData = {"ProductDataArray":{"Feid":"GIQNY","AltData":"SaneID:null","Debug":"false","PageId":"1.1"}};
request({
url: 'http://192.168.99.100/Iqs/api.php/beginIqsSession',
method: "POST",
json: requestData
}, function(error, response, json){
if(json.responseDataPayload.SessionId)
{
SessionId = json.responseDataPayload.SessionId;
assert.equal(1,1);
resolve(SessionId);
}
});
assert.end();
});
})
p1.then((SessionId) => {
test('validateAddress', function (assert) {
assert.plan(2);
console.log("Retrieving validateAddress response");
var requestData = {"SessionId":SessionId,"ValidateAddressDataArray":{"PropertyStreetNumber":"20671","PropertyStreetName":"mountain vista dr","PropertyCity":"anchorage","PropertyState":"AK","PropertyZipCode":"99577"}};
console.log(SessionId);
request({
url: 'http://192.168.99.100/Iqs/api.php/validateAddress',
method: "POST",
json: requestData
}, function (error, response, body) {
if (!error) {
console.log(body);
}
else {
console.log("error: " + error)
}
});
assert.end();
});
});