I have several API Get request at once in nodejs. Each API have new data every couple minutes.
var express = require('express');
var router = express.Router();
var request = require("request");
let value1, value2, bodyData1, bodyData2;
var options = { method: 'GET',
url: 'https://api.example.com/data1',
qs:
{
valueType: 'MAXIMUM'
},
headers:
{
authorization: 'ABC123456',
accept: 'application/json; charset=utf-8' } };
request(options, function (error, response, body) {
if (error) throw new Error(error);
bodyData1 = JSON.parse(body);
value1 = bodyData1.value;
});
var options = { method: 'GET',
url: 'https://api.example.com/data2',
qs:
{
valueType: 'MAXIMUM'
},
headers:
{
authorization: 'ABC123456',
accept: 'application/json; charset=utf-8' } };
request(options, function (error, response, body) {
if (error) throw new Error(error);
bodyData2 = JSON.parse(body);
value2 = bodyData2.value;
});
router.get('/', function(req, res, next) {
res.render('home', {valueA : value1, valueB: value2});
});
module.exports = router;
I want to know if it is possible to combine them into one function?
Any other things I should concern?
It is possible if you have promises which is currently not the case. You have to wrap your request() call in a Promise. You can do it manually with a custom function requestToPromise.
You can then use Promise.all to call multiple promises in parallel.
function requestToPromise(options) {
return new Promise((resolve, reject) => {
request(options, (error, response, body) => {
if (error) return reject(error);
resolve(body);
});
});
}
var optionsRequest1 = {
method: "GET",
url: "https://api.example.com/data1",
qs: {
valueType: "MAXIMUM"
},
headers: {
authorization: "ABC123456",
accept: "application/json; charset=utf-8"
}
};
var optionsRequest2 = {
method: "GET",
url: "https://api.example.com/data2",
qs: {
valueType: "MAXIMUM"
},
headers: {
authorization: "ABC123456",
accept: "application/json; charset=utf-8"
}
};
var requestPromise1 = requestToPromise(optionsRequest1);
var requestPromise2 = requestToPromise(optionsRequest2);
Promise.all([requestPromise1, requestPromise2]).then(results => {
var [resultPromise1, resultPromise2] = results;
}).catch(error => {
//handle error
});
Instead of using the custom function requestToPromise you can also use util.promisify
const util = require('util');
const requestAsync = util.promisify(request);
Promise.all([requestAsync(optionsRequest1), requestAsync(optionsRequest2)]).then(results => {
var [resultPromise1, resultPromise2] = results;
}).catch(error => {
//handle error
});
You can use Redis cache to store data in memory for fast retrieval and fetch from memory very quickly.
Also, after some interval, you can add them to a database through bulk creation. It will decrease your database call.
// Example in sequilize
await db.table_name.bulkcreate([ {0bj1}, {obj2}..,{obj3 } ]);
Related
I have some code that looks like this in Next JS
const resApp = await fetch('/api/club/createsignalapp', {
body: JSON.stringify({
name: event.target.name.value,
}),
headers: {
'Content-Type': 'application/json',
},
method: 'POST',
})
const appResult = await resApp.json()
--createsignalapp
export default async (req, res) => {
var mydata
var createApp = function (data) {
var headers = {
'Content-Type': 'application/json; charset=utf-8',
Authorization: `Basic APIKKEY`,
}
var options = {
host: 'onesignal.com',
port: 443,
path: '/api/v1/apps',
method: 'POST',
headers: headers,
}
var https = require('https')
var myreq = https.request(options, function (myres) {
myres.on('data', function (data) {
mydata = JSON.parse(data)
res.statusCode = 200
res.json({
response: {
boolean: true,
alert: '',
message: '',
},
data: mydata
})
})
})
myreq.on('error', function (e) {
console.log(e)
})
myreq.write(JSON.stringify(data))
myreq.end()
}
var message = {
name: req.body.name
}
createApp(message)
}
This hits the error API resolved without sending a response for /api/club/createsignalapp, this may result in stalled requests.
I'm not sure how to do this correctly as i'm getting confused with the awaits and asyncs and requests everywhere.
Happy to hear suggestions.
Thanks
I have problems with the synchronous and asynchronous functions on NodeJS, here is my problem,
I have 4 functions, a global variable (users), and a rendering function to have the html page.
the scraper1 and scraper2 function can be performed asynchronously, they scrape a website and fill in the global user variable, and the data_selector1 cannot be done unless scraper1 has done the same for data_selector2 with the scraper2 functions.
1-I need that scraper1 and scraper2 work asynchronously and fill the available users, and only render the HTML page if the 2 scrapers have finished their work.
2-I need an animation to be displayed in the browser during the drilling process, how can I do this
Here is what I tried ..
var express = require('express');
var request = require('request');
var cheerio = require('cheerio');
var fs = require('fs');
var router = express.Router();
/* globale variable to populate */
var users = {
'name':null,
'age':null,
};
//function of scraping link1
function scarper1(callback){
console.log("-------------scraper---------");
var url = 'link1';
request(
{
method: 'GET',
url: 'http://api.myscarperwebservice.com/?url=' + url,
headers: {
Accept: 'application/json',
},
},
function(error, response, body) {
if (error) throw error;
// call the data-selector1 after scraper are finish rendering
data_selector1(body);
}
);
}
//function of scraping link2
function scarper2(callback){
console.log("-------------scraper2---------");
var url = 'link2';
request(
{
method: 'GET',
url: 'http://api.myscarperwebservice.com/?url=' + url,
headers: {
Accept: 'application/json',
},
},
function(error, response, body) {
if (error) throw error;
// call the data-selector2 after scraper are finish rendering
data_selector2(body);
}
);
}
function data_selector1(body)
{
console.log("-------------data-selector---------");
const $ = cheerio.load(body);
$("div[class='.user']").each(function(i,elem){
users['name'] =$(elem).find('span[class=".name]').text();
users['age'] =$(elem).find('span[class=".age]').text();
});
}
function data_selector2(body)
{
console.log("-------------data-selector2---------");
const $ = cheerio.load(body);
$("ul[class='.user']").each(function(i,elem){
users['name'] =$(elem).find('li[class=".name]').text();
users['age'] =$(elem).find('li[class=".age]').text();
});
}
/* GET home page. */
router.post('/recherche', function(req, res, next) {
// i dont know how to make it here to say that scraper1 and scraper2 can be executed async and to render page after that the two scraper are finished
// and while scraper are working to display animation in the client
scarper1(function(results){
console.log(results);res.render('rechercher', { title: 'Express' });
});
});
You can use promise.all() to do that but before you can use it, you need to promisify your functions:
function scarper1(callback) {
return new Promise((resolve, reject) => {
console.log("-------------scraper---------");
var url = 'link1';
request(
{
method: 'GET',
url: 'http://api.myscarperwebservice.com/?url=' + url,
headers: {
Accept: 'application/json',
},
},
function (error, response, body) {
if (error) reject(error);
// call the data-selector1 after scraper are finish rendering
data_selector1(body);
resolve('Done successfully');
}
);
});
}
function scarper2(callback) {
return new Promise((resolve, reject) => {
console.log("-------------scraper2---------");
var url = 'link2';
request(
{
method: 'GET',
url: 'http://api.myscarperwebservice.com/?url=' + url,
headers: {
Accept: 'application/json',
},
},
function (error, response, body) {
if (error) reject(error);
// call the data-selector2 after scraper are finish rendering
data_selector2(body);
resolve('Done successfully');
}
);
});
}
let scarper1 = scarper1(function(results){
console.log(results);res.render('rechercher', { title: 'Express' });
});
let scarper2 = scarper2(function(results){
console.log(results);res.render('rechercher', { title: 'Express' });
});
Promise.all([scarper1, scarper2]).then(function(values) {
console.log(values);
});
For more about promise.all check this docs.
An even better approach is using async.eachLimit() to loop through requests (asynchronously) but first you need to install async package then merge both scraper functions:
const async = require("async");
let urls = [
'link1',
'link2'
]
async.eachLimit(urls, 2, (url) => {
console.log("-------------scraper---------");
request(
{
method: 'GET',
url: 'http://api.myscarperwebservice.com/?url=' + url,
headers: {
Accept: 'application/json',
},
},
function (error, response, body) {
if (error) reject(error);
// call the data-selector1 after scraper are finish rendering
if(url == 'link1')
data_selector1(body);
else
data_selector2(body);
resolve('Done successfully');
}
);
}, (err) => {
console.log("Finished all urls")
});
I would like to know how to send form data with headers to request module.
I have function getToken which makes post request with headers and formdata, and then response data will again make a request to new url and return data.
Currenlty returning
{ type: 'http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html',
title: 'Unsupported Media Type',
status: 415,
detail: 'Invalid content-type specified' }.
Apologies since not able to provide the apiurl and key
//app.js
var express = require('express');
var router = express.Router();
var helper= require('../help.js');
router.get('/', function (req, res) {
helper.getToken(req.originalUrl).then(token=>{
helper.getData(token).then(data=>{
res.send({api:data})
})
})
})
//helper.js
module.exports.getToken= function (rquery) {
return new Promise(async function (resolve, reject) {
try {
const form_data = {
grant_type: 'auth_token',
auth_token: ''//apikey,
}
var headers = {
"Content-Type": "multipart/form-data",
};
url = ``//apiurl1;
request.post({ url: url, form: form_data, headers: headers }, (e, r, body) => {
if (!e) {
resolve(JSON.parse(body));
}
else {
resolve(e);
}
});
}
catch (err) {
reject(err);
}
})
}
module.exports.getData= function (token) {
return new Promise(async function (resolve, reject) {
try {
const form_data = {
grant_type: 'refresh_token',
refresh_token: token.refresh_token,
}
var headers = {
"Content-Type": "multipart/form-data",
"Authorization":"Bearer "+token.access_token
};
url = ``//apiurl2;
request.post({ url: url, form: form_data, headers: headers }, (e, r, body) => {
if (!e) {
console.log(JSON.parse(body));// returns errors 415
resolve(JSON.parse(body));
}
else {
resolve(e);
}
});
}
catch (err) {
reject(err);
}
})
}
It seems that the API you are posting to does not support multipart/form-data. Can you confirm the Content-Type on the API?
EDIT: Found this in npmjs page of request module: https://www.npmjs.com/package/request#multipartform-data-multipart-form-uploads
The key that you are sending is "form". Try sending "formData" instead
I am trying to invoke services in node express in a loop. But the problem is that before all the services are invoked, the remaining code is getting executed.
I tried some options with Promise, async/await but they didn't work.
Basically I need to invoke the service in a synchronous way.
I have created 2 mock services in JSON stub. In the first service response, I will get an array of vehicles. Once I got this, I need to update one more value in each array by calling another service.
Here the problem I faced is that the 2nd service is not called synchronously.
const express = require('express');
const request = require("request");
const router = express.Router();
router.get('/make', (req, res) => {
var options = {
headers: {
'Content-Type': 'application/json',
'JsonStub-User-Key': 'ddc159a0-5aa8-4a38-a0f1-913e4d768b56',
'JsonStub-Project-Key': '34ba28a9-471c-435d-ab61-b7732c9583c6'
},
method: "GET",
json: true,
strictSSL : false,
url: `http://jsonstub.com/vehicle/make`
};
request(options, function(error, response, body) {
if (body){
checkModelType(body);
res.status(200).json(body).end();
} else {
console.log("REST Request timeout: ", JSON.stringify(error));
res.status(400).json('Error').end();
}
});
});
function checkModelType(response){
let vehicleList = response.vehicleList;
console.log("--->"+vehicleList.length);
for(var i = 0;i<vehicleList.length;++i){
const modelType = findModel();
vehicleList[i].modelType = modelType;
}
console.log("Updated Vehicle List:"+JSON.stringify(vehicleList));
}
const findModel = () =>{
var options = {
headers: {
'Content-Type': 'application/json',
'JsonStub-User-Key': 'ddc159a0-5aa8-4a38-a0f1-913e4d768b56',
'JsonStub-Project-Key': '34ba28a9-471c-435d-ab61-b7732c9583c6'
},
method: "GET",
json: true,
strictSSL : false,
url: `http://jsonstub.com/vehicle/details`
};
request(options, function(error, response, body) {
if (body){
console.log("Model Type-->"+body.output.modelType);
return body.output.modelType;
} else {
console.log("REST Request timeout: ", JSON.stringify(error));
}
});
}
module.exports = router;
Response :
-----------
PS F:\workSpace_Node\TestApp> node app.js
server running at 9086
--->4
Updated Vehicle List:[{"make":"Audi","model":"A3","vin":"QVFCFQT7894563214"},{"make":"Audi","model":"A4","vin":"ASECFQT7894563214"},{"make":"Audi","model":"Q5","vin":"QWECFQT7894993214"}]
Model Type-->SD
Model Type-->SD
Model Type-->SD
Expected result :
[{"make":"Audi","model":"A3","modelType":"SD", "vin":"QVFCFQT7894563214"},{"make":"Audi","model":"A4","modelType":"SD","vin":"ASECFQT7894563214"}]
You can switch to request-promise library instead of request and then go with some async\await:
const express = require('express');
const request = require('request-promise'); // switched library
const router = express.Router();
router.get('/make', async (req, res) => {
var options = {
headers: {
'Content-Type': 'application/json',
'JsonStub-User-Key': 'ddc159a0-5aa8-4a38-a0f1-913e4d768b56',
'JsonStub-Project-Key': '34ba28a9-471c-435d-ab61-b7732c9583c6'
},
method: "GET",
json: true,
strictSSL : false,
url: `http://jsonstub.com/vehicle/make`
};
const body = await request(options);
if (body) {
await checkModelType(body);
res.status(200).json(body).end();
} else {
console.log("REST Request timeout: ", JSON.stringify(error));
res.status(400).json('Error').end();
}
});
async function checkModelType(response){
let vehicleList = response.vehicleList;
console.log("--->"+vehicleList.length);
for(var i = 0;i<vehicleList.length;++i){
const modelType = await findModel();
vehicleList[i].modelType = modelType;
}
console.log("Updated Vehicle List:"+JSON.stringify(vehicleList));
}
const findModel = async () =>{
var options = {
headers: {
'Content-Type': 'application/json',
'JsonStub-User-Key': 'ddc159a0-5aa8-4a38-a0f1-913e4d768b56',
'JsonStub-Project-Key': '34ba28a9-471c-435d-ab61-b7732c9583c6'
},
method: "GET",
json: true,
strictSSL : false,
url: `http://jsonstub.com/vehicle/details`
};
const body = await request(options);
if (body){
console.log("Model Type-->"+body.output.modelType);
return body.output.modelType;
} else {
console.log("REST Request timeout: ", JSON.stringify(error));
}
}
module.exports = router;
And it will change the order of operations:
--->4
Model Type-->SD
Model Type-->SD
Model Type-->SD
Model Type-->SD
Updated Vehicle List:[{"make":"Audi","model":"A3","vin":"QVFCFQT7894563214","modelType":"SD"},{"make":"Audi","model":"A4","vin":"ASECFQT7894563214","modelType":"SD"},{"make":"Audi","model":"Q7","modelType":"SD"},{"make":"Audi","model":"Q5","vin":"QWECFQT7894993214","modelType":"SD"}]
app.get('/profile/:id', function(req, res){
var options = { method: 'GET',
url: 'https://api.favoriot.com/v1/streams?max=1',
headers:
{ 'cache-control': 'no-cache',
'content-type': 'application/json',
'apikey': 'api key' } };
request(options, function (error, response, body) {
res.render('profile', {data:body});
console.log(body)
});
});
when I run code above, I get this data:
{"debugCode":null,"statusCode":200,"numFound":1,"results":[{"user_id":"xxx510","stream_created_at":"2019-03-05T16:13:01.982Z","stream_developer_id":"f8b8fcb9-6f3e-4138-8c6b-d0a7e8xxxxx#xxxx510","device_developer_id":"raspberryPIxx#xxx510","data":{"distance":"12.4","status":"1"}}]}
how can I make it only display status only?
AFAIK there is no issue with the code as such. Are you sure that you got the distance and status in the data field of body or is it the intended output? By try using their API playground by setting your API key on it. I have rewritten the code using ES6 standards by promisifying request module or you can use the request-promise-native.
function requestPromisified(options) {
return new Promise(function(resolve, reject) {
request(options, function(error, res, body) {
if (!error && res.statusCode == 200) {
resolve(body);
} else {
reject(error);
}
});
});
}
app.get("/profile/:id", async (req, res) => {
const options = {
method: "GET",
url: "https://api.favoriot.com/v1/streams?max=1",
headers: {
"cache-control": "no-cache",
"content-type": "application/json",
apikey: "api key"
}
};
try {
const body = await requestPromisified(options);
console.log(body);
res.render("profile", { data: body });
} catch (error) {
res.status(400).send('Unable to find a profile')
}
});
1) There is no middleware in this example... you're just making a call to get some data.
2) status is available in body.results[0].data.status so just use that instead of the entire body object