I have a node server that pulls info from a php server and sends it over where it needs to be through a callback. Once or twice ad day the server gets stuck in an infinite loop and I think it's because I'm not properly handling the connection between the the php and node server. I posted my authentication code below. Anyone have any ideas?
exports.authenticate = function(request, callback) {
var https = require('https');
var options = {
hostname: 'www.mysite.com',
port: 443,
path: '/site/chatauth?id=' + request.sessionID,
method: 'GET',
};
var req = https.request(options, function(res) {
//console.log("statusCode: ", res.statusCode);
// console.log("headers: ", res.headers);
res.on('data', function(d) {
// process.stdout.write(d);
});
});
req.end();
req.on('response', function (response) {
var data = '';
response.setEncoding('utf8');
response.on('data', function(chunk) {
data += chunk;
});
// console.log (request.sessionID);
response.on('end', function() {
try {
callback(JSON.parse(data));
} catch(e) {
callback();
console.log("authentication failed");
}
});
});
};
are you sure authenticate function? Your code is seems to be perfect. There is two thing you have to do
1.Make callback on request error, request timeout and request abort.
2.Get deep insight about your callback, In following code you call same function on throwing exception. Are you sure about this?. This could make possibly loop, if it is again reach "authentication" function.
try {
callback(JSON.parse(data));
} catch(e) {
callback();
console.log("authentication failed");
}
I can't find any errors in your code, but due to emitter/asynchronous way perhaps I didn't catch it completely.
I personally use npm module request (Simplified HTTP request client) w/o any problems so far.
You could easily give it a try:
var request = require('request');
request({
uri: 'https://www.mysite.com/site/chatauth?id=' + request.sessionID,
encoding: 'utf8',
timeout: 20000,
strictSSL: true
}, function (err, response, body) {
if (err || response.statusCode !== 200) {
// some error handling
callback('error');
return;
}
callback(null, JSON.parse(body));
});
Related
I am trying to develop a google cloud function that will make an external https GET request and return the response body to the client.
Flow:
client makes request to mockServer function
function makes GET request to example.com
function returns "results" from response body from example.com to client
exports.mockServer = (req, res) => {
'use strict';
var https = require('https');
var options = {
host: 'example.com',
path: '/path',
headers: {
'accept': 'application/json',
'X-API-Key': 'XXXX'
}
};
if (req.method == 'GET'){
https.get(options, function (res) {
var data = '';
res.on('data', function (chunk) {
data += chunk;
});
res.on('end', function () {
if (res.statusCode === 200) {
var res_body = JSON.parse(data);
var results = JSON.stringify(res_body.result)
console.log("results:"+results);
} else {
console.log('Status:', res.statusCode);
}
});
}).on('error', function (err) {
console.log('Error:', err);
});
} else {
console.log("Wrong Method");
}
res.end()
};
I am able to successfully log the results with console.log("results:"+results); but I cannot figure out how to get it returned to the client. I am still new to this and am learning, so thank you so much in advance to any help!
Posting #YouthDev's solution as an answer:
Thanks to #DougStevenson and #Deko in the comments, I switched to an axios library and it works like a charm. Thank you to both for pointing me in the correct direction. Below is the working axios code.
exports.mockServer = (req, res) => {
const axios = require('axios').create({
baseURL: 'https://example.com'
});
return axios.get('/path',{ headers: {'accept': 'application/json','X-API-Key': 'XXXXXX'} })
.then(response => {
console.log(response.data);
return res.status(200).json({
message: response.data
})
})
.catch(err => {
return res.status(500).json({
error: err
})
})
};
How can I execute many request in foreach without error please ?
Currently, I send a request on each entry on my array with foreach :
users.forEach(function (user) {
request({
url : 'myurl.com/api',
method: 'POST',
auth : {
'bearer': CONFIGURATION.auth.token
},
body : {
sender_id: user.sender_id
},
json : true
}, function (error, response, body) {
if (!error && response.statusCode === 200) {
resolve(body);
} else {
console.log('Error on coreService');
console.log('############### ERROR ###############');
console.log(error);
console.log('############### BODY ###############');
console.log(body);
console.log('############### RESPONSE ###############');
console.log(response);
reject(error);
}
});
});
With some request it's ok, but with some request I have this error :
Error on coreService
############### ERROR ###############
{ Error: connect ECONNRESET 127.0.0.1:80
at Object._errnoException (util.js:1022:11)
at _exceptionWithHostPort (util.js:1044:20)
at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1198:14)
code: 'ECONNRESET',
errno: 'ECONNRESET',
syscall: 'connect',
address: '127.0.0.1',
port: 80 }
############### BODY ###############
undefined
############### RESPONSE ###############
undefined
Do you have an idea how can I fix this problem please ?
I tried with :
server.timeout = 0;
or
server.timeout = 1000;
but same problem...
If I execute the request, user by user, it's fine !
But with the foreach, it's break on some request...
In the comments colinux proposes that the connection reset errors are due to the server protecting itself from too many simultaneous requests, and I think they are probably correct. This code shows how you can use async/await to make your requests to the server one at a time. This might be too slow for you, but it could help you to confirm that the problem is as explained by colinux.
Here is another answer which does not require the use of request-promise-native and instead wraps the request calls in its own Promise.
const request = require('request');
const users = [{sender_id: 1}, {sender_id: 2}, {sender_id: 3}];
// make this driver function async so we can use await which allows waiting for
// a request to finish before starting the next one
async function runUserRequests() {
for (let i = 0; i < users.length; i++) {
const user = users[i];
try {
const response = await requestPromise(user);
console.log("response for user", user, response);
} catch (error) {
console.log("error for user", user, error);
}
};
}
// wrap the request call in a Promise so that it will run synchronously
function requestPromise(user) {
return new Promise(function(resolve, reject) {
request({
url: 'http://localhost:4000/',
method: 'GET', // method 'POST'
// auth : {
// 'bearer': CONFIGURATION.auth.token
// },
// body : {
// sender_id: user.sender_id
// },
// json : true
}, function (error, response, body) {
if (!error && response.statusCode === 200) {
resolve(body);
console.log("request successful for user", user, " at ", (new Date()).getTime());
} else {
console.log('Error on coreService');
console.log('############### ERROR ###############');
console.log(error);
console.log('############### BODY ###############');
console.log(body);
console.log('############### RESPONSE ###############');
console.log(response);
reject(error);
}
});
});
}
runUserRequests();
/*
Sample server used to test the code above:
const express = require('express')
const app = express()
const port = 4000
app.get('/', (req, res) => {
console.log("spinning for a bit");
setTimeout( () => {
console.log(" now responding");
res.send('Hello World!');
}, 1000);
});
app.listen(port, () => console.log(`Example app listening on port ${port}!`))
*/
In the comments colinux proposes that the connection reset errors are due to the server protecting itself from too many simultaneous requests, and I think they are probably correct. This code shows how you can use async/await to make your requests to the server one at a time. This might be too slow for you, but it could help you to confirm that the problem is as explained by colinux.
To get this to work you'll need to install request-promise-native. If you can't do that let me know and I can work up an example wrapping the request api in your own Promise.
const request = require('request-promise-native');
//const users = [1, 2, 3, 4]; // dummy user array for testing
async function runUserRequests(users) {
for (let i = 0; i < users.length; i++) {
const user = users[i];
console.log("starting request for user ", user);
await request({
url: 'http://localhost:4000/',
method: 'GET',
auth : {
'bearer': CONFIGURATION.auth.token
},
body : {
sender_id: user.sender_id
},
json : true
}, function (error, response, body) {
if (!error && response.statusCode === 200) {
console.log("request successful for user", user, " at ", (new Date()).getTime());
resolve(body);
} else {
console.log('Error on coreService');
console.log('############### ERROR ###############');
console.log(error);
console.log('############### BODY ###############');
console.log(body);
console.log('############### RESPONSE ###############');
console.log(response);
reject(error);
}
});
};
}
runUserRequests();
I want to send a https request and process the result with serverless framework, but it seems not working.
Nodejs (serverless) skip my request and jump directly to the last result without waiting https reponse
here my function:
import { APIGatewayEvent, Callback, Context, Handler } from "aws-lambda";
var AWS = require("aws-sdk");
const querystring = require('querystring');
const https = require('https');
const TOKEN: String = "token";
export const hello: Handler = (
event: APIGatewayEvent,
context: Context,
cb: Callback
) => {
function https_request() {
var postData = querystring.stringify({
query: "query"
});
var options = {
hostname: 'example.com',
port: 443,
path: '/path',
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': TOKEN
}
};
return new Promise(function(resolve, reject) {
console.log("before request")
var req = https.request(options, (res) => {
console.log('statusCode:', res.statusCode);
console.log('headers:', res.headers);
if (res.statusCode !== 200) {
// REJECT IF THE RESPONSE WASN'T AS EXPECTED
return reject(new Error('Request failed'));
}
res.on('data', (d) => {
process.stdout.write(d);
resolve(d); // RESOLVE ON SUCCESS WITH EXPECTED DATA
});
});
req.on('error', (e) => {
console.error(e);
reject(e); // REJECT ON REQUEST ERROR
});
// req.write(postData);
req.end();
})
}
let x:any;
async function myAsyncF() {
x= await https_request();
console.log(x.body)
return x.body
}
myAsyncF()
const response = {
statusCode: 200,
body: JSON.stringify({
message: x,
input: event
})
};
cb(null, response);
};
I used the async await, but nothing is returned (i should receive at least an error if there is some king of network error)
here is my output:
before request
{
"statusCode": 200,
"body": "{\"input\":\"\"}"
}
is there something missing ?
Thank you in advance
At no point do you resolve your Promise, I've no idea what you deem a "successful" request but here's an example that should get you on the right track
return new Promise(function(resolve, reject) {
console.log("before request")
var req = https.request(options, (res) => {
console.log('statusCode:', res.statusCode);
console.log('headers:', res.headers);
if (res.statusCode !== 200) {
// REJECT IF THE RESPONSE WASN'T AS EXPECTED
return reject(new Error('Request failed'));
}
res.on('data', (d) => {
process.stdout.write(d);
resolve(d); // RESOLVE ON SUCCESS WITH EXPECTED DATA
});
});
req.on('error', (e) => {
console.error(e);
reject(e); // REJECT ON REQUEST ERROR
});
req.write(postData);
req.end();
})
Remove process.stdout.write will is not a great practice as you are running your code on lambda, use a context object or a callback function to return
process.stdout.write(d) is write operation on file, sometimes it silently fails as well, running on EC2, ECS your solution looks gem. But on serverless it's always better to avoid file operation functions
I have an https.get request in Node for which I need to handle the errors - preferably in a try-catch block. For instance, when the url is incorrect
I have tried wrapping the https.get block in a try catch and I have tried handling with res.on('error'). It seems as if in both instances the error doesn't reach the error handling block.
const https = require('https');
const hitApi = () => {
const options = {
"hostname": "api.kanye.rest"
};
try {
https.get(options, res => {
let body = '';
res.on('data', data => {
body += data;
});
res.on('end', () => {
body = JSON.parse(body);
console.dir(body);
});
});
} catch (error) {
throw error;
}
}
hitApi();
If I change the url to a nonexistent API (e.g. api.kaye.rest) I would expect to see a handled e.rror response. Instead I see 'Unhandled error event'
The reason why try...catch.. fails is that it is meant for handling synchronous errors. https.get() is asynchronous and cannot be handled with a usual try..catch..
Use req.on('error',function(e){}); to handle the error. Like so:
var https = require('https');
var options = {
hostname: 'encrypted.google.com',
port: 443,
path: '/',
method: 'GET'
};
var req = https.request(options, function(res) {
console.log("statusCode: ", res.statusCode);
console.log("headers: ", res.headers);
res.on('data', function(d) {
process.stdout.write(d);
});
});
req.end();
// Error handled here.
req.on('error', function(e) {
console.error(e);
});
You can read more about the same in the documentation over here
I'm trying to access all repositories that have more than 5000 stars on Github. I've written this scraper to work with Node.js (it's running on a Cloud9 environment):
var request = require('request');
var fs = require('fs');
var options = {
url: 'https://api.github.com/repositories',
headers: {
'User-Agent': 'myusernamehere'
},
qs: {
stargazers: 5000
}
};
function callback(error, response, body) {
if (!error && response.statusCode == 200) {
console.log(response.headers);
fs.writeFile('output_teste.json', body, function (err) {
if (err) throw err;
console.log('It\'s saved!');
console.log(response.statusCode);
});
} else {
console.log(response.statusCode);
}
}
request(options, callback);
But the result is not all of the repositories, just the first page of all of them. How can I use pagination with the Request module? I've tried to find examples within the documentation, but they aren't that clear. Or do I need to do this with another library or maybe another language?
Thanks!
you should modify your querystring to include the value of "since". You can read more on the github documentation.
https://developer.github.com/v3/repos/#list-all-public-repositories
Sample URL with query string of since
https://api.github.com/repositories?since=364
You could use the pagination data provided in response.headers.link that's received when making calls to the GitHub API to find out if there are any more pages left for your call.
One approach is to loop through the pages until there are no more new pages left, at which point you can write to file and return from function.
On each loop you can add to the data that you already have by using concat (I assume that the response body is delivered as an array) and then passing on the data to the next function call.
I rewrote your code to include a basic implementation of such a technique:
var request = require('request');
var fs = require('fs');
var requestOptions = function(page) {
var url = 'https://api.github.com/repositories?page=' + page;
return {
url: url,
headers: {
'User-Agent': 'myusernamehere'
},
qs: {
stargazers: 5000
}
};
};
function doRequest(page, incomingRepos) {
request(requestOptions(page), function(error, response, body) {
if (!error && response.statusCode == 200) {
console.log(response.headers);
var currentPageRepos = JSON.parse(body);
var joinedRepos = incomingRepos.concat(currentPageRepos);
var linkData = response.headers.link;
// if response does not include reference to next page
// then we have reached the last page and can save content and return
if (!(linkData.includes('rel="next"'))) {
fs.writeFile('output_teste.json', JSON.stringify(joinedRepos), function(err) {
if (err) throw err;
console.log('It\'s saved!');
});
return;
}
page++;
doRequest(page, joinedRepos);
} else {
console.log(response.statusCode);
}
});
}
doRequest(1, []);