I have 2 asynchronous functions on gives me some urls and other one gives me the content I need from the url and one that writes the content I need to an file, but it write to file first and then it gives me the content I need to write, how can I get him to write to file after I get the content I need. Thanks
var http = require('http');
var fs = require('fs');
var list = {};
var k = 0;
function getContent(url){
var options = {
host: 'www.exammple.org',
path: url
}
var request = http.request(options, function (res) {
var data = '';
res.on('data', function (chunk) {
data += chunk;
});
res.on('end', function () {
list[k] = data;
k++;
});
});
request.on('error', function (e) {
console.log(e.message);
});
request.end();
}
var options = {
host: 'www.example2.org',
path: '/mypath'
}
var request = http.request(options, function (res) {
var data = '';
res.on('data', function (chunk) {
data += chunk;
});
res.on('end', function () {
///Here I get some urls from data
data = data.split(";");
for(key in data){
data = data.split('href=\"')[1].split('"')[0];
getContent(data[key]);
}
writeToFile();
}
});
});
request.on('error', function (e) {
console.log(e.message);
});
function writeToFile(){
fs.writeFile("file.txt", JSON.stringify(list), function(err) {
if(err) {
console.log(err);
} else {
console.log("The file was saved");
}
});
}
request.end();
Related
I'm trying to get my function to return the http get request, however, whatever I do it seems to get lost in the ?scope?. I'm quit new to Node.js so any help would be appreciated
function getData(){
var http = require('http');
var str = '';
var options = {
host: 'www.random.org',
path: '/integers/?num=1&min=1&max=10&col=1&base=10&format=plain&rnd=new'
};
callback = function(response) {
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function () {
console.log(str);
});
//return str;
}
var req = http.request(options, callback).end();
// These just return undefined and empty
console.log(req.data);
console.log(str);
}
Of course your logs return undefined : you log before the request is done. The problem isn't scope but asynchronicity.
http.request is asynchronous, that's why it takes a callback as parameter. Do what you have to do in the callback (the one you pass to response.end):
callback = function(response) {
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function () {
console.log(req.data);
console.log(str);
// your code here if you want to use the results !
});
}
var req = http.request(options, callback).end();
Simple Working Example of Http request using node.
const http = require('https')
httprequest().then((data) => {
const response = {
statusCode: 200,
body: JSON.stringify(data),
};
return response;
});
function httprequest() {
return new Promise((resolve, reject) => {
const options = {
host: 'jsonplaceholder.typicode.com',
path: '/todos',
port: 443,
method: 'GET'
};
const req = http.request(options, (res) => {
if (res.statusCode < 200 || res.statusCode >= 300) {
return reject(new Error('statusCode=' + res.statusCode));
}
var body = [];
res.on('data', function(chunk) {
body.push(chunk);
});
res.on('end', function() {
try {
body = JSON.parse(Buffer.concat(body).toString());
} catch(e) {
reject(e);
}
resolve(body);
});
});
req.on('error', (e) => {
reject(e.message);
});
// send the request
req.end();
});
}
Shorter example using http.get:
require('http').get('http://httpbin.org/ip', (res) => {
res.setEncoding('utf8');
res.on('data', function (body) {
console.log(body);
});
});
from learnyounode:
var http = require('http')
http.get(options, function (response) {
response.setEncoding('utf8')
response.on('data', console.log)
response.on('error', console.error)
})
'options' is the host/path variable
from learnyounode:
var http = require('http')
var bl = require('bl')
http.get(process.argv[2], function (response) {
response.pipe(bl(function (err, data) {
if (err)
return console.error(err)
data = data.toString()
console.log(data)
}))
})
I think it's too late to answer this question but I faced the same problem recently my use case was to call the paginated JSON API and get all the data from each pagination and append it to a single array.
const https = require('https');
const apiUrl = "https://example.com/api/movies/search/?Title=";
let finaldata = [];
let someCallBack = function(data){
finaldata.push(...data);
console.log(finaldata);
};
const getData = function (substr, pageNo=1, someCallBack) {
let actualUrl = apiUrl + `${substr}&page=${pageNo}`;
let mydata = []
https.get(actualUrl, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', async () => {
if (JSON.parse(data).total_pages!==null){
pageNo+=1;
somCallBack(JSON.parse(data).data);
await getData(substr, pageNo, someCallBack);
}
});
}).on("error", (err) => {
console.log("Error: " + err.message);
});
}
getData("spiderman", pageNo=1, someCallBack);
Like #ackuser mentioned we can use other module but In my use case I had to use the node https. Hoping this will help others.
This is my solution, although for sure you can use a lot of modules that give you the object as a promise or similar. Anyway, you were missing another callback
function getData(callbackData){
var http = require('http');
var str = '';
var options = {
host: 'www.random.org',
path: '/integers/?num=1&min=1&max=10&col=1&base=10&format=plain&rnd=new'
};
callback = function(response) {
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function () {
console.log(str);
callbackData(str);
});
//return str;
}
var req = http.request(options, callback).end();
// These just return undefined and empty
console.log(req.data);
console.log(str);
}
somewhere else
getData(function(data){
// YOUR CODE HERE!!!
})
I am trying to make a HTTP get request, but nothing is being returned as text. Here is my code
const http = require('http');
const ws = require('ws');
const wss = new ws.Server({noServer: true});
var XMLHttpRequest = require('xhr2');
function accept(req, res) {
console.log("got request")
wss.handleUpgrade(req, req.socket, Buffer.alloc(0), function(ws) {
//party name, sockets on party, limit 2
ws.on('message', function (msg) {
var message = JSON.parse(msg.toString())
if(message.type==='add') {
//do later
} else if(message.type==='start') {
httpGetAsync('192.168.26.23', "/startRelay")
} else if(message.type==='stop') {
httpGetAsync('92.168.26.23', "/stopRelay")
} else if(message.type==='status') {
httpGetAsync('92.168.26.23', "/")
}
});
ws.on('close', function () {
console.log('closed')
});
ws.on('error', function(error){
console.log('error - ' + JSON.stringify(error));
});
})
}
function httpGetAsync(theUrl, path) {
var options = {
host: theUrl,
path: path,
method: "GET",
}
callback = async function(response) {
var str = ''
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function () {
console.log(JSON.parse(str));
});
};
http.request(options, callback).end();
}
http.createServer(accept).listen(3000);
Here is what 192.168.26.23 has in it (if you type it up from my wifi onto google, this is what shows - and this data is what I need to get in a string)
{"success":true,"ms_left":0,"relayStarted":false}
so am using core-https to do a Get request to a webstie , iam doing this get reqeust using a proxy by using this code :
const { HttpsProxyAgent } = require("https-proxy-agent");
const proxy = new HttpsProxyAgent(`http://user:pass#host:port`);
https.get("https://www.google.com/",
{ agent: proxy },
(res) => {
var body = "";
res.on("data", function (chunk) {
body += chunk;
// console.log(body)
});
res.on("end", function () {
}
);
})
so sometimes the proxy would be invalid or expired , or even use a local-host for debugging using fiddler or Charles
const { HttpsProxyAgent } = require("https-proxy-agent");
const proxy = new HttpsProxyAgent(`http://127.0.0.1:8888`); // For Debugging
https.get("https://www.google.com/",
{ agent: proxy },
(res) => {
var body = "";
res.on("data", function (chunk) {
body += chunk;
// console.log(body)
});
res.on("end", function () {
}
);
})
and would also result an error if i forgot to open a proxy-debugger .
i tried doing it in this way :
res.on("error" , function(e){
console.log("an error have been occurred ")
})
but nothing seems to work
So i found the answer , it would be done like this
https.get(
"https://www.google.com/",
{ agent: proxy },
(res) => {
var body = "";
res.on("data", function (chunk) {
body += chunk;
});
res.on("end", function () {
// console.log(body)
})
.on('error', function (e) {
console.error("error");
}).end();
My main.js file:
var spaceJson = require('./space.js');
var options = {
"method": "GET",
"hostname": "10.10.111.226",
"port": null,
"path": "/API/Org",
"headers": {
"authorization": "Bearer eyJ0eXAiOiJKV1QiLCJhbG...",
"cache-control": "no-cache"
}
};
var JSONdata = spaceJson.space.getSpaceData(options);
my space.js file(this actually is making a request to a web api and returns json)
var module = module.exports = {};
var http = require("http");
module.space = {
sayHelloInSpanish: function() {
return "Hola";
},
getSpaceData : function(options) {
var body = '';
http.request(options, function (req,res) {
var chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", function () {
body = Buffer.concat(chunks);
console.log(body.toString());
});
req.end();
});
//return http.get();
}
};
var require = function(path){
return module.exports;
};
when I call
sayHelloInSpanish
method, it returns "holla" when I run main.js. But when I call the
getSpaceData
I dont see anything on command line and it hangs. Maybe I am not handling callback. With this I basically want to consume JSON data in my main.js
You don't see anything because The HTTP response is async.
You should return a callback or use a promise:
getSpaceData : function(options, clbk) {
var body = '';
http.request(options, function (req,res) {
var chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", function () {
body = Buffer.concat(chunks);
console.log(body.toString());
clbk(body.toString());
});
req.end();
});
}
spaceJson.space.getSpaceData(options, function(data){
console.log(data);
});
I recommend you to watch this video: https://vimeo.com/96425312
-- UPDATE --
Delete the require code:
var require = function(path){
return module.exports;
};
add to main:
// main.js
var spaceJson = require("./space.js");
Tried various techniques now to no prevail, can anybody shed any light on this.
//Google search -------------------
app.get("/google/:name", function (req, res) {
var term = req.params.name;
var searchTerm = function (search, callback) {
var options = {
hostname: "ajax.googleapis.com",
path: "/ajax/services/search/web?v=1.0&q="+ search + ""
};
var gsaReq = http.get(options, function (response) {
var completeResponse = '';
response.on('data', function (chunk) {
completeResponse += chunk;
});
response.on('end', function() {
outputResult(completeResponse);
});
}).on('error', function (e) {
console.log('problem with request: ' + e.message);
});
}(term, outputResult);
function outputResult(result){
console.log(result);
res.send(result);
// res.send('hello world');
};
});
This works fine if the res.send() sends hello world just not the result of my call back, despite the result being a string? Any ideas?
If the above doesn't make sense the code below is simplified (without the callback) and the res.send() still fails..?
The res.send doesn't work without the callback - ahhh:
app.get("/google/:name", function (req, res) {
var term = req.params.name;
var searchTerm = function (search) {
var options = {
hostname: "ajax.googleapis.com",
path: "/ajax/services/search/web?v=1.0&q="+ search + ""
};
var gsaReq = http.get(options, function (response) {
var completeResponse = '';
response.on('data', function (chunk) {
completeResponse += chunk;
});
response.on('end', function() {
res.send(completeResponse);
});
}).on('error', function (e) {
console.log('problem with request: ' + e.message);
});
}(term);
});