I seem to be doing something wrong.
I have a student website that I want to scrape, but first I need to log in. Currently I have a python scraper that does it. The website logs in with a post request to a url containing a sid and PIN.
var login_url = 'https://example.com';
var formData = {
sid: 'username',
PIN: 'password'
}
How would I go about creating the same scraper but with javascript? I have seen the request library, which seems like what I want to use but cannot get it to work.
You need to use the request module to POST the form data to your endpoint. The response from the server will be in the call back to the .post() method.
const request = require('request');
// do not reassign "request", if you need to set properties us a different variable
// use the action= value from the form for the URL
const url = 'https://central.carleton.ca/prod/twbkwbis.P_ValLoginn';
const data = {
sid: 'username',
PIN: 'password',
};
request.post({ url: url, formData: data }, (err, response, body) => {
if (err) {
console.log('failed', err);
} else {
console.log('the response', body);
}
});
If you are interesting in parsing the resulting HTML I recommend using CheerioJS - much like jQuery but server side.
Related
I have created several Node.JS APIs and tested all APIs using Chrome API Tester. But when I call that API from HTML file using AJAX it doesn't work.
Here is example API to login user.
app.post('/loginuser', async (req, res) => {
var query = 'select id,user_pw from t_user where id=?'
rows = await selectQuery(query, [req.query.id])
res.status(200).send({'result': rows})
});
When I call this API from API tester with user_id and password I can get the parameters from req.query.user_id and req.query.password. But when I call that same API from HTML file, req.query is empty.
Call from API tester:
http://localhost:8081/loginuser?id=userid001&pw=123
Call From HTML file using Ajax:
<script type="text/javascript">
$('#btnSubmit').click(function(){
var email = $('#txtEmail').val()
var password = $('#txtPassword').val()
if(email == "" || password == ""){
alert("Please Enter Id and Password")
return
}
url = "http://localhost:8081/loginuser"
$.ajax({
data: {
id: email,
pw: password,
},
dataType: 'json',
type: 'POST',
url: url
}).done(function(data){
console.log(data)
});
});
</script>
Sending credentials via URL query parameters is generally a very bad idea.
Your Express app should read parameters from the request body...
app.post('/loginuser', async (req, res) => {
const query = "select id, user_pw from t_user where id = ?";
try {
const result = await selectQuery(query, [req.body.id]);
res.json({ result });
} catch (err) {
console.error("/loginuser", err);
res.status(500).send("DB error");
}
});
Make sure it can handle either urlencoded data (that's what jQuery will send), JSON or both
app.use(express.urlencoded()); // handle application/x-www-form-urlencoded
app.use(express.json()); // handle application/json
If you absolutely must send URL query parameters, you need to encode them into the URL jQuery uses
// only adding `id` since you're not using `pw` anyway
const params = new URLSearchParams({ id: email });
$.ajax({
url: `http://localhost:8081/loginuser?${params}`,
method: "post",
dataType: "json"
}).done(console.log);
I'm getting no proper response while make an API request to external API using basic authentication (username and password) in nodejs (javascript)
I used the below code and the response is "undefined", not sure what is missing here.
But I was able to make a request using postman tool without any issues.
const request = require('request')
const user = '*****';
const pass = '*****!';
const url = 'https://servicenow.com/api/table'
var options = {
url: url,
auth: {
username: user,
password: pass
}
};
request.get(options, (err, res, body) => {
if (err) {
return console.log(err);
}
console.log(body.url);
console.log(body.explanation);
});
Response:
undefined
undefined
if your api right with postman you can do like this based on photo
send a request
click on code
select nodejs- Request
copy
I'm trying to do a redirect after an ajax put request. I plan on using pure JS client side for validation.
Client:
$(document).ready(function() {
login = () => {
var username = $("[name='username']").val()
var password = $("[name='password']").val()
$.ajax({
type: "put",
url: '/login',
data: {
username: username,
password: password
}
// success: function(response) {
// console.log('Success:')
// console.log(response.user)
// Cookies.set('username', response.user.username)
// Cookies.set('first_name', response.user.first_name)
// Cookies.set('last_name', response.user.last_name)
// Cookies.set('email', response.user.email)
// window.location.href = window.location.origin + '/'
// },
// error: function(error) {
// console.log("Error:")
// console.log(error)
// }
})
}
logout = () => {
console.log("Log out clicked.")
Cookies.remove('username')
Cookies.remove('first_name')
Cookies.remove('last_name')
Cookies.remove('email')
window.location.href = window.location.origin + '/logout'
}
})
Server:
/* GET home page. */
router.get('/', function(req, res, next) {
res.render('main')
});
router.put('/login', function(req, res) {
// Password is not encrypted here
console.log('req.body')
console.log(req.body)
User.findOne({ username: req.body.username }, function(err, user) {
// Password is encrypted here
if (err) throw err
console.log('user')
console.log(user)
bcrypt.compare(req.body.password, user.password, function(err, result) {
if (result) {
var token = jwt.encode(user, JWT_SECRET)
// return res.status(200).send({ user: user, token: token })
return res.redirect('/')
} else {
return res.status(401).send({error: "Something is wrong."})
}
})
})
})
I can't get main.hbs to render after a successful login. My commented code works, but I'm trying to do my redirect server side rather than client side because I'm told that it's better for security.
You should know when to use href and replace functionalities.
window.location.replace(...) will best way to represent an HTTP redirect.
Reason
When compared to window.location.href, window.location.replace(...) is better to use in a HTTP redirect scenario because replace() avoids keeping the originating page in the session history, this helps users to avoid get stuck in a never-ending back-button fiasco.
Summary
If you want to illustrate clicking on a link, use location.href
If you want to illustrate an HTTP redirect, use location.replace
Sample
// an HTTP redirect
window.location.replace("http://example.com");
// clicking on a link
window.location.href = "http://example.com";
Update
The server cannot do a redirect from an ajax request. In the end ajax involves the client (browser).
If you want, you can send the instruction of a redirection through the server side call, but it will be end up again on client side, in the callback.
You can do that by returning an object from the server which contains the url you want to redirect to. Then use javascript to change the document's location property. Like Below:
Server Side Code
if (result) {
var token = jwt.encode(user, JWT_SECRET)
return res.status(200).send({result: 'redirect', url:'/'})
} else {
return res.status(401).send({error: "Something is wrong."})
}
And then in Client Side Javascript:
$.ajax({
type: "put",
url: '/login',
data: {
username: username,
password: password
}
success: function(response) {
if (response.result == 'redirect') {
//redirecting to main page from here.
window.location.replace(response.url);
}
}
});
Apart from this your commented code is the correct way to do this. Just like one of the comments in you question "server side redirect is deadend for an ajax request as the instruction is not for the browser but some javascript handler."
I don't think what you want to do is possible. An AJAX request is meant just to pass data back and forth. What happens now is that you need to script client side behavior on your end. This means the AJAX request will pass a 302 and other data that comes along for the ride to the callback on JS. No client side behavior can be altered from the server. It is up to you to do something with the AJAX returned values. If 500, throw an error message, 200 do something etc.
The only way you can get a server redirect to work is by traditional HTML form submission.
I'm having trouble scraping a website that needs authentication, and is using session cookies. The session requires a request with POST, and the authentication then approves. But when I want to GET the webpage that need authentication, it returns "Unauthorized". I guess I need a way to bring the session cookie with the GET-request, but I don't know how! My dependencies is request-promise(https://www.npmjs.com/package/request-promise).
The code looks like this:
var rp = require("request-promise");
var options = {
method: "POST",
uri: "http://website.com/login",
form: {
username: "user",
password: "pass",
},
headers: {},
simple: false
};
rp(options).then(function(response) {
console.log(response); // --> "Redirecting to login/AuthPage"
request("http://website.com/login/AuthPage", function(err, res, body) {
console.log(body); // --> "Unauthorized"
})
}).catch(function(e) {
console.log(e)
})
I'm guessing you have to put the request in a "Jar" (https://github.com/request/request#requestjar), to be able to reach the next request-URL, but how can I set the request-promise to create a cookie-jar?
Your problem is how to keep the session after authentication.
That means, after logging in by using username and password, the server will return a cookie with an identifier. Then you need to attach that cookie to all your feature requests.
It's simple with request-promise. Just keep tracking session by enabling jar option then use the same request object for all requests.
Let take a look
var request = require("request-promise").defaults({ jar: true });
var options = {
method: "POST",
uri: "http://website.com/login",
form: {
username: "user",
password: "pass",
},
headers: {},
simple: false
};
request(options).then(function(response) {
request("http://website.com/login/AuthPage", function(err, res, body) {
console.log(body);
})
}).catch(function(e) {
console.log(e)
})
Use the following object while making rest calls.
var request = require("request-promise").defaults({jar: true});
To add your own cookies
var tough = require('tough-cookie');
// Easy creation of the cookie - see tough-cookie docs for details
let cookie = new tough.Cookie({
key: "some_key",
value: "some_value",
domain: 'api.mydomain.com',
httpOnly: true,
maxAge: 31536000
});
// Put cookie in an jar which can be used across multiple requests
var cookiejar = rp.jar();
cookiejar.setCookie(cookie, 'https://api.mydomain.com');
// ...all requests to https://api.mydomain.com will include the cookie
var options = {
uri: 'https://api.mydomain.com/...',
jar: cookiejar // Tells rp to include cookies in jar that match uri
};
and then make the call. More details about request-promise:
https://www.npmjs.com/package/request-promise
I am very new to networking and I have this code which, when I use a REST API like Postman, does exactly what I want it to do:
router.post('/', function(req,res,next){
var reqObj = req.body;
console.log(reqObj);
req.getConnection(function(err, conn){
if(err)
{
console.error('SQL Connection error: ', err);
return next(err);
}
else
{
var query = conn.query("INSERT INTO coordinates (id,lat,lon) VALUES(3,2,1);");
if(err)
{
console.error('SQL error: ', err);
return next(err);
}
res.json("Coordinates sent.");
}
})
} );
That is, it sends the query request to the MYSQL database. My question is, how do I do this without using Postman to send the POST request?
Thank you.
You can't unless you make a post request from within your application or something. If you don't intend on sending data, you can just make it a GET request by changing
router.post('/', function(req,res,next){
to
router.get('/', function(req,res,next){
Then you can just go to the relevant URL from your browser. If you're using chrome and you just wanna see the JSON data, I'd also recommend installing the JSONView chrome extension.
EDIT
Here's the example request using request-promise
var request = require('request-promise');
var objectData = {
name: 'Bruce',
alias: 'Batman'
};
var options = {
method: 'POST',
uri: 'http://your.api/endpoint/',
body: objectData,
json: true // Automatically stringifies the body to JSON
};
request(options).then(function(response){
// handle success response
}, function(error){
// handle error response
})