phantom.casperPath=('C:\Users\lee0\Desktop\casperjs-1.1.4-1\casperjs-1.1.4-
1');
phantom.injectJs('C:\Users\lee0\Desktop\casperjs-1.1.4-1\casperjs-1.1.4-
1\bin\bootstrap.js');
var fs=require('fs')
var casper = require('casper').create()({
pageSettings: {
userAgent: 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36
(KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36'
}
});
//open Facebook
casper.start().thenOpen("https://dcid.dcinside.com/join/login.php?",
function() {
console.log("Facebook website opened");
});
//로그인
casper.then(function(){
console.log("해당 id와 pw로 로그인 합니다.");
this.evaluate(function(){
document.getElementById("user_id").value="chermy018";
document.getElementById("password").value="kl424321";
document.getElementById("login_ok").children[1].click();
});
});
var request = require('request'),
cheerio = require('cheerio');
var url = "http://gallog.dcinside.com/inc/_mylog.php?
gid=chermy018&oneview=Y&cid=59&page=1";
request(url, function(err, res, html) {
if (!err) {
var $ = cheerio.load(html);
}
console.log(html);
})
casper.run();
TypeError: undefined is not a constructor evaluating casper.start().
i already posted same question of above texts. i added comments that it's seems i have problem with installation of casperjs because other proper codes also doesn't work resulting "undefined is not a constructor evaluating casper.start()". just in case of possibility of ask, i post same question.
Your script is written for CasperJS, but you've ran it via PhantomJS, which is incorrect. Instead you should run it this way (supposing CasperJS is in PATH):
casperjs text2.js
Related
I have a final project where the data I want to retrieve is through scraping data on shopee, but I have a problem when I scrape shopee on a hidden API, when I try it on the Insomnia script the script runs, but when I try it on local or google colab script this is an error.
How can I fix this?
import requests
import json
headers = {
'sec-ch-ua-mobile': '?0',
'cookie': 'REC_T_ID=e67c02b5-ae54-11ec-b368-46ac8e8cc9d8; SPC_F=KrJ9Ck0EYC252EWJ3FSH5QFNzjmvng6O; SPC_IA=-1; _gcl_au=1.1.459910866.1654678938; _fbp=fb.2.1654678939550.956784750; G_ENABLED_IDPS=google; SPC_CLIENTID=S3JKOUNrMEVZQzI1jkqfwanvqrwehsep; _gcl_aw=GCL.1660529943.Cj0KCQjwuuKXBhCRARIsAC-gM0g5RPYu1Cfx0PZbXHrR5qqd7JqgFEy4XrCAxXEGFD4quU2tORTIR9caAsVdEALw_wcB; _gac_UA-61904553-8=1.1660529949.Cj0KCQjwuuKXBhCRARIsAC-gM0g5RPYu1Cfx0PZbXHrR5qqd7JqgFEy4XrCAxXEGFD4quU2tORTIR9caAsVdEALw_wcB; _med=refer; _gid=GA1.3.792417909.1660891119; csrftoken=Hk3UgpYhG30zu0CO9Vhk2OIKptWNBS0g; _QPWSDCXHZQA=9be12e07-9c49-426e-e0d8-01a11f73956b; AMP_TOKEN=%24NOT_FOUND; __LOCALE__null=ID; _dc_gtm_UA-61904553-8=1; SPC_T_ID="uXbSXytLbRMSr+KtQpRkW7f5FHiriPO+CdAryBv6THa5ljtJhfxKSiI5g2Ps2Fl4eILJBWgkAYeR+c0hO4843b12KCXHt56jNWASfgA5Uq8="; SPC_U=616200160; SPC_T_IV="eB95as87FjhL8HoasAA0kw=="; _ga_KK6LLGGZNQ=GS1.1.1660961596.8.0.1660961596.0.0.0; SPC_R_T_ID=uXbSXytLbRMSr+KtQpRkW7f5FHiriPO+CdAryBv6THa5ljtJhfxKSiI5g2Ps2Fl4eILJBWgkAYeR+c0hO4843b12KCXHt56jNWASfgA5Uq8=; SPC_R_T_IV=eB95as87FjhL8HoasAA0kw==; SPC_T_ID=uXbSXytLbRMSr+KtQpRkW7f5FHiriPO+CdAryBv6THa5ljtJhfxKSiI5g2Ps2Fl4eILJBWgkAYeR+c0hO4843b12KCXHt56jNWASfgA5Uq8=; SPC_T_IV=eB95as87FjhL8HoasAA0kw==; SPC_SI=id+yYgAAAABBUWdBaGJJRaEQWwAAAAAAbDFtUDZZZ2k=; SPC_ST=.aVZDcFoyVjBuUWIwUXVSUnkGCGuGI58EkFOzdykhsuSCGz0GrBWotkUiREvJO38YxTxyl3Pgbl73NUs1AmCexDhPneO/ABd8bgUkVqlhCvZTNPDPg8jv/9KaHwWagKm9FM55IY61eECu5twdRUQl9u3xgfshk26TRkvpli4dlCUZzIE0boMi5/5B/CcqUgoXsDH567+KunuKEe92wUSC1Q==; _ga=GA1.3.1352849021.1654678939; cto_bundle=ZLv7oF9EWUpOZWVHYUkyUHh0d2RBWDJvTWk5eXllWVpia1F1eXJ4RkdZcjhVZ1Q3NVRYYnE0c0hOWERsMm1tTjFER09MbmdMTW1VZG5VbkQ0MjByVnpxYlNRdk1MRk9TUGtNSzZpRzRnZFNXU1ZUVlElMkY1dXRpbGFUZm5vdjdvcklFQzk0YzBuVm1qMUJzZnRyb2xPMHpRMldVQSUzRCUzRA; _ga_SW6D8G0HXK=GS1.1.1660959836.35.1.1660961617.22.0.0; SPC_EC=U1A1Vk5JdzlVaFVYdjJRUk4fyVPKEHSso64GpvFSCO/oihfsUpaQrXO9e4XqPT/AjNQJP7hcW+o+A7chna6AIbCtFRsocFdW1x1oS3A8+pNHmK3oRTDCZe2BDyAP0cOp133wiyu0GTSCetXIhbIRwvkOTJYqOXYBGKuTW6tGY1o=; shopee_webUnique_ccd=veSMI3XpR84mDT6rWJgoWg%3D%3D%7C9xD6GCFDkurxx4Cxf%2F72oK7%2FP2ilXgSYBkzRAd4F%2BSkKrCsqCWGVzz0SHGMINBr5KgoTxt7LXhBKejCILMQlWRcetFY%3D%7ClXsfMcnYECC51PEy%7C05%7C3',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36',
'referer':'https://shopee.co.id/Perawatan-Kecantikan-cat.11043145?page=0&ratingFilter=4',
}
x=0
number_page = x*60
url =("https://shopee.co.id/api/v4/search/search_items?by=relevancy&limit=60&match_id=11043145"
"&newest={}&order=desc&page_type=search&rating_filter=4&scenario=PAGE_CATEGORY&version=2").format(number_page)
y = requests.get(url, headers=headers).json()
y
Output:
{'tracking_id': '745a6f4b-0fc3-48af-b563-5a7ec483a601',
'action_type': 2,
'error': 90309999}
You can add a requests header, like this headers key af-ac-enc-dat value null, this works for me.
var request = WebRequest.Create(apiURL);
request.Headers["x-api-source"] = "pc";
request.Headers["af-ac-enc-dat"] = "null";
var response = request.GetResponse();
var reader = new StreamReader(response.GetResponseStream());
s_ResponseString = reader.ReadToEnd();
dynamic Prodata = Newtonsoft.Json.Linq.JObject.Parse(s_ResponseString);
im trying to connect to my local node server route (http://localhost:3000/user-ragnarok-controller/saveragnarokuser/) from my javascript application and im getting no response from it. The server is receiving the request and is processing the same, but i cant get the response at client side.
my javascript app are running at localhost:80 (Apache XAMPP) and my node server at localhost:3000.
this is my javascript code to connect to node server end-point:
function handler() {
alert('handler');
if(invocation.readyState === XMLHttpRequest.DONE && invocation.status === 200) {
alert('entrei aqui');
console.log(invocation.responseText);
} else
alert('nao foi hj ' + invocation.status.toString());
}
function saveUser() {
alert('dourado ');
var eml = document.getElementById('emailInputRegister');
var user = document.getElementById('userInputText');
var sx = document.getElementById("sexInputSelected");
var selectedSex = sx.options[sx.selectedIndex].value;
var pwd = document.getElementById("passwordInputRegister");
var uri = 'http://localhost:3000/user-ragnarok-controller/saveragnarokuser/';
var body = {
'userid': user.value,
'userpass': pwd.value,
'email': eml.value,
'sex': selectedSex
};
invocation.open('POST', uri);
invocation.withCredentials = true;
invocation.setRequestHeader('Content-Type', 'application/json');
invocation.onreadystatechange = this.handler;
invocation.send(JSON.stringify(body));
}
this is my request at google chrome console
Now let's talk about the server side. Here i have a middleware for CORS treatment.
// Add headers
app.use(function (req, res, next) {
console.log('reqHeaders: ' + JSON.stringify(req.headers));
res.header('Access-Control-Allow-Origin', 'http://localhost');
res.header('Access-Control-Allow-Headers', 'content-type');
res.header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS, PUT, PATCH, DELETE');
res.header('Access-Control-Allow-Credentials', true);
if(req.method === 'OPTIONS')
return res.status(200).send({});
next();
});
After CORS, the server starts my route POST:
router.post('/saveragnarokuser',function(req,res,next){
console.log('######################### Iniciando saveragnarokuser.');
UserRagnarokController.addUser(req.body,function(err,count){
if(err){
console.log('entrei aqui error: ' + err);
res.json(err);
}
else{
console.log('entrei aqui ok');
var userObj = {
response: "OK"
};
res.status(200).json(userObj);
}
});
});
looking to the server log its possible to see the request coming, processing, but for some reason the POST response is -- ms --. With no Status and execution time.
reqHeaders: {"host":"localhost:3000","connection":"keep-alive","content-length":"88","user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36","content-type":"application/json","accept":"*/*","origin":"http://localhost","sec-fetch-site":"same-site","sec-fetch-mode":"cors","sec-fetch-dest":"empty","referer":"http://localhost/co-cadastro/register.html?","accept-encoding":"gzip, deflate, br","accept-language":"pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7"}
######################### Iniciando saveragnarokuser.
myReqbody: {"userid":"fernandocabeca","userpass":"1234","email":"acosta.aluiz#gmail.com","sex":"F"}
POST /user-ragnarok-controller/saveragnarokuser/ - - ms - -
entrei aqui ok
My function UserRagnarokController.addUser performed perfectly, the requisition data was successfully filled in the database, i just need to get this response at client side (google chrome app), i got no error and no success.
NOTE: when I execute the request at POSTMAN it works normally, the answer is 200 OK.
There is probably a warning in the browser's console about breaking of CORS policy.
Headers about CORS should come from your server (Response Headers) and I don't see that tab here in your screenshot.
Postman ignores CORS, and server process all request by default.
Here is my code:
var system = require('system');
var page = require('webpage').create();
var server = require('webserver').create();
var system = require('system');
var someUrl = "http://sportfun.bg";
var port = 3000;
//Screen resolution
page.viewportSize = {
width: 1920,
height: 1080
};
//User agent
page.settings.userAgent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36';
page.onConsoleMessage = function(msg) {
system.stderr.writeLine('Console: ' + msg);
};
var service = server.listen(port, function (request, response) {
//console.log('Request received at ' + new Date());
//console.log("Request" + JSON.stringify(request.post));
var POST = request.post;
//console.log("Bank:" + POST.bank);
// TODO: parse `request` and determine where to go
var step = 0;
page.open(someUrl, function (status) {
if (status !== 'success') {
console.log('Unable to post!');
} else {
console.log("Here0");
page.onLoadFinished = function(status) {
if(status === "success"){
console.log("Here 1");
response.statusCode = 200;
response.headers = {
'Cache': 'no-cache',
'Content-Type': 'text/plain;charset=utf-8'
};
// TODO: do something on the page and generate `result`
response.write("We good");
//console.log(page.content);
response.close();
}
};
}
});
});
console.log("Server started!");
I'm using PhantomJS 2.1.1.
When i launch the server and send a post request to it in the console i see strange thing:
Server started!
Here0
Here 1
Here 1
Why i see twice Here 1 when actually the page should be once loaded ?
Is there any reason about it and how can i fix it ?
Multiple callbacks of onLoadFinished happen if there are multiple widgets embedded via iframe — like Facebook widgets or Google Maps. You can get rid of them by blacklisting calls to those sites:
block_urls = ['gstatic.com', 'google-analytics.com', 'tawk.to', 'perdeta.net', 'facebook.net', 'facebook.com'];
page.onResourceRequested = function(requestData, request){
for(url in block_urls) {
if(requestData.url.indexOf(block_urls[url]) !== -1) {
request.abort();
console.log(requestData.url + " aborted");
return;
}
}
}
If you implement this solution you will notice that "Here 1" is not printed anymore. That is because the first onLoadFinished is actually fired even before page.open but you create it only after page.open is called.
If you want to make real use of page.onLoadFinished you should decalre it even before page.open.
This is my very first question I am posting on StackOverflow. I usually tend to find answers to any question I have for whatever project, but I am really stuck on this one. I'm writing a Chrome extension that attempts to send JSON-objects over a HTTP post request with certain data in it to a basic Java server that will -in a later stage- process some of the data and return a HTTP response with relevant (processed) information for the client. This is the relevant JavaScript snippet on client-side:
function postRequest(jsonObject) {
var param = JSON.stringify(jsonObject);
var request = new XMLHttpRequest();
request.onreadystatechange = function () {
if (request.readyState == 4) {
console.log(request.response);
}
};
request.open("POST", "http://"+link);
request.setRequestHeader("Content-type","text/plain");
request.send(param);
// request.abort();
}
The input JSON-object is legit and the link-item is the predefined host IP of the server. The server-side code is Java-based and is the following:
public class Server {
public static void main(String args[]) throws Exception {
while (true) {
try {
ServerSocket ss = new ServerSocket(1024);
Socket s = ss.accept();
System.out.println("Request: ");
BufferedReader br = new BufferedReader(new InputStreamReader(s.getInputStream()));
PrintWriter pw = new PrintWriter(s.getOutputStream());
String line;
while ((line = br.readLine()) != null) {
if (line.length() == 0) {
break;
}
System.out.print(line + "\r\n");
}
System.out.println("");
System.out.println("Response:");
String response = "HTTP/1.1 200 OK \r\n" + "Access-Control-Allow-Origin: * \r\n"
+ "Connection: close \r\n" + "\r\n";
System.out.println(response);
pw.write(response);
while ((line = br.readLine()) != null) {
if (line.length() == 0) {
break;
}
System.out.print(line + "\r\n");
}
br.close();
pw.close();
System.out.println("");
System.out.println("Socket Closed");
System.out.println("");
s.close();
ss.close();
} catch (Exception ex) {
// Do whatever
}
}
}}
The server does receive the request and the JSON-object, but no response is received on client side (readyState is usually 4, but status code is 0 and responseText is empty). Additionally the connection is not closed when I don't use request.abort() or close the extension's window. It for sure has nothing to do with the permissions in the manifest file, so I didn't include it in this post. Can somebody see what my mistake is or anybody have had a similar experience?
EDIT:
The HTTP request is received on server side and looks like:
POST / HTTP/1.1
Host: [IP address of the server; I deleted the exact address for privacy]
Connection: keep-alive
Content-Length: 173
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36
Origin: chrome-extension://nhpingbdjkkljheamibjobneeehiednk
Content-type: text/plain
Accept: */*
Accept-Encoding: gzip, deflate
Accept-Language: nl-NL,nl;q=0.8,en-US;q=0.6,en;q=0.4,fr;q=0.2
Since the request is received on server side, I figured that the actual sending of the response is currently unsuccessful. Might it have something to do with the origin domain of the client? When I copy-paste that domain in my browser, I get a ERR_FILE_NOT_FOUND.
i have a cpp server using WinSock2 and im trying to connect to this server with my javascript client, and it doesnt work, the chrome console says "Error during WebSocket handshake: Incorrect 'Sec-WebSocket-Accept' header value".
i compared my sha1 and base64 functions with online sha1 and base64 so the problem isnt here.
Chrome Response Header:
HTTP/1.1 101 Switching Protocols
Upgrade: websocket
Connection: Upgrade
Sec-Websocket-Accept: NzdkYjg1Y2I4MDRlNTk0OGNmNzI1NzdjZDgwOTEwZWZiYWI1NzQ3Yw==
Chrome Request Header:
GET ws://localhost:8820/ HTTP/1.1
Host: localhost:8820
Connection: Upgrade
Pragma: no-cache
Cache-Control: no-cache
Upgrade: websocket
Origin: file://
Sec-WebSocket-Version: 13
User-Agent: Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36
Accept-Encoding: gzip, deflate, sdch, br
Accept-Language: he-IL,he;q=0.8,en-US;q=0.6,en;q=0.4
Sec-WebSocket-Key: Y7a2ZKEz/VCM92Wya49iPA==
Sec-WebSocket-Extensions: permessage-deflate; client_max_window_bits
Server Code:
//key is already defined.
key += "258EAFA5-E914-47DA-95CA-C5AB0DC85B11";
key = sha1(key);
key = base64_encode(reinterpret_cast<const unsigned char*>(key.c_str()), key.length());
toClient = "HTTP/1.1 101 Switching Protocols\r\n";
toClient += "Upgrade: websocket\r\n";
toClient += "connection: Upgrade\r\n";
toClient += "Sec-Websocket-Accept: ";
toClient += key;
toClient += "\r\n\r\n";
sendData(sc, toClient);
Client Code:
<!DOCTYPE HTML>
<script type="text/javascript">
function WebSocketTest()
{
if ("WebSocket" in window)
{
//alert("WebSocket is supported by your Browser!");
// Let us open a web socket
var ws = new WebSocket("ws://localhost:8820");
console.log("test");
ws.onopen = function()
{
alert("Connection.")
// Web Socket is connected, send data using send()
ws.send("20304user04user04user");
//alert("Message is sent...");
};
ws.onmessage = function (evt)
{
var received_msg = evt.data;
alert("Hey");
};
ws.onclose = function()
{
// websocket is closed.
alert("Connection is closed...");
};
}
else
{
// The browser doesn't support WebSocket
alert("WebSocket NOT supported by your Browser!");
}
}
</script>
<div id="sse">
Run WebSocket
</div>
I tried manually to build the response:
Raw sha1 digest bytes of the concatenation are 77db85cb804e5948cf72577cd80910efbab5747c
Bytes to base 64: d9uFy4BOWUjPcld82AkQ77q1dHw=
This is different than what you have. You are transforming the hexadecimal string representation of those bytes to base 64 instead of raw bytes to base 64.