We Keep Coding

curl get request brings no js page version

When I bring www.amazon.com with curl get request I get no js page version.
My request looks like this:
$ch = curl_init ($url );
$header[0] = "Accept: text/xml,application/xml,application/xhtml+xml,";
$header[0] .= "text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5";
$header[] = "Accept-Language: en-us,en;q=0.5";
$header[] = "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7";
$header[] = "Accept-Encoding: deflate";
curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
curl_setopt ($ch, CURLOPT_COOKIEJAR, $ckfile);
curl_setopt ($ch, CURLOPT_COOKIEFILE, $ckfile);
curl_setopt ($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt ($ch, CURLOPT_FOLLOWLOCATION,1);
curl_setopt ($ch, CURLOPT_SSL_VERIFYPEER,0);
$res = curl_exec($ch);

How to emulate Ajax request using PHP CURL?

I am getting 400 http_code error when I send a Ajax request with CURL below is my Code.
$header = array(
"Accept : application/json, text/javascript, */*; q=0.01",
"Accept-Encoding : gzip, deflate",
"Accept-Language : en-US,en;q=0.5",
"Content-Type : application/json; charset=UTF-8",
"Host : https://some.com",
"Referer : https://some.com/dashboard/reports",
"X-Requested-With : XMLHttpRequest"
);
$c = curl_init('https://domain.com/report.php');
//curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false);
//curl_setopt($c, CURLOPT_VERBOSE, true);
curl_setopt($c, CURLOPT_USERAGENT, $_SERVER['HTTP_USER_AGENT']);
curl_setopt($c, CURLOPT_RETURNTRANSFER, 1);
curl_setopt( $c, CURLOPT_POST, true );
curl_setopt( $c, CURLOPT_POSTFIELDS, $data_url );
curl_setopt($c, CURLOPT_HTTPHEADER, $header);
//curl_setopt($ch, CURLOPT_HTTPHEADER, array("X-Requested-With: XMLHttpRequest"));
curl_setopt($c, CURLOPT_COOKIEFILE, $cookie_jar);
$page = curl_exec($c);
$httpcode = curl_getinfo($c);
// I am getting following response after making Curl request
Array
(
[url] => some_url
[content_type] =>
[http_code] => 400
[header_size] => 70
[request_size] => 935
[filetime] => -1
[ssl_verify_result] => 0
[redirect_count] => 0
[total_time] => 0.205562
[namelookup_time] => 0.000132
[connect_time] => 0.032866
[pretransfer_time] => 0.170225
[size_upload] => 272
[size_download] => 0
[speed_download] => 0
[speed_upload] => 1323
[download_content_length] => 0
[upload_content_length] => 272
[starttransfer_time] => 0.205498
[redirect_time] => 0
[certinfo] => Array
(
)
[primary_ip] => 66.35.58.70
[primary_port] => 443
[local_ip] => 198.1.92.85
[local_port] => 53627
[redirect_url] =>
)
////

I am not sure it would help you but i am able to successfully query SSL site with the following code.
There are several things like user agent, cookiefile, SSL verify, url form encoded for post data and subsequent request should use the same cookie and agent data
$config['useragent'] = 'Mozilla/5.0 (Windows NT 6.2; WOW64; rv:17.0) Gecko/20100101 Firefox/17.0';
$ch = curl_init();
curl_setopt($ch,CURLOPT_URL,$url);
curl_setopt($ch, CURLOPT_USERAGENT, $config['useragent']);
curl_setopt($ch, CURLOPT_REFERER, 'http://google.com/');
$dir = dirname(__FILE__);
$config['cookie_file'] = $dir."/".md5($_SERVER['REMOTE_ADDR']) . '.txt';
curl_setopt($ch, CURLOPT_COOKIEFILE, $config['cookie_file']);
curl_setopt($ch, CURLOPT_COOKIEJAR, $config['cookie_file']);
curl_setopt($ch, CURLOPT_VERBOSE, 1);
curl_setopt($ch, CURLOPT_NOBODY, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch,CURLOPT_HEADER,0);
$html=curl_exec($ch);
Subsequent call request(note it is using the cookie, useragent generated in first request)
$post = 'city_id=3&length-3';
$headers = array();
$headers[] = 'Accept: application/json, text/javascript, */*; q=0.01';
$headers[] = 'Accept-Language: en-US,en;q=0.5';
$headers[] = 'Content-Length:'.strlen($post);
$headers[] = 'Content-Type: application/x-www-form-urlencoded; charset=utf-8';
$headers[] = 'X-Requested-With: XMLHttpRequest';
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
$ch = curl_init();
curl_setopt($ch,CURLOPT_URL,$link);
curl_setopt($ch, CURLOPT_USERAGENT, $config['useragent']);
curl_setopt($ch, CURLOPT_COOKIEFILE, $config['cookie_file']);
curl_setopt($ch, CURLOPT_COOKIEJAR, $config['cookie_file']);
curl_setopt($ch, CURLOPT_VERBOSE, 1);
curl_setopt($ch, CURLOPT_NOBODY, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch,CURLOPT_HEADER,0);
$linkhtml=curl_exec($ch);

Problems by using cURL from PHP for scraping source code

i try to automate a download from a HTML-datasheet to generate a customized reporting. The following i was doing with CURL:
// init cURL HTTP Client
$header = array();
$header[0] = "Accept: text/xml,application/xml,application/xhtml+xml,";
$header[0] .= "text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5";
$header[] = "Cache-Control: max-age=0";
$header[] = "Connection: keep-alive";
$header[] = "Keep-Alive: 300";
$header[] = "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7";
$header[] = "Accept-Language: en-us,en;q=0.5";
$header[] = "Pragma: ";
$ch = curl_init();
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US; rv:1.8.1.7) Gecko/20070914 Firefox/2.0.0.7');
curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
curl_setopt($ch, CURLOPT_COOKIEFILE, '/.cookies');
curl_setopt($ch, CURLOPT_COOKIEJAR, '/.cookies');
curl_setopt($ch, CURLOPT_AUTOREFERER, TRUE);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, TRUE);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
curl_setopt($ch, CURLOPT_FAILONERROR, TRUE);
curl_setopt($ch, CURLOPT_HEADER, TRUE);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 600);
curl_setopt($ch, CURLOPT_URL, 'https:// ... /signin.html');
curl_setopt($ch, CURLOPT_POST, TRUE);
curl_setopt($ch, CURLOPT_POSTFIELDS, "username=".$login."&password=".$pass);
$response = curl_exec($ch);
The login works fine and i can get many pages without any problems. Now i try to get the datasheet by the following:
curl_setopt($ch, CURLOPT_URL, 'https:// ... /data.html');
curl_setopt($ch, CURLOPT_POST, FALSE);
curl_setopt($ch, CURLOPT_POSTFIELDS, '');
$response = curl_exec($ch);
But now i get the following answer:
<html>
<head>
<script language='javascript'>function autoNavigate() {window.location="/data.html";}</script>
</head>
<body onload='autoNavigate()'></body>
</html>
The javaScript call refresh the same page as i loaded before. In a browser it works fine, but if i load the same page again with "curl_exec($ch)" i've got a 302-error?
Is there a possibilty the refresh the page with curl without a full reload? Or any other idea to get the content of the page?
Thanks

try:
$postfields = '';
curl_setopt($ch, CURLOPT_URL, 'https:// ... /data.html');
curl_setopt($ch, CURLOPT_POST, TRUE);
curl_setopt($ch, CURLOPT_POSTFIELDS, $postfields);
$response = curl_exec($ch);
It creates problem when you set false the CURLOPT_POSTFIELDS value but earlier you set it as True bacause it holds the previous details in Cookie.
I hope this will helpful for you.

Did you check the link of data.html?
If the data.html in window.location="data.html"; is the same location of data.html in curl_setopt($ch, CURLOPT_URL, 'https:// ... /data.html'); try to double curl_exec($ch) so may be it needs to access two times. Or if it different, just simple change your link.

get dynamic generated content after using curl to login

I know that curl does not execute javascript, it only grabs static html, so this is why a simple curl will not work for me.
I do not know much about php, I'm new to this, but what I understand so for is that if I did not have to first login to grab the content I can simple use file_get_contents witch will first execute the dynamic content and then grab the html content, witch in return give me what I need, but I first have to login and then get the page.
I tried to login using curl
$user = "myuser";
$pass = "mypassword";
//create cookie file
$random = rand(0,9999999);
$cookie = $random."cookie.txt";
$fp = fopen("$cookie","w") or die("<BR><B>Unable to open cookie file $cookie_file_path for write!<BR>");
fclose($fp);
//do login using curl
$LOGINURL = "https://controlpanel.example.com/index.html";
$agent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:29.0) Gecko/20120101 Firefox/29.0";
$v2 = array( 'userName'=>$user, 'password'=>$pass);
$reffer = "https://www.google.com";
//this first call is to set the cookie
$ch = curl_init();
curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie);
curl_setopt($ch, CURLOPT_URL,$LOGINURL);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_USERAGENT, $agent);
ob_start(); // Prevent output
curl_exec ($ch);
ob_end_clean(); // End preventing output
curl_close ($ch);
unset($ch);
//now that the cookie is set, do login
$ch = curl_init();
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS,$v2);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_COOKIEFILE, $cookie);
curl_setopt($ch, CURLOPT_URL,$LOGINURL);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_REFERER, $reffer);
curl_setopt($ch, CURLOPT_USERAGENT, $agent);
$result = curl_exec($ch);
//now we are logged-in
//now grab the page you need
$profileurl = 'https://controlpanel.example.com/information.html';
curl_setopt($ch, CURLOPT_URL, $profileurl);
curl_setopt($ch, CURLOPT_POST, 0);
curl_setopt($ch, CURLOPT_TIMEOUT, 5);
$result = curl_exec ($ch);
But this will only get the static html, not the dynamic content too.
Let me explain better.
The code I get, at this point using above curl method, in $result is:
.....
<div id="DisplayAccountInfo"><span class="loading">Loading info</span></div>
.....
If I do this manually using firefox and inspect element with firebug the source is:
.....
<div id="DisplayAccountInfo">
<div class="formModule" id="formContainer">
......
<legend>Your code for this hour is 8T5D9LO</legend>
.....
</div>
</div>
.....
What I notice in firebug console is:
GET https://controlpanel.example.com/async/information.html
200 OK
669ms
jquery-....min.js (line 19)
What I, as a noob, understand from this is that the content is dinamicly loaded using jquery, and curl does not know how to do that.
I tried to put instead of
$profileurl = 'https://controlpanel.example.com/information.html';
curl_setopt($ch, CURLOPT_URL, $profileurl);
curl_setopt($ch, CURLOPT_POST, 0);
curl_setopt($ch, CURLOPT_TIMEOUT, 5);
$result = curl_exec ($ch);
//replaced the above with this
$result = file_get_contents($profileurl);
but I get the html from login page because I think it does not recognize anymore that I'm logged in.
So how can I solve this? Can you please help me?

I think I got what you're doing.
The key point here is, most website handle login with cookie. In https://controlpanel.example.com/information.html, if the website set a cookie after you login in your browser, then the good news is you can solve this problem.
The problem in your code is, PHP won't set cookie for you.
You need 2 steps:
Step 1. You need to obtain the cookie when your php curl the login
Here's how you get cookie header returned from the login page.
$ch = curl_init('https://controlpanel.example.com/index.html');
....
$result = curl_exec($ch);
preg_match('/^Set-Cookie:\s*([^;]*)/mi', $result, $m);
parse_str($m[1], $cookies);
echo $cookies;//See if you've successfully obtained the return cookie
Step 2. You access https://controlpanel.example.com/information.html with the cookie you obtained in step 1. (like you've already did in your own code)

haha, so easy it did not cross my mind.
For me it is simple, I did not have to call
https://controlpanel.example.com/information.html
but
https://controlpanel.example.com/async/information.html
to get the div I wanted :)
Lucky for me I noticed the get function in firebug :)
So the cod now is :
$user = "myuser";
$pass = "mypassword";
//create cookie file
$random = rand(0,9999999);
$cookie = $random."cookie.txt";
$fp = fopen("$cookie","w") or die("<BR><B>Unable to open cookie file $cookie for write!<BR>");
fclose($fp);
//do login using curl
$LOGINURL = "https://controlpanel.example.com/index.html";
$agent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:29.0) Gecko/20120101 Firefox/29.0";
$v2 = array( 'userName'=>$user, 'password'=>$pass);
$reffer = "https://www.google.com";
//this first call is to set the cookie
$ch = curl_init();
curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie);
curl_setopt($ch, CURLOPT_URL,$LOGINURL);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_USERAGENT, $agent);
ob_start(); // Prevent output
curl_exec ($ch);
ob_end_clean(); // End preventing output
curl_close ($ch);
unset($ch);
//now that the cookie is set, do login
$ch = curl_init();
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS,$v2);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_COOKIEFILE, $cookie);
curl_setopt($ch, CURLOPT_URL,$LOGINURL);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_REFERER, $reffer);
curl_setopt($ch, CURLOPT_USERAGENT, $agent);
$result = curl_exec($ch);
//now we are logged-in
//now grab the page you need
$profileurl = 'https://controlpanel.example.com/async/information.html';
curl_setopt($ch, CURLOPT_URL, $profileurl);
curl_setopt($ch, CURLOPT_POST, 0);
$result = curl_exec ($ch);

Uploading photo using Appcelerator Cloud Using PHP

I am trying to request the Appcelerator Cloud for creating the user with photo, the user is getting created without photo , but when i attach the photo with request i get this error
This Code Gives me error "{ "meta": { "status": "fail", "code": 400, "message": "Failed to upload photo: Failed to indentify photo file" } }"
Please Help me with this , i m new to PHP
<?php
$agent = "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7.12) Gecko/20050915 Firefox/1.0.7";
$data = json_encode(array(
'securitycode' => $_POST['code'],
'designation' => $_POST['desi'],
'comapanyname' => $_POST['cname'],
'companylogoid' => $_POST['clogoid'],
'companyurl' => $_POST['curl'],
'location' => $_POST['location'],
'emailprivateflag' => $_POST['emailp'],
'linkedinurl' => $_POST['linkedin'],
'twitterhandle' => $_POST['twitter'],
'isspeaker' => $_POST['isspeaker'],
'allowaccess' => $_POST['access']
));
//$photo = array('photo' => '#' . $_FILES['file']['name'][0]);
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL,"https://api.cloud.appcelerator.com/v1/users/login.json?key=LJOVi9A95Y3r6TqlFmxS314v6ox4xaPf");
curl_setopt($ch, CURLOPT_USERAGENT, $agent);
curl_setopt ($ch, CURLOPT_HTTPHEADER, Array("Content-Type: application/x-www-form-urlencoded","Accept: */*"));
curl_setopt($ch, CURLOPT_COOKIEFILE, 'cookie1.txt');
curl_setopt($ch, CURLOPT_COOKIEJAR, 'cookie1.txt');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt( $ch, CURLOPT_AUTOREFERER, 1);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS,"login=test#test.com&password=test");
// $html=curl_exec($ch);
$ch1 = curl_init();
curl_setopt($ch1, CURLOPT_URL,"https://api.cloud.appcelerator.com/v1/users/create.json?key=LJOVi9A95Y3r6TqlFmxS314v6ox4xaPf");
curl_setopt($ch1, CURLOPT_USERAGENT, $agent);
curl_setopt ($ch1, CURLOPT_HTTPHEADER, Array('Content-Type: multipart/form-data',"Accept: */*"));
// curl_setopt($ch1, CURLOPT_COOKIEFILE,'cookie2.txt');
// curl_setopt($ch1, CURLOPT_COOKIEJAR, 'cookie2.txt');
curl_setopt($ch1, CURLOPT_RETURNTRANSFER, 1);
curl_setopt( $ch1, CURLOPT_AUTOREFERER, 1);
curl_setopt($ch1, CURLOPT_FOLLOWLOCATION, 1);
curl_setopt($ch1, CURLOPT_SSL_VERIFYPEER, 0);
curl_setopt($ch1, CURLOPT_POST, true);
curl_setopt($ch1, CURLOPT_POSTFIELDS, 'email='.$_POST['email'].'&first_name='.$_POST['fname'].'&last_name='.$_POST['lname'].'&password=ciosummit&password_confirmation=ciosummit&photo=#'. $_FILES['file']['tmp_name'][0].'&custom_fields='.$data);
$html=curl_exec($ch1);
header('Location:index.php?status='. rawurlencode($html));
//echo $html;
?>

You need to send the data in an array just like this:
$post = array('photo' => '#YOURIMAGENAME',
'email'=>$_POST['email'],
'first_name'=>$_POST['fname'],
'last_name'=>$_POST['lame'],
'password'=>'ciosummit',
'password_confirmation'=>'ciosummit',
'custom_fields'=>$data
);
And then send the curl values like:
curl_setopt($ch1, CURLOPT_POSTFIELDS,$post);