Jun 15, 2017
0
Level 6
How to get more contents from url using curl
HI I have a function to get the contents from a url using the curl and its called file_get_contents_curl($url); Now the issue is I am getting only limited contents what we want to do is download more content, possible couple of paragraph and save it in our database.
function file_get_contents_curl($url) {
$agent = 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.3705; .NET CLR 1.1.4322)';
$ch = curl_init();
curl_setopt($ch, CURLOPT_HEADER, 1);
curl_setopt($ch, CURLOPT_COOKIEJAR, 'cookie.txt');
curl_setopt($ch, CURLOPT_COOKIEFILE, 'cookie.txt');
curl_setopt($ch, CURLOPT_COOKIESESSION, 1);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_URL, $url);
//curl_setopt($ch, CURLOPT_REFERER, $url);
//curl_setopt($ch, CURLOPT_URL, 'https://graph.facebook.com/v1.0/?id='. urlencode($url). '&scrape=1');
//curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
//curl_setopt($ch, CURLOPT_MAXREDIRS,10);
//curl_setopt($ch, CURLOPT_NOBODY, true);
//curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');
curl_setopt($ch, CURLOPT_USERAGENT, $agent);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
//curl_setopt($ch, CURLOPT_VERBOSE, true);
$data = curl_exec($ch);
$info = curl_getinfo($ch, CURLINFO_CONTENT_TYPE);
//$info = curl_getinfo($ch) ;
$err = curl_error($ch);
$http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
$eff_url = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);
//checking mime types
if(strstr($info,'text/html')) {
curl_close($ch);
return $data;
} elseif(strstr($info,'text/plain')) {
curl_close($ch);
$data = shell_exec ('/usr/bin/curl' . ' ' . $url);
return $data;
} else {
return false;
}
}
Please assist how will I achieve this behavior. Thank you.
Please or to participate in this conversation.