加快cURL页面登录和抓取[重复]

Question

This question already has an answer here:

• php - Fastest way to check presence of text in many domains (above 1000) 1 answer

I have a function that logins into a site and searches for a string in the following page. The process currently takes 10 seconds, but wanted to see if there was anything I could do to speed it up. I wonder if was possible to have the curl login persist over clients session or maybe search the document better.

public function curlLogin($url, $post_values, $cookieJar) {

        $timeout = 30;

        $curl_connection = curl_init();
        curl_setopt($curl_connection, CURLOPT_URL, $url);
        curl_setopt($curl_connection, CURLOPT_TIMEOUT, $timeout);
        curl_setopt($curl_connection, CURLOPT_USERAGENT,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)");
        curl_setopt($curl_connection, CURLOPT_COOKIEJAR, $cookieJar);
        curl_setopt($curl_connection, CURLOPT_COOKIEFILE, $cookieJar);
        curl_setopt($curl_connection, CURLOPT_COOKIESESSION, 0);
        curl_setopt($curl_connection, CURLOPT_HEADER, 1);
        curl_setopt($curl_connection, CURLOPT_RETURNTRANSFER, 1);
        curl_setopt($curl_connection, CURLOPT_SSL_VERIFYPEER, 0);
        curl_setopt($curl_connection, CURLOPT_POST, 1);
        curl_setopt($curl_connection, CURLOPT_POSTFIELDS, $post_values);
        curl_setopt($curl_connection, CURLOPT_HTTPHEADER,
        array("Content-type: application/x-www-form-urlencoded"));
        curl_exec($curl_connection);
        return $curl_connection;

    }

    public function curlPost($curl_connection, $url, $post_values, $cookieJar) {

        $timeout = 30;

        curl_setopt($curl_connection, CURLOPT_URL, $url);
        curl_setopt($curl_connection, CURLOPT_TIMEOUT, $timeout);
        curl_setopt($curl_connection, CURLOPT_USERAGENT,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)");
        curl_setopt($curl_connection, CURLOPT_COOKIEJAR, $cookieJar);
        curl_setopt($curl_connection, CURLOPT_COOKIEFILE, $cookieJar);
        curl_setopt($curl_connection, CURLOPT_COOKIESESSION, 0);
        curl_setopt($curl_connection, CURLOPT_HEADER, 1);
        curl_setopt($curl_connection, CURLOPT_RETURNTRANSFER, 1);
        curl_setopt($curl_connection, CURLOPT_SSL_VERIFYPEER, 0);
        curl_setopt($curl_connection, CURLOPT_POST, 1);
        curl_setopt($curl_connection, CURLOPT_POSTFIELDS, $post_values);
        curl_setopt($curl_connection, CURLOPT_HTTPHEADER,
        array("Content-type: application/x-www-form-urlencoded"));
        $result = curl_exec($curl_connection);
        return $result;

    }

$cookieJar = tempnam ("/tmp", "CURLCOOKIE");

$curl_connection = $this->curlLogin($login_url, $post_values, $cookieJar);

$result = $this->curlPost($curl_connection, $next_url, $params, $cookieJar);

if (strpos($result,'string 1') > 0) {
    $success = true;
    $message = 'string 1 is present';
}else if (strpos($result,'string 2') > 0){
    $success = false;
    $message = 'string 2 is present';
}else if (strpos($result,'string 3') > 0){
    $success = false;
    $message = 'string 3 is present';
}else{
    $success = false;
    $message = 'None of the above strings are present.';
}

curl_close($curl_connection);
unlink($cookieJar);

</div>

Answer 1

You can avoid logging in every time by reusing your cookiejar.

Create a file called cookies.txt in the directory containing your script and assign: $cookieJar = 'cookies.txt'.

After running the script for the first time, simply remove call to the curlLogin() function and your curlPost() function should use the cookies correctly and return data as if you were logged in.

Remember, CURLOPT_COOKIEFILE is to specify where to "read" cookies from and CURLOPT_COOKIEJAR is where you want the response cookies to be written.

So you could probably do without CURLOPT_COOKIEJAR in your curlPost() function.