基于PHP的cURL快速入门教程 (小偷采集程序)

$content = file_get_contents("http://www.jb51.net"); // or$lines = file("http://www.jb51.net");
// or
readfile(http://www.jb51.net);

// 1. 初始化
$ch = curl_init(); // 2. 设置选项，包括URL curl_setopt($ch, CURLOPT_URL, "http://www.jb51.net");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); curl_setopt($ch, CURLOPT_HEADER, 0);
// 3. 执行并获取HTML文档内容
$output = curl_exec($ch);
// 4. 释放curl句柄
curl_close($ch);

// ...
$output = curl_exec($ch);
if ($output === FALSE) {     echo "cURL Error: " . curl_error($ch);
}
// ...

// ...
curl_exec($ch);$info = curl_getinfo($ch); echo '获取'.$info['url'] . '耗时'. $info['total_time'] . '秒';
// ...

// 测试用的URL
$urls = array(     "http://www.cnn.com",     "http://www.mozilla.com",     "http://www.facebook.com" ); // 测试用的浏览器信息$browsers = array(
    "standard" => array (
        "user_agent" => "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 (.NET CLR 3.5.30729)",
        "language" => "en-us,en;q=0.5"
        ),
    "iphone" => array (
        "user_agent" => "Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1A537a Safari/419.3",
        "language" => "en"
        ),
    "french" => array (
        "user_agent" => "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; GTB6; .NET CLR 2.0.50727)",
        "language" => "fr,fr-FR;q=0.5"
        )
);
foreach ($urls as$url) {
    echo "URL: $url\n";     foreach ($browsers as $test_name =>$browser) {
        $ch = curl_init();         // 设置 url         curl_setopt($ch, CURLOPT_URL, $url);         // 设置浏览器的特定header         curl_setopt($ch, CURLOPT_HTTPHEADER, array(
                "User-Agent: {$browser['user_agent']}",                 "Accept-Language: {$browser['language']}"             ));         // 页面内容我们并不需要         curl_setopt($ch, CURLOPT_NOBODY, 1);
        // 只需返回HTTP header
        curl_setopt($ch, CURLOPT_HEADER, 1);         // 返回结果，而不是输出它         curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
        $output = curl_exec($ch);
        curl_close($ch);         // 有重定向的HTTP头信息吗?         if (preg_match("!Location: (.*)!",$output, $matches)) {             echo "$test_name: redirects to $matches[1]\n";         } else {             echo "$test_name: no redirection\n";
        }
    }
    echo "\n\n";
}

$url = "http://localhost/post_output.php";$post_data = array (
    "foo" => "bar",
    "query" => "Nettuts",
    "action" => "Submit"
);
$ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
// 我们在POST数据哦！
curl_setopt($ch, CURLOPT_POST, 1); // 把post的变量加上 curl_setopt($ch, CURLOPT_POSTFIELDS, $post_data);$output = curl_exec($ch); curl_close($ch);
echo $output;

$url = "http://localhost/upload_output.php";$post_data = array (
    "foo" => "bar",
    // 要上传的本地文件地址
    "upload" => "@C:/wamp/www/test.zip"
);
$ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POST, 1); curl_setopt($ch, CURLOPT_POSTFIELDS, $post_data);$output = curl_exec($ch); curl_close($ch);
echo $output;

// 创建两个cURL资源
$ch1 = curl_init();$ch2 = curl_init();
// 指定URL和适当的参数
curl_setopt($ch1, CURLOPT_URL, "http://lxr.php.net/"); curl_setopt($ch1, CURLOPT_HEADER, 0);
curl_setopt($ch2, CURLOPT_URL, "http://www.php.net/"); curl_setopt($ch2, CURLOPT_HEADER, 0);
// 创建cURL批处理句柄
$mh = curl_multi_init(); // 加上前面两个资源句柄 curl_multi_add_handle($mh,$ch1); curl_multi_add_handle($mh,$ch2); // 预定义一个状态变量$active = null;
// 执行批处理
do {
    $mrc = curl_multi_exec($mh, $active); } while ($mrc == CURLM_CALL_MULTI_PERFORM);
while ($active &&$mrc == CURLM_OK) {
    if (curl_multi_select($mh) != -1) {         do {             $mrc = curl_multi_exec($mh, $active);         } while ($mrc == CURLM_CALL_MULTI_PERFORM);     } } // 关闭各个句柄 curl_multi_remove_handle($mh, $ch1); curl_multi_remove_handle($mh, $ch2); curl_multi_close($mh);

// CONFIG
$db_host = 'localhost';$db_user = 'root';
$db_pass = '';$db_name = 'wordpress';
$excluded_domains = array(     'localhost', 'www.mydomain.com');$max_connections = 10;
// 初始化一些变量
$url_list = array();$working_urls = array();
$dead_urls = array();$not_found_urls = array();
$active = null; // 连到 MySQL if (!mysql_connect($db_host, $db_user,$db_pass)) {
    die('Could not connect: ' . mysql_error());
}
if (!mysql_select_db($db_name)) {     die('Could not select db: ' . mysql_error()); } // 找出所有含有链接的文章$q = "SELECT post_content FROM wp_posts
    WHERE post_content LIKE '%href=%'
    AND post_status = 'publish'
    AND post_type = 'post'";
$r = mysql_query($q) or die(mysql_error());
while ($d = mysql_fetch_assoc($r)) {
    // 用正则匹配链接
    if (preg_match_all("!href=\"(.*?)\"!", $d['post_content'],$matches)) {
        foreach ($matches[1] as$url) {
            // exclude some domains
            $tmp = parse_url($url);
            if (in_array($tmp['host'],$excluded_domains)) {
                continue;
            }
            // store the url
            $url_list []=$url;
        }
    }
}
// 移除重复链接
$url_list = array_values(array_unique($url_list));
if (!$url_list) {
    die('No URL to check');
}

// 1. 批处理器
$mh = curl_multi_init(); // 2. 加入需批量处理的URL for ($i = 0; $i <$max_connections; $i++) {     add_url_to_multi_handle($mh, $url_list); } // 3. 初始处理 do {     $mrc = curl_multi_exec($mh, $active); } while ($mrc == CURLM_CALL_MULTI_PERFORM);
// 4. 主循环
while ($active &&$mrc == CURLM_OK) {
    // 5. 有活动连接
    if (curl_multi_select($mh) != -1) {         // 6. 干活         do {             $mrc = curl_multi_exec($mh, $active);         } while ($mrc == CURLM_CALL_MULTI_PERFORM);         // 7. 有信息否？         if ($mhinfo = curl_multi_info_read($mh)) {             // 意味着该连接正常结束             // 8. 从curl句柄获取信息             $chinfo = curl_getinfo($mhinfo['handle']);             // 9. 死链么？             if (!$chinfo['http_code']) {                 $dead_urls []= $chinfo['url'];             // 10. 404了?             } else if ($chinfo['http_code'] == 404) {                 $not_found_urls []= $chinfo['url'];             // 11. 还能用             } else {                 $working_urls []= $chinfo['url'];             }             // 12. 移除句柄             curl_multi_remove_handle($mh, $mhinfo['handle']);             curl_close($mhinfo['handle']);             // 13. 加入新URL，干活             if (add_url_to_multi_handle($mh, $url_list)) {                 do {                     $mrc = curl_multi_exec($mh, $active);                 } while ($mrc == CURLM_CALL_MULTI_PERFORM);             }         }     } } // 14. 完了 curl_multi_close($mh);
echo "==Dead URLs==\n";
echo implode("\n",$dead_urls) . "\n\n"; echo "==404 URLs==\n"; echo implode("\n",$not_found_urls) . "\n\n";
echo "==Working URLs==\n";
echo implode("\n",$working_urls); // 15. 向批处理器添加url function add_url_to_multi_handle($mh, $url_list) {     static$index = 0;
    // 如果还剩url没用
    if ($url_list[$index]) {
        // 新建curl句柄
        $ch = curl_init();         // 配置url         curl_setopt($ch, CURLOPT_URL, $url_list[$index]);
        // 不想输出返回的内容
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);         // 重定向到哪儿我们就去哪儿         curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
        // 不需要内容体，能够节约带宽和时间
        curl_setopt($ch, CURLOPT_NOBODY, 1);         // 加入到批处理器中         curl_multi_add_handle($mh, $ch);         // 拨一下计数器，下次调用该函数就能添加下一个url了$index++;
        return true;
    } else {
        // 没有新的URL需要处理了
        return false;
    }
}

<img border="0" src="http://files.jb51.net/upload/201106/20110602225008534.png" />

$url = "http://www.somesite.com/members/";$ch = curl_init();
curl_setopt($ch, CURLOPT_URL,$url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); // 发送用户名和密码 curl_setopt($ch, CURLOPT_USERPWD, "myusername:mypassword");
// 你可以允许其重定向
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1); // 下面的选项让 cURL 在重定向后 // 也能发送用户名和密码 curl_setopt($ch, CURLOPT_UNRESTRICTED_AUTH, 1);
$output = curl_exec($ch);
curl_close($ch);

// 开一个文件指针
$file = fopen("/path/to/file", "r"); // url里包含了大部分所需信息$url = "ftp://username:password@mydomain.com:21/path/to/new/file";
$ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
// 上传相关的选项
curl_setopt($ch, CURLOPT_UPLOAD, 1); curl_setopt($ch, CURLOPT_INFILE, $fp); curl_setopt($ch, CURLOPT_INFILESIZE, filesize("/path/to/file"));
// 是否开启ASCII模式 (上传文本文件时有用)
curl_setopt($ch, CURLOPT_FTPASCII, 1);$output = curl_exec($ch); curl_close($ch);

$ch = curl_init(); curl_setopt($ch, CURLOPT_URL,'http://www.example.com');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); // 指定代理地址 curl_setopt($ch, CURLOPT_PROXY, '11.11.11.11:8080');
// 如果需要的话，提供用户名和密码
curl_setopt($ch, CURLOPT_PROXYUSERPWD,'user:pass');$output = curl_exec($ch); curl_close ($ch);

$ch = curl_init(); curl_setopt($ch, CURLOPT_URL,'http://net.tutsplus.com');
curl_setopt($ch, CURLOPT_WRITEFUNCTION,"progress_function"); curl_exec($ch);
curl_close ($ch); function progress_function($ch,$str) {     echo$str;
    return strlen($str);
}