httpclient怎么才可以不遵守robots.txt

Question

想抓大众点评网的数据，发现他用了robots.txt,用httpclient取不到某一页的代码了.
希望有人能解决。本人全部分数放送.给能解决的人
[b]问题补充：[/b]
最好是用java的httpclient,因为我这里的程序就是java的.
只是想突破那个限制.
既然页面在网上公开显示着，不可能不能抓的
[b]问题补充：[/b]
下面就是得到一个URL对应网页代码的程序,希望改造一下,就可以突破限制
public static String getWebContentGetMethod( String url, String coding ){
url = checkUrl(url) ;
if( StringProcessor.isEmpty(url)){
return null ;
}
//构造HttpClient的实例
HttpClient httpClient = new HttpClient();
// 创建GET方法的实例
GetMethod getMethod = new GetMethod( url );
// 使用系统提供的默认的恢复策略
getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
new DefaultHttpMethodRetryHandler());
try {
// 执行getMethod
int statusCode = httpClient.executeMethod(getMethod);
if (statusCode != HttpStatus.SC_OK) {
System.err.println("Method failed: "
+ getMethod.getStatusLine());
}

        // 读取内容
        byte[] responseBody = getMethod.getResponseBody();
        // 处理内容
        String rs = new String(responseBody , coding );

        return rs ;
    } catch (HttpException e) {
        // 发生致命的异常，可能是协议不对或者返回的内容有问题
        //System.out.println("Please check your provided http address!");
        //e.printStackTrace();
    } catch (IOException e) {
        //// 发生网络异常
        //e.printStackTrace();
    } finally {
        // 释放连接
        getMethod.releaseConnection();
    }
    return null ;
}

Answer 1

import java.io.IOException;

import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;

public class GetSample{
public static void main(String[] args) {
//构造HttpClient的实例
HttpClient httpClient = new HttpClient();
//创建GET方法的实例
GetMethod getMethod = new GetMethod("http://www.dianping.com");
[color=red]getMethod.setRequestHeader( "User-Agent", "fake"); [/color]
//使用系统提供的默认的恢复策略
getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
new DefaultHttpMethodRetryHandler());
try {
//执行getMethod
int statusCode = httpClient.executeMethod(getMethod);
if (statusCode != HttpStatus.SC_OK) {
System.err.println("Method failed: "
+ getMethod.getStatusLine());
}
//读取内容
byte[] responseBody = getMethod.getResponseBody();
//处理内容
System.out.println(new String(responseBody));
} catch (HttpException e) {
//发生致命的异常，可能是协议不对或者返回的内容有问题
System.out.println("Please check your provided http address!");
e.printStackTrace();
} catch (IOException e) {
//发生网络异常
e.printStackTrace();
} finally {
//释放连接
getMethod.releaseConnection();
}
}
}