java 网页页面抓取标题和正文

Question

现在公司需要做一个功能。就是给个url地址。然后读出这个页面中的标题和正文。将其保存到数据库中。请问各位高手怎么做？有源码吗？先谢谢了！
[b]问题补充：[/b]
各位，我现在想要的是源码？你们说的这个控件我没有时间去学习他了。没有时间了！
[b]问题补充：[/b]
现在公司需要做一个功能。就是给个url地址。然后读出这个页面中的标题和正文。将其保存到数据库中。请问各位高手怎么做？有源码吗？先谢谢了！
[b]问题补充：[/b]
各位，我现在想要的是源码？你们说的这个控件我没有时间去学习他了。没有时间了！
各位我的问题是无法用第一正则表达式的因为我不是只读一个页面。

Answer 1

[code="java"]
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class WebContent
{
/**

• 读取一个网页全部内容
  */
  public String getOneHtml(final String htmlurl) throws IOException
  {
  URL url;
  String temp;
  final StringBuffer sb = new StringBuffer();
  try
  {
  url = new URL(htmlurl);
  final BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream(), "utf-8"));// 读取网页全部内容
  while ((temp = in.readLine()) != null)
  {
  sb.append(temp);
  }
  in.close();
  }
  catch (final MalformedURLException me)
  {
  System.out.println("你输入的URL格式有问题！请仔细输入");
  me.getMessage();
  throw me;
  }
  catch (final IOException e)
  {
  e.printStackTrace();
  throw e;
  }
  return sb.toString();
  }

  /**

• @param s

• @return 获得网页标题
  /
  public String getTitle(final String s)
  {
  String regex;
  String title = "";
  final List list = new ArrayList();
  regex = ".?";
  final Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
  final Matcher ma = pa.matcher(s);
  while (ma.find())
  {
  list.add(ma.group());
  }
  for (int i = 0; i < list.size(); i++)
  {
  title = title + list.get(i);
  }
  return outTag(title);
  }

  /**

• @param s

• @return 获得链接
  /
  public List getLink(final String s)
  {
  String regex;
  final List list = new ArrayList();
  regex = "]*href=(\"([^\"])\"|\'([^\']*)\'|([^\s>]*))[^>]*>(.*?)";
  final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
  final Matcher ma = pa.matcher(s);
  while (ma.find())
  {
  list.add(ma.group());
  }
  return list;
  }

  /**

• @param s

• @return 获得脚本代码
  /
  public List getScript(final String s)
  {
  String regex;
  final List list = new ArrayList();
  regex = "<script.?";
  final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
  final Matcher ma = pa.matcher(s);
  while (ma.find())
  {
  list.add(ma.group());
  }
  return list;
  }

  /**

• @param s

• @return 获得CSS
  /
  public List getCSS(final String s)
  {
  String regex;
  final List list = new ArrayList();
  regex = "<style.?";
  final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
  final Matcher ma = pa.matcher(s);
  while (ma.find())
  {
  list.add(ma.group());
  }
  return list;
  }

  /**

• @param s

• @return 去掉标记
  /
  public String outTag(final String s)
  {
  return s.replaceAll("<.?>", "");
  }

  /**

• @param s

• @return 获取雅虎知识堂文章标题及内容
  /
  public HashMap getFromYahoo(final String s)
  {
  final HashMap hm = new HashMap();
  final StringBuffer sb = new StringBuffer();
  String html = "";
  System.out.println("\n------------------开始读取网页(" + s + ")--------------------");
  try
  {
  html = getOneHtml(s);
  }
  catch (final Exception e)
  {
  e.getMessage();
  }
  // System.out.println(html);
  System.out.println("------------------读取网页(" + s + ")结束--------------------\n");
  System.out.println("------------------分析(" + s + ")结果如下--------------------\n");
  String title = outTag(getTitle(html));
  title = title.replaceAll("_雅虎知识堂", "");
  // Pattern pa=Pattern.compile("

  // class=\"original\">(.?)((\r\n)*)(.*?)((\r\n)*)(.*?)
  ",Pattern.DOTALL);
  final Pattern pa = Pattern.compile("

  (.*?)

  ", Pattern.DOTALL);
  final Matcher ma = pa.matcher(html);
  while (ma.find())
  {
  sb.append(ma.group());
  }
  String temp = sb.toString();
  temp = temp.replaceAll("(
  )+?", "\n");// 转化换行
  temp = temp.replaceAll("

  .*?

  ", "");// 去图片注释
  hm.put("title", title);
  hm.put("original", outTag(temp));
  return hm;

  }

  /**

• @param args

•        测试一组网页，针对雅虎知识堂
  

  /
  public static void main(final String args[])
  {
  String url = "";
  final List list = new ArrayList();
  System.out.print("输入URL，一行一个，输入结束后输入 go 程序开始运行: \n");
  /

  • http://ks.cn.yahoo.com/question/1307121201133.html
  • http://ks.cn.yahoo.com/question/1307121101907.html
  • http://ks.cn.yahoo.com/question/1307121101907_2.html
  • http://ks.cn.yahoo.com/question/1307121101907_3.html
  • http://ks.cn.yahoo.com/question/1307121101907_4.html
  • http://ks.cn.yahoo.com/question/1307121101907_5.html
  • http://ks.cn.yahoo.com/question/1307121101907_6.html
  • http://ks.cn.yahoo.com/question/1307121101907_7.html
  • http://ks.cn.yahoo.com/question/1307121101907_8.html / final BufferedReader br = new BufferedReader(new InputStreamReader(System.in)); try { while (!(url = br.readLine()).equals("go")) { list.add(url); } } catch (final Exception e) { e.getMessage(); } final WebContent wc = new WebContent(); HashMap hm = new HashMap(); for (int i = 0; i < list.size(); i++) { hm = wc.getFromYahoo(list.get(i)); System.out.println("标题： " + hm.get("title")); System.out.println("内容： \n" + hm.get("original")); } /
  • String htmlurl[] = {
  • "http://ks.cn.yahoo.com/question/1307121201133.html",
  • "http://ks.cn.yahoo.com/question/1307121101907.html",
  • "http://ks.cn.yahoo.com/question/1307121101907_2.html",
  • "http://ks.cn.yahoo.com/question/1307121101907_3.html",
  • "http://ks.cn.yahoo.com/question/1307121101907_4.html",
  • "http://ks.cn.yahoo.com/question/1307121101907_5.html",
  • "http://ks.cn.yahoo.com/question/1307121101907_6.html",
  • "http://ks.cn.yahoo.com/question/1307121101907_7.html",
  • "http://ks.cn.yahoo.com/question/1307121101907_8.html" }; WebContent
  • wc = new WebContent(); HashMap hm = new HashMap<String,
  • String>(); for (int i = 0; i < htmlurl.length; i++) { hm =
  • wc.getFromYahoo(htmlurl[i]); System.out.println("标题： " +
  • hm.get("title")); System.out.println("内容： \n" + hm.get("original")); } / /
  • String html=""; String link=""; String sscript=""; String content="";
  • System.out.println(htmlurl+" 开始读取网页内容：");
  • html=wc.getOneHtml(htmlurl); System.out.println(htmlurl+"
  • 读取完毕开始分析……"); html=html.replaceAll("()","
  • ");//去除脚本 html=html.replaceAll("()","
  • ");//去掉CSS html=html.replaceAll(".*?"," ");//除去页面标题
  • html=html.replaceAll("]*href=(\"([^\"]*)\"|\'([^\']*)\'|([^\s>]*))[^>]*>(.*?)","
  • ");//去掉链接 html=html.replaceAll("(\s){2,}?"," ");//除去多余空格
  • html=wc.outTag(html);//多余标记 System.out.println(html); */

  /*

  • String s[]=html.split(" +"); for(int i=0;i<s.length;i++){
  • content=(content.length()>s[i].length())?content:s[i]; }
  • System.out.println(content); */

  // System.out.println(htmlurl+"网页内容结束");
  /*

  • System.out.println(htmlurl+"网页脚本开始："); List
  • script=wc.getScript(html); for(int i=0;i<script.size();i++){
  • System.out.println(script.get(i)); }
  • System.out.println(htmlurl+"网页脚本结束：");
  • System.out.println(htmlurl+"CSS开始："); List css=wc.getCSS(html);
  • for(int i=0;i<css.size();i++){ System.out.println(css.get(i)); }
  • System.out.println(htmlurl+"CSS结束：");
  • System.out.println(htmlurl+"全部链接内容开始："); List list=wc.getLink(html);
  • for(int i=0;i<list.size();i++){ link=list.get(i).toString(); }
  • System.out.println(htmlurl+"全部链接内容结束：");
  • System.out.println("内容"); System.out.println(wc.outTag(html)); */ } } [/code]