用python提取div里的文本

Question

import requests
from bs4 import BeautifulSoup
import pprint
import json
url="http://www.miaomu.com/qyml/default.asp"
r= requests.get(url)
html=r.content.decode('gbk','ignore')
soup=BeautifulSoup(html,"html.parser")
articles=soup.find_all("div",{"class":"gyjtnr"})
articles

Answer 1

可以参考下

# -*- coding:utf-8 -*-

import requests
from lxml import etree
from pprint import pprint

def get_html(url):
    headers = {"User-Agent":"Mozilla/5.0 (compatible; MSIE 9.0; AOL 9.0; Windows NT 6.0; Trident/5.0)",}
    r = requests.get(url,headers=headers)
    return r.content.decode('gb2312','ignore')

def parse_html(text):
    infos = {}
    html = etree.HTML(text)
    datas = html.xpath("//div[@class='gynr']/div[@class='gyjtnr']")
    for index,data in enumerate(datas,1):
        name = html.xpath("//div[@class='gynr']/p[%s]//b/text()" % index)[0]
        content = ''.join(data.xpath("./text()"))
        if name not in infos.keys():
            infos[name] = content
    return infos


if __name__ == '__main__':
    url="http://www.miaomu.com/qyml/default.asp"
    html = get_html(url=url)
    if html:
        infos = parse_html(text=html)
        pprint(infos)