import requests
from lxml import etree
获取页面源码数据
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6821.400 QQBrowser/10.3.3040.400'
}
page_text = requests.get(url=url,headers=headers).text
实例化etree对象并且将页面源码数据加载到对象中
tree = etree.HTML(page_text)
li_list = tree.xpath('//ul[@class="huose-list-wrap"]/li')
all_data_list = []
for li in li_list:
title = li.xpath('.//div[@class="list-info"]/h2/a/text()')[0]
detail_url = li.xpath('.//div[@class="list-info"]/h2/a/@href')[0]
if not 'https:' in detail_url:
detail_url = 'https:'+detail_url
price = li.xpath('.//ul[@class="price"]/p//text()')
price = ''.join(price)
# 对详情页发起请求,获取页面数据
detail_page_text = requests.get(url=detail_url,headers=headers).text
tree = etree.HTML(detail_page_text)
desc = tree.xpath('.//div[@class="general-item-wrap""]//text()')
desc = ''.join(desc).strip("\n\b\t")
dic = {
'title':title,
'price':price,
'desc':desc
}
all_data_list.append(dic)
print(all_data_list)