guangkun77 2019-12-07 19:38 采纳率: 0%
浏览 754

为什么用request-html爬取的内容为空?

以下是我的代码和运行结果,请问各位大佬为什么我运行内容为空?谢谢各位

from requests_html import HTMLSession
from requests_html import HTML
import requests
import time
import json
import random
import sys
import os
import csv

from fake_useragent import UserAgent#

session = HTMLSession()
url = 'https://car.autohome.com.cn/'
#https://car.autohome.com.cn/config/series/3862.html

USER_AGENTS = [
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
    "Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre",
    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
    "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10"
]
# 品牌列表
def get_bank():
    response = session.get(url)#爬取网页
    response.encoding='gbk'
    response.html.render()
    banks = response.html.find('div.cartree>ul>li>h3')
    for bank in banks:
        #print(bank.text)
        #print(bank.html)
        bk = bank.text
        start = bk.find("(")
        end = bk.find(")")
        print(bk[0:start])
        print(bk[start+1:end])
    print(banks)

# 汽车品牌
def get_url():
    current_dir = os.path.abspath('.')#返回绝对路径
    print(current_dir)
    #print('1')
    file_name = os.path.join(current_dir, "\\bank.csv")
    print(file_name)
    response.encoding='gbk'
   # print('2')
    with open(file_name, 'wt', newline='')  as csvfile1:
        header = ['bank','count', 'url']
        writer = csv.writer(csvfile1)
        writer.writerow(header)
        #访问页面
        response = session.get(url)
        #print(response)
        #response.html.render()
        banks = response.html.find('.cartree ul li h3 a')
        #banks = response.html.find('#cartree')
        print(banks)
        print('1')
        for bank in banks:
            #print(bank.text)
            #格式化
            bk = bank.text
            print('2')
            print(bk)
            start = bk.find("(")
            end = bk.find(")")
            bank1 = bk[0:start]
            Num = bk[(start+1):end]
            url2 = url + bank.attrs.get("href", None)
            print(url2)

            save2csc(writer,bank1,Num,url2)
        #print(banks)
    csvfile1.close()
#
#写入CSV文件
def save2csc(writer,bank,num,url):
        header = ['bank','num' 'url']
        writer.writerow(header)
        csvrow1 = []
        csvrow1.append(bank)
        csvrow1.append(num)
        csvrow1.append(url)

        #writer.writerow(csvrow1)
        #print(csvrow1)
if __name__ == '__main__':
    print("开始处理")

    get_url()
    get_bank() 
    print("处理结束")

图片说明图片说明

  • 写回答

1条回答 默认 最新

  • 行走蜗牛 2019-12-07 21:17
    关注

    先看下响应状态是不是200?

    评论

报告相同问题?

悬赏问题

  • ¥50 有数据,怎么建立模型求影响全要素生产率的因素
  • ¥50 有数据,怎么用matlab求全要素生产率
  • ¥15 TI的insta-spin例程
  • ¥15 完成下列问题完成下列问题
  • ¥15 C#算法问题, 不知道怎么处理这个数据的转换
  • ¥15 YoloV5 第三方库的版本对照问题
  • ¥15 请完成下列相关问题!
  • ¥15 drone 推送镜像时候 purge: true 推送完毕后没有删除对应的镜像,手动拷贝到服务器执行结果正确在样才能让指令自动执行成功删除对应镜像,如何解决?
  • ¥15 求daily translation(DT)偏差订正方法的代码
  • ¥15 js调用html页面需要隐藏某个按钮