0.问题背景:全站爬虫crawlSpider,如何添加爬虫代理池?
1.代码
# spider文件: sun.py
import scrapy
from time import sleep
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from faker import Faker
import requests
from lxml import etree
from ..items import Main00CrawlproItem
class SunSpider(CrawlSpider):
fake = Faker()
headers = {
'User-Agent': fake.user_agent()
}
def __init__(self):
super().__init__()
self.process_proxy_pool()
name = "sun"
# allowed_domains = ["www.xxx.com"]
start_urls = ["https://wz.sun0769.com/political/index/politicsNewest"]
# link就表示创建出来的一个链接提取器
# allow参数后面要跟一个正则表达式,就可以作为链接提取的规则
# link首先会去start_urls表示的页面中进行链接的提取
link_title = LinkExtractor(allow=r'id=1&page=(\d+)') # 爬取标题的link
link_detailPageUrl = LinkExtractor(allow=r'/political/politics/index?id=(\d+)') # 爬取详情页的link
# 创建了一个规则解析器
# 链接提取器LinkExtractor提取到的链接会发送给Rule这个规则解析器
# 规则解析器接收到链接后,会对链接进行请求发送,并且根据callback指定的函数进行数据解析
# 如果callback 为None,follow 默认设置为 True;添加回调函数callback后,为 False,不跟踪
rules = (
# 启用ip_pool之前,follow设置为False,否则被反爬
Rule(link_title, callback="parse_item", follow=True),
Rule(link_detailPageUrl, callback='parse_detail')
)
# 解析方法的调用次数取决于link提取到的链接的个数 - 此处response为每一个页面,即第X页的数据
def parse_item(self, response):
# item = {}
li_list = response.xpath('//ul[@class="title-state-ul"]/li')
for li in li_list:
title = li.xpath('./span[@class="state3"]/a/text()').extract_first()
print(title)
sleep(1)
#item["domain_id"] = response.xpath('//input[@id="sid"]/@value').get()
#item["name"] = response.xpath('//div[@id="name"]').get()
#item["description"] = response.xpath('//div[@id="description"]').get()
# return item
def parse_detail(self, response):
detail_content = response.xpath('//div[@class="details-box"]/pre/text()').extract_first()
print(detail_content)
def check_proxy(self, proxy, url='http://httpbin.org/get'): # # this def is what I added.;检测ip是否可用
proxies = {
'http': 'http://' + proxy,
'https': 'http://' + proxy,
}
try:
response = requests.get(url=url, proxies=proxies, headers=self.headers, timeout=5)
if response.status_code == 200:
return True
else:
return False
except:
False
def process_proxy_pool(self): # this def is what I added. # 通过API获取ip
# for cookie
sess = requests.Session()
url_getapi = 'https://jahttp.zhimaruanjian.com/getapi/'
sess.get(url=url_getapi, headers=self.headers)
# 构建一个Proxy pool
proxy_pool = []
# API url
proxy_url = 'http://webapi.http.zhimacangku.com/getip?neek=321a408a&num=3&type=1&time=4&pro=0&city=0&yys=0&port=11&pack=0&ts=0&ys=0&cs=0&lb=1&sb=&pb=4&mr=1®ions='
response = sess.get(url=proxy_url, headers=self.headers)
print(f'{response} in class.')
response.encoding = 'utf8'
response_text = response.text
tree = etree.HTML(response_text)
ip_list = tree.xpath('/html/body//text()')
print(f'len(ip_list): {len(ip_list)} \nip: {ip_list}')
all_ips = ip_list[0].split('\r\n')[
:-1] # ['113.237.5.78:4256', '123.245.248.58:4231', '218.86.19.32:4215', '']
print(all_ips)
for ip in all_ips:
if self.check_proxy(ip):
proxy_pool.append(ip)
item = Main00CrawlproItem()
item['ip_list'] = all_ips
yield item
print(proxy_pool)
# return proxy_pool
##################################################
# item文件: items.py
import scrapy
class Main00CrawlproItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
ip_list = scrapy.Field()
2.个人初步思路:在爬虫文件(spider.py),写了获取代理IP池的函数,ip池是一个list,想要然后封装给item对象。
目前问题:0.爬虫文件(spider.py)的ip代理池函数该如何触发运行?
1.想通过中间键的下载中间键(Main00CrawlproDownloaderMiddleware)的def process_request(self, request, spider)来获取代理ip(池),以达到设置代理的目的。想把item对象(即代理ip池)通过request.meta传给中间键,怎么传?
PS:或者另有其它什么方法来添加代理ip池
3.操作环境: windows 11, py 3.10, pycharm