Scrapy爬虫问题

Question

class LiepinspiderSpider(scrapy.Spider):
name = 'liepinspider'
allowed_domains = ['www.liepin.com']
start_urls = ['http://www.liepin.com/']
#要抓取的最大页数
max_page = 20
# dqs = "010" 是北京 20是上海 050020是广州 070020是杭州 170020是武汉
out_url = 'https://www.liepin.com/zhaopin/?ckid=2d4732e20cffdcd9&fromSearchBtn=2&init=-1&flushckid=1&dqs={dqs}&flushckid=1&key={key}&imscid=R000000058&headckid=2d4732e20cffdcd9&d_pageSize=40&siTag=I-7rQ0e90mv8a37po7dV3Q%7EfA9rXquZc5IkJpXC-Ycixw&d_headId=6857362076fa97aa53548faedae4487b&d_ckId=6857362076fa97aa53548faedae4487b&d_sfrom=search_fp_bar&d_curPage=0'
next_url_base = 'https://www.liepin.com/zhaopin/?ckid=f2d1b254babfd641&fromSearchBtn=2&init=-1&flushckid=1&dqs={dqs}&degradeFlag=0&key={key}&imscid=R000000058&headckid=2d4732e20cffdcd9&d_pageSize=40&siTag=I-7rQ0e90mv8a37po7dV3Q%7EF5FSJAXvyHmQyODXqGxdVw&d_headId=6857362076fa97aa53548faedae4487b&d_ckId=0818b2c9abf73593582b6a90e780804f&d_sfrom=search_fp_bar&d_curPage=0&curPage={page}'
def start_requests(self):
yield Request(url=self.out_url.format(key="python", dqs="010", ),
callback=self.out_html_parse)
def out_html_parse(self,response):
#解析外部网页，生成各个工作的工作链接
if response.status ==200:
job_urls = response.xpath('//div[@class="sojob-result "]/ul//a/@href').extract()
for url in job_urls:
if re.search((r"(.*shtml)"),url): #有的网页内容不是工作链接，用正则删去
if re.search((r"(.*shtml)"),url).group(1).startswith("https"):
job_url = re.search((r"(.*shtml)"),url).group(1)
else:
job_url = "https://www.liepin.com"+ url
yield Request(url = job_url , callback= self.job_url_parse)
for page in range(1, self.max_page):
yield Request(url=self.next_url_base.format(key="python", dqs="010", page=page),
callback=self.out_html_parse)
def job_url_parse(self, response):
解析页面的就先不写了
现在的问题是抓取时候只能抓取70-90个数据就结束了进程，所以现在想是不是callback遇到了问题，希望各位大神帮忙

Answer 1

不知道你这个问题是否已经解决, 如果还没有解决的话:

• 这篇文章：scrapy 异常处理 也许有你想要的答案，你可以看看

如果你已经解决了该问题, 非常希望你能够分享一下解决方案, 以帮助更多的人 ^-^