我这样写逻辑有错误吗?为什么在parse_search函数里取不到href的值呢?
# -*- coding: utf-8 -*-
import scrapy
from GP_Spider.items import GpItem
from scrapy import Request
class GoogleSpider(scrapy.Spider):
name = 'google'
allowed_domains = ['google.play.com']
start_urls = ['https://play.google.com/store']
def parse(self, response):
keywords = [
'stuttering', 'speech%20therapy', 'speech%20and%20language%20therapy', 'aphasia', 'apraxia', 'dysarthria'
]
link_flag = 0
urls = []
for each in keywords:
app_url = ("https://play.google.com/store/search?q=" + keywords[link_flag] + '&c=apps')
print(app_url)
yield Request(url=app_url, callback=self.parse_search, dont_filter=True)
link_flag += 1
def parse_search(self, response):
print("START PARSING")
selector = scrapy.Selector(response)
#print(response.body)
urls = selector.xpath('//a[@class="poRVub" and aria-hidden="true"]/@href').extract()
#urls = selector.xpath('//*[@id="fcxH9b"]/div[4]/c-wiz/div/div[2]/div/c-wiz/c-wiz/c-wiz/div/div[2]/div[1]/c-wiz/div/div/div[1]/div/div/a/@href').extract()
print(urls)
link_flag = 0
links = []
for link in urls:
links.append(link)
for each in urls:
yield Request(url="https://play.google.com" + links[link_flag], callback=self.parse_detail, dont_filter=True)
print("https://play.google.com" + links[link_flag])
link_flag += 1
def parse_detail(self, response):
item = GpItem()
item['app_url'] = response.url
item['app_name'] = response.xpath('//h1[@itemprop="name"]/span').xpath('text()').get()
item['app_icon'] = response.xpath('//img[@itemprop="image"]/@src').get()
item['app_rate'] = response.xpath('//div[@class="K9wGie"]/div[@class="BHMmbe"]').xpath('text()').get()
item['app_version'] = response.xpath('//div[@class="IQ1z0d"]/span[@class="htlgb"]').xpath('text()').get()
item['app_description'] = response.xpath('//div[@itemprop="description"]/span/div').xpath('text()').get()
# item['app_developer'] = response.xpath('//')
# print(response.text)
yield item
这个xpath路径是我自己写的,如果直接从chrome浏览器复制下来的话,就可以爬到特定的那个搜索结果页面的url,但是其他搜索结果页就爬不到,这是为什么?
求教各位大佬