import json
import scrapy
from news.items import NewsItem
class AbcSpider(scrapy.Spider):
name = 'abc'
allowed_domains = ['gelonghui.com']
start_urls = ['https://www.gelonghui.com/api/channels/web_home_page/articles/v8']
def parse(self, response):
item = NewsItem()
dict = json.loads(response)
item['title'] = dict['result'][0]['data'][0]['title']
yield item
使用print(response),结果是[].已经设置了
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/93.0.4577.63 Safari/537.36'