scrapy每个url提取的数据保存mongobd不同的集合zhong,几百个url?
各位大佬怎么处理帮帮忙! 不知道怎么处理
# -*- coding: utf-8 -*-
import scrapy
from dongQiuDi.items import DongqiudiItem
import time
class ZuqiuSpider(scrapy.Spider):
name = 'zuqiu'
allowed_domains = ['dongqiudi.com']
start_urls = [
"https://www.dongqiudi.com/data/1",
'https://www.dongqiudi.com/data/2',
"https://www.dongqiudi.com/data/3",
"https://www.dongqiudi.com/data/4",
"https://www.dongqiudi.com/data/5",
"https://www.dongqiudi.com/data/10",
"https://www.dongqiudi.com/data/16",
"https://www.dongqiudi.com/data/12",
"https://www.dongqiudi.com/data/16",
"https://www.dongqiudi.com/data/19", #土超
"https://www.dongqiudi.com/data/55",
"https://www.dongqiudi.com/data/17",
"https://www.dongqiudi.com/data/18",
"https://www.dongqiudi.com/data/20", #巴西
"https://www.dongqiudi.com/data/21", #阿根廷
"https://www.dongqiudi.com/data/68", #卡特尔
"https://www.dongqiudi.com/data/69", #伊朗
"https://www.dongqiudi.com/data/71", #乌兹别克
]
def parse(self, response):
p_list = response.xpath(".//div[@class='team_point_ranking']/div/div/div/p")
for p in p_list:
item = DongqiudiItem()
item['No'] = p.xpath(
"./span[1]/text()").extract_first()
item['image'] = p.xpath(
"./span[2]/img/@src").extract_first()
item['name'] = p.xpath(
"./span[2]/b/text()").extract_first()
item['sestion'] = p.xpath(
"./span[3]/text()").extract_first()
item['win'] = p.xpath(
"./span[4]/text()").extract_first()
item['mean'] = p.xpath(
"./span[5]/text()").extract_first()
item['lose'] = p.xpath(
"./span[6]/text()").extract_first()
item['coal'] = p.xpath(
"./span[7]/text()").extract_first()
item['fumbole'] = p.xpath(
"./span[8]/text()").extract_first()
item['goaldifference'] = p.xpath(
"./span[9]/text()").extract_first()
item['score'] = p.xpath(
"./span[10]/text()").extract_first()
yield item
time.sleep(1) #每一个URL,暂停一秒