关于爬虫遇到的问题，请问一下

Question

main文件代码：

import yaml
import json
import base64
import requests
import random
from scrapy import Spider
from ..items import FirstItem


class MainSpider(Spider):
    name = "main"
    start_urls = ["https://yftx-test.xailife.com/login?redirect=%2Findex"]



    def parse(self, response):
        name = response.xpath('//div[@class="charts-left-right"]/div')
        name_a = response.xpath('//section[@class="app-main"]/div/div[2]/div[2]/div[2]/div/div[3]/div/div/div/div[2]/div')
        url = 'https://yftx-test.xailife.com'
        # for xe in name:
        #     imgbase = xe.xpath('./div/img/@src')
        #     img = imgbase.extract()
        #     for img_a in img:
        #         random_string = random.randint(1, 1000)
        #         base64_split = str(img_a).split(',',1)
        #         base64_str = base64_split[1]
        #         base64_decode = base64.b64decode(base64_str)
        #         with open(rf"D:\python\first\first\img\img{random_string}.png","wb") as f:
        #             f.write(base64_decode)

        for tubiao in name_a:
            items = FirstItem()
            random_stringa = random.randint(1, 1000)
            tubiao_a = tubiao.xpath('./div/img/@src')
            tubiao_b = tubiao_a.extract()
            print('获取的图片路径：',tubiao_b)
            for tubiao_c in tubiao_b:
                url_jion = url + tubiao_c
                response = requests.get(url_jion).content
                items['tubiao'] = response
                items['random_a'] = random_stringa
                print('次数：')
        return items

pipelin文件代码：

from itemadapter import ItemAdapter


class FirstPipeline(scrapy.Item):
    def process_item(self, item, spider):
        print(item)
        adaoter = ItemAdapter(item)
        t = adaoter.get('tubiao')
        y = adaoter.get('random_a')
        print("随机数：", y)
        with open(rf"D:\python\first\first\img\img{y}.png", "wb") as f:
            f.write(t)

为什么这么写，无法获取到图片？

Answer 1

引自免费微信小程序：皆我百晓生

在您提供的代码中，存在一些问题导致图片无法正确获取并保存。首先，我们来分析MainSpider中的parse方法：

1. 你尝试从响应中提取图片URL，并使用requests.get直接下载图片内容到items['tubiao']。但是，在pipeline中，你将items['tubiao']当作一个文件流进行处理并写入文件，而实际上它是一个字节串。

解决办法是在Spider中先将图片内容转换为合适格式再存入item中，例如：

for tubiao in name_a:
    items = FirstItem()
    random_stringa = random.randint(1, 1000)
    tubiao_a = tubiao.xpath('./div/img/@src')
    tubiao_b = tubiao_a.extract_first()  # 使用extract_first而不是extract，假设只有一张图

    url_jion = url + tubiao_b
    response = requests.get(url_jion)
    if response.status_code == 200:
        img_content = response.content
        items['tubiao'] = img_content
        items['random_a'] = random_stringa
        print('次数：')

    yield items  # 将itemsyield出来以供Pipeline处理

接下来是FirstPipeline部分：

1. 您的FirstPipeline类继承了scrapy.Item，但应该继承自scrapy.pipeline.ItemPipeline。

修改后的pipeline应如下所示：

from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem


class FirstPipeline:
    def process_item(self, item, spider):
        adapter = ItemAdapter(item)

        t = adapter.get('tubiao')
        if t is not None:
            y = adapter.get('random_a')
            with open(rf"D:\python\first\first\img\img{y}.png", "wb") as f:
                f.write(t)
        else:
            raise DropItem(f"Missing 'tubiao' for item {item!r}")

        return item

通过上述调整，当item在pipeline中被处理时，正确的图片内容会被写入到磁盘上指定的文件路径中。同时，请确保您的Scrapy项目已启用此pipeline（在settings.py中配置ITEM_PIPELINES）。