python爬虫怎么改成多线程

Question

python爬虫怎么改成多线程

import parsel
import requests
import os


url = "****"

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
                  'Chrome/100.0.4896.75 Safari/537.36 Edg/100.0.1185.39 '
}

response = requests.get(url=url, headers=headers)
html_str = response.text
selector = parsel.Selector(html_str)
lis = selector.xpath('//div[@id="primary"]/main/article')[1:]
for li in lis:
    pic_title = li.xpath('.//h2/a/text()').get()
    pic_url = li.xpath('.//h2/a/@href').get()
    print('正在下载相册：', pic_title)

    if not os.path.exists('img\\' + pic_title):
        os.mkdir('img\\' + pic_title)

    response_pic = requests.get(url=pic_url, headers=headers).text

    selector_2 = parsel.Selector(response_pic)
    pic_url_list = selector_2.xpath('//div[@class="entry-content"]//img/@src').getall()[1:-1]
    # print(pic_url_list)
    for pic_url in pic_url_list:
        img_data = requests.get(url=pic_url, headers=headers).content

        file_name = pic_url.split('/')[-1]
        with open(f'img\\{pic_title}\\{file_name}', mode='wb') as f:
            f.write(img_data)
            print('保存完成:', file_name)

Answer 1

楼上的这些线程改造方式放在这里行不通吧。楼主代码中IO操作在循环的地方，这里线程不是应该将循环改成多线程吗。。。
##改了一个循环，试一下

```python

import parsel
import requests
import os
from multiprocessing.dummy import Pool
 
 
url = "****"
 
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
                  'Chrome/100.0.4896.75 Safari/537.36 Edg/100.0.1185.39 '
}
 
response = requests.get(url=url, headers=headers)
html_str = response.text
selector = parsel.Selector(html_str)
lis = selector.xpath('//div[@id="primary"]/main/article')[1:]
for li in lis:
    pic_title = li.xpath('.//h2/a/text()').get()
    pic_url = li.xpath('.//h2/a/@href').get()
    print('正在下载相册：', pic_title)
 
    if not os.path.exists('img\\' + pic_title):
        os.mkdir('img\\' + pic_title)
 
    response_pic = requests.get(url=pic_url, headers=headers).text
 
    selector_2 = parsel.Selector(response_pic)
    pic_url_list = selector_2.xpath('//div[@class="entry-content"]//img/@src').getall()[1:-1]
    # print(pic_url_list)
    def saveimg(pic_url):
        img_data = requests.get(url=pic_url, headers=headers).content
        file_name = pic_url.split('/')[-1]
        with open(f'img\\{pic_title}\\{file_name}', mode='wb') as f:
            f.write(img_data)
            print('保存完成:', file_name)
   
    p = Pool(10)
    result=p.map(saveimg,pic_url_list)
    p.close()
    p.join()

```