如何在爬取网页数据时用多线程？（python）

Question

如何在爬取网页数据时用多线程？（python）

Answer 1

可以使用concurrent.futures的ThreadPoolExecutor，用一个线程池执行异步调用。例：

import requests
from bs4 import BeautifulSoup as bs
import time
from concurrent.futures import ThreadPoolExecutor
cookies = {
    '__cfduid': 'xxx',
    'PHPSESSID': 'xxx',
}#从页面分析获取

headers = {
    'Connection': 'keep-alive',
    'Upgrade-Insecure-Requests': '1',
    'DNT': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36 Edg/85.0.564.44',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'Referer': 'http://www.enagames.com/escape-game/top10newgames-rescue-the-tiger-2',
    'Accept-Language': 'en-US;q=0.6',
}

response = requests.get('http://www.enagames.com/escape-game/top10newgames-rescue-the-tiger-2',headers=headers, cookies=cookies, verify=False)
soup=bs(response.text,'html.parser')

game_link = set([x['href']  for x in soup.select(
    'div.home_game_image_featured center a')])
gl=list(game_link)

def get_swf(url):
    r=requests.get(url,headers=headers,cookies=cookies)

    s=bs(r.text,'html.parser')

    sl =s.select_one('div#flash_container1 object param')['value']
    with open(f'./{url.split("/")[-1]}.swf', 'wb') as f:
        try:
            req = requests.get(sl, headers=headers)
            f.write(req.content)
        except:
            pass
with ThreadPoolExecutor(max_workers=5)  as ex:
    ex.map(get_swf,gl[:5])