我想多线程爬取笔趣阁的一部小说但是函数不执行

Question

import re
import time
import os
import requests
from lxml import etree
from threading import Thread
import threading
from queue import Queue
headers = {
'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Mobile Safari/537.36'}
'''要爬取小说的网址'''
url = '

您访问的页面不存在 - 新笔趣阁 https://www.xbiquge.la/0/951/'

'这个网址是笔趣阁任意一部小说的网址'
'''对这页发起请求'''
resp = requests.get(url=url, headers=headers)
contentpage = resp.content.decode('utf-8', 'ignore')
html = etree.HTML(contentpage)
print(resp.status_code)
'''章节和内容队列+'''
titles_quenue = Queue(50)
text_urls = Queue(50)

'''定义生产者'''
'''获取数据
1.---获取小说名字
2.获取每一章的内容和标题'

title1 = html.xpath('/n//dd/a/text()')
for k in title1:
    titles_quenue.put(k)#标题
text_newurl = []
text_url2 =  html.xpath('/n//dd/a/@href')
for i in text_url2:
    j = 'https://www.biquwx.la/1_1760/' + i
    text_newurl.append(j)
for j in text_newurl:
    text_urls.put(j)

'''定义消费者'''
def consumer():

'''小说名字'''
name = re.findall('<h1>(.+?)</h1>', contentpage, re.DOTALL)[0]
print(name)
'''小说章节'''
title = titles_quenue.get()
'''小说链接--发起请求获取数据'''
text_url = text_urls.get()
resp = requests.get(url=text_url,headers =headers)
content = resp.content.decode('utf-8')
html1 = etree.HTML(content)
text = html1.xpath('//div[@id="content"]/text()')
text = "".join(text)
while len(text) == 0:
    '''如果没有内容，再次发起请求'''
    resp2 = requests.get(url=text_url, headers=headers)
    content2 = resp2.content.decode('utf-8')
    html2 = etree.HTML(content2)
    text = html2.xpath('//div[@id="content"]/text()')
    text = "".join(text)
    if len(text) != 0:
        break
if len(text) != 0:
    with open('D:\AAAA桃花青帝\Python文件\爬虫\爬取小说\{}/'.format(name) + '{}'.format(title) + '.txt', 'a', encoding='utf-8') as  f:
        f.write(text)
    print(title+'下载完成！！！！')

def multi():

#定义生产者
for i  in range(50):
    t = threading.Thread(target=producer)
    t.start()
for  j in range(50):
    t= threading.Thread(target=consumer)
    t.start()

multi()

章节链获取不到，一直卡在resp.staus的哪里，

Answer 1

流程如下代码所示，但是笔趣阁服务器不咋地，也没啥太大限制的反爬，所以开50个线程基本上就是反馈503了，如果你有代理ip可以加进去，然后就是 线程开少点，爬取速度限制一下，比如在每个章节获取里sleep一下。可以的话，希望能采纳！

import os
from queue import Queue
import threading
import requests
from lxml import etree


class BQG:
    def __init__(self):
        self.url_queue = Queue()
        self.prefix_url = 'https://www.biquwx.la/1_1760/'
        self.suffix_url = None
        self.dir_path = os.path.join(os.getcwd(), '武炼巅峰')
        self.create_dir()
        self.get_links()
        self.get_consumers()

    def create_dir(self):
        """创建目录"""
        if not os.path.exists(self.dir_path):
            os.mkdir(self.dir_path)

    def get_links(self):
        """获取所有url"""
        top_res = requests.get(self.prefix_url)
        top_html = etree.HTML(top_res.content.decode())
        suffix_url = top_html.xpath('//div[@id="list"]/dl/dd/a/@href')
        for i in suffix_url:
            url = self.prefix_url + i
            print(url)
            self.url_queue.put(url)

    def get_consumers(self):
        """多线程爬取"""
        for i in range(50):
            t = threading.Thread(target=self.get_content)
            t.start()

    def get_content(self):
        """获取章节内容"""
        url = self.url_queue.get()
        res = requests.get(url)
        html = etree.HTML(res.content.decode())
        title = ''.join(html.xpath('//h1/text()'))
        content = '\n'.join(html.xpath('//div[@id="content"]//text()'))
        file_path = os.path.join(self.dir_path, '{}.txt'.format(title))
        with open(file_path, 'w', encoding='utf-8') as f:
            f.write(title)
            f.write(content)


if __name__ == '__main__':
    bqg = BQG()