下面的代码
import requests
import re
"""
https://www.xsbiquge.com/20_20331/
https://www.xsbiquge.com/20_20331/1135932.html
</a></dd><dd><a href="/20_20331/1135938.html">
"""
url = 'https://www.xsbiquge.com/20_20331/'
headers = {
'Host': 'www.xsbiquge.com',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'
}
response = requests.get(url=url, headers=headers)
response.encoding = response.apparent_encoding
html_data = response.text
# print(html_data)
fiction_zj_url_list = re.findall('<dd><a href="(.*?)">', html_data, re.S)
print(fiction_zj_url_list)
try:
for page in range(0, 1, 1):
print('https://www.xsbiquge.com/20_20331{}'.format(fiction_zj_url_list[page]))
fiction_zj_url = 'https://www.xsbiquge.com/20_20331{}'.format(fiction_zj_url_list[page])
response_2 = requests.get(url=fiction_zj_url, headers=headers)
response_2.encoding = response_2.apparent_encoding
html_data_2 = response_2.text
result_title = re.findall('<h1>(.*?)</h1>', html_data_2, re.S)
result_zw = re.findall('<div id="content">(.*?)</div>', html_data_2, re.S)
zw_data = result_zw[0].replace(' ', '').replace('<br />', '')
with open('万古神帝.txt', mode='a', encoding='utf-8') as f:
f.write(result_title[0])
f.write(zw_data)
except Exception as e:
print(e)
pass