是否是使用了代理后
ssl._create_default_https_context = ssl._create_unverified_context
无效
获取代理ip
def get_proxy():
return requests.get("http://127.0.0.1:5010/get/").content
获得页面数据
def do_spider(page_no=1, encoding="gbk", parse_url="null"):
ssl._create_default_https_context = ssl._create_unverified_context
proxy = request.ProxyHandler({
"sock5": get_proxy()
})
opener = request.build_opener(proxy)
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
request.install_opener(opener)
# 对于二级页面进行爬取
if parse_url == "null":
html_bytes = request.urlopen(url.format(page_no)).read()
else:
html_bytes = request.urlopen(parse_url).read()
html_code = str(html_bytes, encoding)
return html_code
异常
Traceback (most recent call last):
File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36-32\Lib\urllib\request.py", line 1318, in do_open
encode_chunked=req.has_header('Transfer-encoding'))
File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36-32\Lib\http\client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36-32\Lib\http\client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36-32\Lib\http\client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36-32\Lib\http\client.py", line 1026, in _send_output
self.send(msg)
File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36-32\Lib\http\client.py", line 964, in send
self.connect()
File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36-32\Lib\http\client.py", line 936, in connect
(self.host,self.port), self.timeout, self.source_address)
File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36-32\Lib\socket.py", line 724, in create_connection
raise err
File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36-32\Lib\socket.py", line 713, in create_connection
sock.connect(sa)
TimeoutError: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。
循环6次左右后就会爆上面这个异常
