使用如下python代码读取配置文件中的linkLists(https://api.douban.com/v2/book/1006639),每次爬取到100条左右时就报错bad request
请教各位大神有木有遇到过这种情况,该怎么解决
import json
import pymysql
import random
import time
import urllib
from urllib import request
import doubans.config.linkconfig
conn = pymysql.connect( user="root",password="root",database="python",charset='utf8')
cur = conn.cursor()
def getDetail(url):
req = urllib.request.Request(url)
req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36')
try :
res = urllib.request.urlopen(req)
jsondata = res.read().decode("UTF-8")
res.close()
result = json.loads(jsondata)
except urllib.error.URLError as e :
print(e.reason)
if ('title' in result.keys()):
title = result["title"]
else:
title = ""
if ('isbn13' in result.keys() and 'ibsn13' != ""):
isbn = result["isbn13"]
else:
isbn = result["isbn10"]
l = []
l.append([title,isbn])
sql = "INSERT INTO allbooks values(%s,%s)"
cur.executemany(sql, l)
conn.commit()
for i in range(0,len(doubans.config.linkconfig.linkLists)):
print(doubans.config.linkconfig.linkLists[i])
getDetail(doubans.config.linkconfig.linkLists[i])
time.sleep(int(format(random.randint(0, 9))))
报错信息如下
Traceback (most recent call last):
File "F:/Python/doubans/getDetail.py", line 143, in getDetail(doubans.config.linkconfig.linkLists[i])
File "F:/Python/doubans/getDetail.py", line 27, in getDetail if ('title' in result.keys()): UnboundLocalError: local variable 'result' referenced before assignment
Bad Request