感谢
import requests,time,random,parsel,csv
def Run():
print("爬虫开始！")
num=eval(input("请输入要爬取几页数据："))
file = open('move_data.csv','w',encoding='gb18030',newline='') #创建文件对象
csv_writer=csv.writer(file) #构建CSV写入对象
csv_writer.writerow(['title','year','stong','runtime','genre','votes','gross']) #存储标题
file.close()
for n in range(0,num):
print("正在爬取第{}页内容：".format(n+1))
URL='https://www.imdb.com/search/title/?release_date=1990-01-01,2020-12-31&sort=num_votes,desc&start={}*50+1&ref_=adv_nxt'.format(n)
headers={'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \ Chrome/94.0.4606.81 Safari/537.36'}
res_text=parse_text(URL,headers) #调用页面响应函数
data=get_text(res_text) #调用获取数据
save_file(data) #调用保存数据
print('爬虫结束...')
def parse_text(url,headers,params=None):
#发送请求，获取响应内容
#休眠，避免被对方反爬检测到
time.sleep(random.randint(1,5))
req = requests.get(url,headers=headers,params=params)
req.encoding=req.apparent_encoding
return req.text
def get_text(content):
list = []
html = parsel.Selector(content)
title = html.xpath('//*[@class="lister-item-content"]//a/text()').getall()
print(title)
def save_file(lists):
"""保存标题和URL"""
with open('zohe.csv', 'a+', encoding='gb18030', newline='') as file: # 创建文件对象
csv_writer = csv.writer(file) # 构建CSV写入对象
# csv_writer=csv.writerrow(['文件名称','链接','文件发布时间'])
for values in lists: # 存入数据
csv_writer.writerow(values)
file.close()
if __name__ == '__main__':
Run()
D:\python\chen\venv\Scripts\python.exe D:/python/chen/作业/综合项目考核.py
爬虫开始！
请输入要爬取几页数据：2
正在爬取第1页内容：
['The Shawshank Redemption', 'Frank Darabont', 'Tim Robbins', 'Morgan Freeman', 'Bob Gunton', 'William Sadler', 'The Dark Knight', 'Christopher Nolan', 'Christian Bale', 'Heath Ledger', 'Aaron Eckhart', 'Michael Caine', 'Inception', 'Christopher Nolan', 'Leonardo DiCaprio', 'Joseph Gordon-Levitt', 'Elliot Page', 'Ken Watanabe', 'Fight Club', 'David Fincher', 'Brad Pitt', 'Edward Norton', 'Meat Loaf', 'Zach Grenier', 'Forrest Gump', 'Robert Zemeckis', 'Tom Hanks', 'Robin Wright', 'Gary Sinise', 'Sally Field', 'Pulp Fiction', 'Quentin Tarantino', 'John Travolta', 'Uma Thurman', 'Samuel L. Jackson', 'Bruce Willis', 'Game of Thrones', 'Emilia Clarke', 'Peter Dinklage', 'Kit Harington', 'Lena Headey', 'The Matrix', 'Lana Wachowski', 'Lilly Wachowski', 'Keanu Reeves', 'Laurence Fishburne', 'Carrie-Anne Moss', 'Hugo Weaving', 'The Lord of the Rings: The Fellowship of the Ring', 'Peter Jackson', 'Elijah Wood', 'Ian McKellen', 'Orlando Bloom', 'Sean Bean', 'The Lord of the Rings: The Return of the King', 'Peter Jackson', 'Elijah Wood', 'Viggo Mortensen', 'Ian McKellen', 'Orlando Bloom', 'Interstellar', 'Christopher Nolan', 'Matthew McConaughey', 'Anne Hathaway', 'Jessica Chastain', 'Mackenzie Foy', 'Breaking Bad', 'Bryan Cranston', 'Aaron Paul', 'Anna Gunn', 'Betsy Brandt', 'The Dark Knight Rises', 'Christopher Nolan', 'Christian Bale', 'Tom Hardy', 'Anne Hathaway', 'Gary Oldman', 'The Lord of the Rings: The Two Towers', 'Peter Jackson', 'Elijah Wood', 'Ian McKellen', 'Viggo Mortensen', 'Orlando Bloom', 'Se7en', 'David Fincher', 'Morgan Freeman', 'Brad Pitt', 'Kevin Spacey', 'Andrew Kevin Walker', 'Django Unchained', 'Quentin Tarantino', 'Jamie Foxx', 'Christoph Waltz', 'Leonardo DiCaprio', 'Kerry Washington', 'Gladiator', 'Ridley Scott', 'Russell Crowe', 'Joaquin Phoenix', 'Connie Nielsen', 'Oliver Reed', 'Batman Begins', 'Christopher Nolan', 'Christian Bale', 'Michael Caine', 'Ken Watanabe', 'Liam Neeson', 'Inglourious Basterds', 'Quentin Tarantino', 'Brad Pitt', 'Diane Kruger', 'Eli Roth', 'Mélanie Laurent', 'The Silence of the Lambs', 'Jonathan Demme', 'Jodie Foster', 'Anthony Hopkins', 'Lawrence A. Bonney', 'Kasi Lemmons', 'The Avengers', 'Joss Whedon', 'Robert Downey Jr.', 'Chris Evans', 'Scarlett Johansson', 'Jeremy Renner', 'Saving Private Ryan', 'Steven Spielberg', 'Tom Hanks', 'Matt Damon', 'Tom Sizemore', 'Edward Burns', 'The Wolf of Wall Street', 'Jordan Belfort', 'Martin Scorsese', 'Leonardo DiCaprio', 'Jonah Hill', 'Margot Robbie', 'Matthew McConaughey', "Schindler's List", 'Oskar Schindler', 'Steven Spielberg', 'Liam Neeson', 'Ralph Fiennes', 'Ben Kingsley', 'Caroline Goodall', 'The Prestige', 'Christopher Nolan', 'Christian Bale', 'Hugh Jackman', 'Scarlett Johansson', 'Michael Caine', 'The Departed', 'Martin Scorsese', 'Leonardo DiCaprio', 'Matt Damon', 'Jack Nicholson', 'Mark Wahlberg', 'The Green Mile', 'Frank Darabont', 'Tom Hanks', 'Michael Clarke Duncan', 'David Morse', 'Bonnie Hunt', 'Shutter Island', 'Martin Scorsese', 'Leonardo DiCaprio', 'Emily Mortimer', 'Mark Ruffalo', 'Ben Kingsley', 'Memento', 'Christopher Nolan', 'Guy Pearce', 'Carrie-Anne Moss', 'Joe Pantoliano', 'Mark Boone Junior', 'Avatar', 'James Cameron', 'Sam Worthington', 'Zoe Saldana', 'Sigourney Weaver', 'Michelle Rodriguez', 'American Beauty', 'Sam Mendes', 'Kevin Spacey', 'Annette Bening', 'Thora Birch', 'Wes Bentley', 'Guardians of the Galaxy', 'James Gunn', 'Chris Pratt', 'Vin Diesel', 'Bradley Cooper', 'Zoe Saldana', 'Titanic', 'James Cameron', 'Leonardo DiCaprio', 'Kate Winslet', 'Billy Zane', 'Kathy Bates', 'Joker', 'Todd Phillips', 'Joaquin Phoenix', 'Robert De Niro', 'Zazie Beetz', 'Frances Conroy', 'Léon', 'Luc Besson', 'Jean Reno', 'Gary Oldman', 'Natalie Portman', 'Danny Aiello', 'Goodfellas', 'Henry Hill', 'Martin Scorsese', 'Robert De Niro', 'Ray Liotta', 'Joe Pesci', 'Lorraine Bracco', 'V for Vendetta', 'James McTeigue', 'Hugo Weaving', 'Natalie Portman', 'Rupert Graves', 'Stephen Rea', 'American History X', 'Tony Kaye', 'Edward Norton', 'Edward Furlong', "Beverly D'Angelo", 'Jennifer Lien', 'Kill Bill: Vol. 1', 'Quentin Tarantino', 'Uma Thurman', 'David Carradine', 'Daryl Hannah', 'Michael Madsen', 'WALL·E', 'Andrew Stanton', 'Ben Burtt', 'Elissa Knight', 'Jeff Garlin', 'Fred Willard', 'Pirates of the Caribbean: The Curse of the Black Pearl', 'Gore Verbinski', 'Johnny Depp', 'Geoffrey Rush', 'Orlando Bloom', 'Keira Knightley', 'Terminator 2: Judgment Day', 'James Cameron', 'Arnold Schwarzenegger', 'Linda Hamilton', 'Edward Furlong', 'Robert Patrick', 'The Usual Suspects', 'Bryan Singer', 'Kevin Spacey', 'Gabriel Byrne', 'Chazz Palminteri', 'Stephen Baldwin', 'The Truman Show', 'Peter Weir', 'Jim Carrey', 'Ed Harris', 'Laura Linney', 'Noah Emmerich', 'Iron Man', 'Jon Favreau', 'Robert Downey Jr.', 'Gwyneth Paltrow', 'Terrence Howard', 'Jeff Bridges', 'Braveheart', 'Mel Gibson', 'Mel Gibson', 'Sophie Marceau', 'Patrick McGoohan', 'Angus Macfadyen', 'The Lion King', 'Roger Allers', 'Rob Minkoff', 'Matthew Broderick', 'Jeremy Irons', 'James Earl Jones', 'Whoopi Goldberg', 'Finding Nemo', 'Andrew Stanton', 'Lee Unkrich', 'Albert Brooks', 'Ellen DeGeneres', 'Alexander Gould', 'Willem Dafoe', 'Up', 'Pete Docter', 'Bob Peterson', 'Ed Asner', 'Jordan Nagai', 'John Ratzenberger', 'Christopher Plummer', 'Avengers: Endgame', 'Avengers: Infinity War', 'Anthony Russo', 'Joe Russo', 'Robert Downey Jr.', 'Chris Evans', 'Mark Ruffalo', 'Chris Hemsworth']
Traceback (most recent call last):
File "D:\python\chen\作业\综合项目考核.py", line 48, in <module>
Run()
File "D:\python\chen\作业\综合项目考核.py", line 21, in Run
save_file(data) #调用保存数据
File "D:\python\chen\作业\综合项目考核.py", line 43, in save_file
for values in lists: # 存入数据
TypeError: 'NoneType' object is not iterable
进程已结束，退出代码为 1
