回答问题前可以先运行一遍我的代码在修改回答问题可以吗
import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
def getUrls():
urls=[]
baseUrl="https://cq.lianjia.com/ershoufang/"
for i in range(1,101):
url="{}pg{}".format(baseUrl, i )
urls.append(url)
return urls
def gethtmltext(url):
try:
r=requests.get(url,timeout=30)
r.raise_for_status()
return r.text
except:
return ""
def ParsePage(html):
soup = BeautifulSoup(html, features="html.parser")
title = soup.select(".title a[data-el]")
positionInfo = soup.select(".positionInfo")
houseInfo = soup.select(".houseInfo")
followInfo = soup.select(".followInfo")
totalPrice = soup.select(".totalPrice")
unitPrice = soup.select(".unitPrice")
tag = soup.find_all("div", attrs={"class": "tag"})
count = len(title)
data = []
for i in range(count):
name = ""
region = ""
pos = str(positionInfo[i].text).split("-")
if len(pos) == 2:
name = pos[0]
region = pos[1]
info = str(houseInfo[i].text).split("|")
if len(info) != 7:
break
info[1] = float(info[1].replace("平米", "").strip())
focus_num = ""
publish_time = ""
f_info= str(followInfo[i].text).split("/")
if len(f_info)==2 :
focus_num=f_info[0]
publish_time= f_info[1]
total=str(totalPrice[i].text)
total= float(total.replace("万","").strip())
unit = str(unitPrice[i].text)
unit = unit.replace(",", "")
unit = str(unit.replace("单价","").replace("元/平米","").strip())
tag1 = tag[i].find_all("span")
tags=""
for t in tag1:
tags += t.text+" "
data.append([title[i].text,name, region ,
info[0],info[1],info[2],info[3],info[4],info[5],
focus_num,publish_time,
total,unit, tags ])
return data
head = []
df = pd.DataFrame(head, columns=["标题", "名称", "区域", "户型", "面积", "朝向", "装修", "楼层", "结构", "关注", "时间", "总价", "单价","标签"])
df.to_csv("data.csv", mode="a", header=True, index=False, encoding="utf-8")
x = getUrls()
count =1
for i in x:
html =gethtmltext(i)
pagedata =ParsePage(html)
df= pd.DataFrame(pagedata)
df.to_csv("data.csv",mode="a",index=False,header=False, encoding="utf-8")
print("第{}页写入完毕！".format(count))
time.sleep(3)
count+=1
只能保存画线处的6个以 “ | ” 切分的数据
5个的不行
