爬取需要登录的网站,登陆之后跳转到下载页面,获取下载链接并下载,现在问题是登录了但是下载不了 40C

https://blog.csdn.net/weixin_39695028/article/details/80299078

import requests
from lxml import html
import sys
from you_get import common as you_get
import urllib.request

import urllib.request as urllib2

import re
from urllib import parse
import os
import sys
from you_get import common as you_get #导入you-get库
import urllib
from urllib import request
import requests
from bs4 import BeautifulSoup
import json
from bs4 import BeautifulSoup

from six.moves import urllib

postData={

'userid': 'llabc199262',

'pwd': 'llabc199262'

}

session_requests = requests.session()

login_url = "http://club.dearedu.com/member/index.php"

result = session_requests.get(login_url)

result = session_requests.post(

login_url,

data = postData,

headers = dict(referer=login_url)

)

import urllib.request
url1 = "http://club.dearedu.com/member/index.php"
headers = {'User-Agent': "Mozilla/5.0 (iPad; CPU OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1",
'cookie': "__jsluid_h=8e388e30d16bb1e10f73865f9a77e413; UM_distinctid=16bbf9691e24c3-0a6e0f5e007e47-3c604504-1fa400-16bbf9691e33c2; guid=42770979.3940818425674125000.1562312902694.094; Hm_lvt_482d1a083f7aef77357d5a920988d5ac=1562287444,1562334791,1563864719; PHPSESSID=gu3663kt6ao1gh67nbh6d9cqg0; DedeUserID=9058331; DedeUserIDckMd5=63c93468fbe47b82; DedeLoginTime=1563865398; DedeLoginTime__ckMd5=53ce53c6df5208fd; UCenter_username=llabc199262; UCenter_uid=9056889; Example_auth=8b2eRflplt%2FzXmqHdAmQ4QwVNJ14wO1lJNnWLUr9EsZnXAxyIMu%2Fcdi8wB%2B5Zdf%2B; monitor_count=2; Hm_lpvt_482d1a083f7aef77357d5a920988d5ac=1563865400"}
req = urllib.request.Request(url=url1, headers=headers)
response = urllib.request.urlopen(req)
with open(r"D:\SW file\pycharm file\400集\爬虫\教育文档\第二教育网\a.html", 'w', encoding='utf-8') as f:
f.write(response.read().decode('gbk'))

url2 = "http://club.dearedu.com/member/down_gb_iweike.php?zid=5408682&price=0"

url3 = "https://dl.glzy8.com/upfiles/soft/2019/0527/jijianhuisefenggongzuojihuazongjiehuibaopptmb.rar"

req1 = urllib.request.Request(url=url3, headers=headers)

response1 = urllib.request.urlopen(req1)

with open(r"D:\SW file\pycharm file\400集\爬虫\教育文档\第二教育网\a.html", 'w') as f:

# f.write(response1.read().decode('gbk', 'ignore').replace(u'\0xb1', u''))
# f.write(response1.read().decode('gbk', 'ignore').replace(u'\0xe7', u''))

麻烦大神帮忙看看,http://club.dearedu.com/member/index.php是一个教育网站,我用cookie登录网站之后获取了下载链接,但是下载不了文件

toPath = r"D:\SW file\pycharm file\400集\爬虫\教育文档\第二教育网"

try:

for k in range(5408682,1000000000000):

url1 = "http://club.dearedu.com/member/down_gb_iweike.php?zid=" + str(k) +"&price=0"

print(url1)

sys.argv = ['you-get', '-o', toPath, url1]

you_get.main()

except:

pass

def bCrawler(url):
req = urllib.request.Request(url=url, headers=headers) # 拿出的是字典
response1 = urllib.request.urlopen(req)
# HtmlStr = response1.read().encode('utf-8', 'ignore').replace(u'\0xa3', u'')
HtmlStr = response1.read().decode('utf-8', 'ignore').replace(u'\0xa3', u'')
# print(HtmlStr)
pat = r'zid=(.*?)&price=0" class="layui-btn">'
re_img = re.compile(pat,re.S)
bList = re_img.findall(HtmlStr)
print(bList)
print(len(bList))
# try:
for bUrl in bList:
url5 = "http://club.dearedu.com/member/down_gb_iweike.php?zid=" + bUrl + "&price=0"
print(url5)
f1 = requests.get(url5,headers=headers)
with open(r"D:\SW file\pycharm file\400集\爬虫\教育文档\第二教育网\a.doc","a+") as f: #encoding="utf-8"
f.write(f1.content)

    # urllib.request.urlretrieve(url5,r"D:\SW file\pycharm file\400集\爬虫\教育文档\第二教育网\sina.docx",data=headers)

# except:
#     pass


# with open(r"D:\SW file\pycharm file\400集\爬虫\教育文档\第二教育网\a.docx","w") as f:    #encoding="utf-8"
#     f.write(HtmlStr)

#url中如果有中文字符必须进行解码
url = r"http://s.dearedu.com/?page=2&rows=10&attr_free=1"

toPath = r"D:\SW file\pycharm file\400集\爬虫\教育文档\第二教育网"#image文件夹必须先建立

bCrawler(url, toPath)

num = 0
for i in range(556, 12204):
url = "http://s.dearedu.com/?page=" + str(i) + "&rows=10&attr_free=1"
print(url)
print(i)
bCrawler(url)

此代码目前存在的问题:登陆成功但是不能下载

caokaishui
HS_ldy 都没有人来给我解答
11 个月之前 回复

3个回答

很明显是你访问下载资源的时候没有附加正确的sessionid作为cookie,导致服务器还是认为你没有登录。
你可以用fiddler这个工具对比下你用浏览器下载和你的程序下载两者附加的cookie的不同,看缺少了什么。

caokaishui
HS_ldy 可否留个联系方式,兄弟直接帮我代码改下,成功了c币好商量
大约一年之前 回复
caokaishui
HS_ldy 我用的是request方法,已经带上了cookie,已经爬取了需要登录才能获得的下载连接,但是就是下载没有反应,开始用的是yoget,发现yoget不能带cookie下载,后面就用request和with open,也还是不行,麻烦根据我上面的代码改改,c币好说
大约一年之前 回复

你下载的request中有没有带上对应的cookie等信息,不然服务器还是可能认为你没有认证

cookie不能直接用字符串,解析处理一下

from http.cookies import SimpleCookie

rawdata = 'Cookie: devicePixelRatio=1; ident=exists; __utma=13103r6942.2918; __utmc=13103656942; __utmz=13105942.1.1.1.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided); mp_3cb27825a6612988r46d00tinct_id%22%3A%201752338%2C%22%24initial_referrer%22%3A%20%22https%3A%2F%2Fwww.pion_created_at%22%3A%20%222015-08-03%22%2C%22platform%22%3A%20%22web%22%2C%%22%3A%20%%22%7D; t_session=BAh7DUkiD3Nlc3NpbWVfZV9uYW1lBjsARkkiH1BhY2lmaWMgVGltZSAoVVMgJiBDYW5hZGEpBjsAVEkiFXNpZ25pbl9wZXJzb25faWQGOwBGaQMSvRpJIhRsYXN0X2xvZ2luX2RhdGUGOwBGVTogQWN0aXZlU3VwcG9ydDo6VGltZVdpdGhab25lWwhJdToJVGltZQ2T3RzAAABA7QY6CXpvbmVJIghVVEMGOwBUSSIfUGFjaWZpZWRfZGFzaGJvYXJkX21lc3NhZ2UGOwBGVA%3D%3D--6ce6ef4bd6bc1a469164b6740e7571c754b31cca'
cookie = SimpleCookie()
cookie.load(rawdata)

# Even though SimpleCookie is dictionary-like, it internally uses a Morsel object
# which is incompatible with requests. Manually construct a dictionary instead.
cookies = {}
for key, morsel in cookie.items():
    cookies[key] = morsel.value
caokaishui
HS_ldy 请直接在我的代码中修改完发给我可好
大约一年之前 回复
caokaishui
HS_ldy 我用的是request方法,已经带上了cookie,已经爬取了需要登录才能获得的下载连接,但是就是下载没有反应,开始用的是yoget,发现yoget不能带cookie下载,后面就用request和with open,也还是不行,麻烦根据我上面的代码改改,c币好说
大约一年之前 回复

cookie要弄成字典的形式显示,不能字符串

caokaishui
HS_ldy 这些人说的都是错的,还是我自己来吧
11 个月之前 回复
caokaishui
HS_ldy 你好,我现在成功爬取了内容,但是内容是这样的:b'\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>\x00\x03\x00\xfe\xff\t\x00\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\xda\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\xdc\x00\x00\x00\x01\x00\x00\x00\xfe\xff\xff\xff\x00\x00\x00\x00\xd8\x00\x00\x00\xd9\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff这个是二进制的吗,还是说已经被反爬虫加密了
大约一年之前 回复
Csdn user default icon
上传中...
上传图片
插入图片
抄袭、复制答案,以达到刷声望分或其他目的的行为,在CSDN问答是严格禁止的,一经发现立刻封号。是时候展现真正的技术了!
立即提问
相关内容推荐