关于国家企业信用信息公示网的抓取？

Question

现在在研究爬虫，但是因为技术有限所以磕磕碰碰，在抓取信用公示网的时候，遇到很多问题，大部分在网上搜索相关解决方法和自己摸索都解决了。目前这个网站主要是用了一种叫做加速乐的技术，也就是当访问时系统会返回521，但是获取参数会有jsluid参数，把这个参数加入cookie之后再次访问，会返回jsl_clearance参数，然后再利用参数进行访问才可以。目前jsluid已经拿到，但是返回的js没有搞明白，已经卡了一天了。请各位大神帮忙看看。

 var _16=function(){setTimeout('location.href=location.pathname+location.search.replace(/[\?|&]captcha-challenge/,\'\')',1500);document.cookie='__jsl_clearance=1536808582.071|0|'+(function(){var _56=[function(_16){return _16},function(_56){return _56},function(_16){return return('String.fromCharCode('+_16+')')}],_41=[((-~~~!!window['callP'+'hantom']<<-~[]-~-~!!window['callP'+'hantom'])+[]+[[]][0]),(-~{}+(-~~~!!window['callP'+'hantom']<<-~[]-~-~!!window['callP'+'hantom'])+[]+[]),(-~!!window['callP'+'hantom']+[])+(~~''+[]+[]),(-~!!window['callP'+'hantom']+[])+((-~[]+[-~-~!!window['callP'+'hantom']]>>-~-~!!window['callP'+'hantom'])+[]),((-~[]+[-~-~!!window['callP'+'hantom']]>>-~-~!!window['callP'+'hantom'])+[]),[-~[]-~-~!!window['callP'+'hantom']-~[]-~-~!!window['callP'+'hantom']],[(+!window.headless)+(+!window.headless)],(-~!!window['callP'+'hantom']+[])+(-~!!window['callP'+'hantom']+[]),((+!window.headless)+(+!window.headless)+(+!window.headless)+(+!window.headless)+[]+[]),(7+[]+[[]][0]),(-~!!window['callP'+'hantom']+[]),(-~!!window['callP'+'hantom']+[])+[(+!window.headless)+(+!window.headless)],(~~''+[]+[]),[-~(((+!window.headless)<<(+!window.headless)+(+!window.headless)))]];for(var _16=0;_16<_41.length;_16++){_41[_16]=_56[[0,2,1,0,1,2,1,2,0,1,2,1,2,1][_16]]([[(7+[]+[[]][0])+(~~''+[]+[]),((-~~~!!window['callP'+'hantom']<<-~[]-~-~!!window['callP'+'hantom'])+[]+[[]][0])+(7+[]+[[]][0])],[((-~[]+[-~-~!!window['callP'+'hantom']]>>-~-~!!window['callP'+'hantom'])+[])+(7+[]+[[]][0])],'99lV','%2B2a',((-~[]+[-~-~!!window['callP'+'hantom']]>>-~-~!!window['callP'+'hantom'])+[]),'k%3D',[(-~!!window['callP'+'hantom']+[])+(-~!!window['callP'+'hantom']+[])+((-~[]+[-~-~!!window['callP'+'hantom']]>>-~-~!!window['callP'+'hantom'])+[]),(-~!!window['callP'+'hantom']+[])+(~~''+[]+[])+(7+[]+[[]][0])],'zS',[window['callP'+'hantom']%~~''+[]+[]][0].charAt(~~!!window['callP'+'hantom']),[[-~[]-~-~!!window['callP'+'hantom']-~[]-~-~!!window['callP'+'hantom']]+[-~[]-~-~!!window['callP'+'hantom']-~[]-~-~!!window['callP'+'hantom']]],'LyEwh35%2F',[[-~[]-~-~!!window['callP'+'hantom']-~[]-~-~!!window['callP'+'hantom']]+[-~[]-~-~!!window['callP'+'hantom']-~[]-~-~!!window['callP'+'hantom']]],'2',({}+[[]][0]).charAt(-~!!window['callP'+'hantom'])][_41[_16]])};return _41.join('')})()+';Expires=Thu, 13-Sep-18 04:16:22 GMT;Path=/;'};if((function(){try{return !!window.addEventListener;}catch(e){return false;}})()){document.addEventListener('DOMContentLoaded',_16,false)}else{document.attachEvent('onreadystatechange',_16)}

这里的jslclearance是 1536808582.071|0| 后面上一部分，但是这部分的代码没有搞懂，希望大家帮忙看看。

谢谢~~~~

Answer 1

document.cookie 到;Expires=Thu, 13-Sep-18 04:16:22 GMT;Path=/;这里是整个cookie的生成过程，
Matcher matcher = Pattern.compile("__jsl_clearance.+?Expires").matcher(js1);
if (matcher.find()) {
String js2 = ("function getKey(aa) { var cookie='" + matcher.group().replace("document.cookie", ""))
.replace("Expires", "").replace("';", ");return cookie + getHalf();}")
.replace(")()+", "")
.replaceAll("document\.createElement.+?firstChild\.href", "'" + requestBase.getURI().toString() + "'")
.replace("+(function()", ";function getHalf()").replace("});", "};")
.replaceAll("window", "'Chrome'").replace("'Chrome' ", "'Chrome'");
cookie1 = JavaScript.getKey(js2);
}
尝试将拆成两个js，将无法执行的window替换成chrome就ok了