lma8811
Lomon----
采纳率18.2%
2019-01-29 20:25 阅读 673

需要一个JAVA采集器的框架

300

我需要对人民网地方领导留言板的留言进行遍历采集,需要一个JAVA采集器,MYSQL数据库方面我会来解决,功能主要是:
1、对每条JSON内容进行采集。
2、对JOSN里面TID的页面内容,也就是在 [查看全文]点开后进行详细内容采集。如:http://liuyan.people.com.cn/threads/content?tid=5668549
3、能实现【加载更多】。

样例页面:http://liuyan.people.com.cn/threads/list?fid=1079

以下是JOSN获取的代码:

package com.java.activiti.controller;



import net.sf.json.JSONObject;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPost;


import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

public class python {
    public static void main(String[] args) {
        String url = "http://liuyan.people.com.cn/threads/queryThreadsList?fid=1079&lastItem=0";
        HttpClient httpClient = null;
        HttpPost httpPost = null;
        HttpResponse response = null;
        try {
            httpClient = HttpClients.createDefault();
            httpPost = new HttpPost(url);// 传入URL地址
            httpPost.addHeader("Accept",
                    "pplication/json, text/javascript, */*; q=0.01");
            httpPost.addHeader("Accept-Encoding", "gzip, deflate");// 设置请求头
            httpPost.addHeader("Accept-Language",
                    "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2");// 设置请求头
            //httpPost.addHeader("Connection", "keep-alive");// 设置请求头
            // httpPost.addHeader("Content-Length", "19");//设置请求头
            //httpPost.addHeader("Content-Type",
                //    "application/x-www-form-urlencoded; charset=UTF-8");// 设置请求头

            httpPost.addHeader("Referer",
                    "http://liuyan.people.com.cn/threads/list?fid=1079");// 设置请求头
            //httpPost.addHeader("User-Agent",
            //        "Mozilla/5.0 (Windows NT 6.1; W…) Gecko/20100101 Firefox/64.0");// 设置请求头
            //httpPost.addHeader("X-Requested-With", "XMLHttpRequest");// 设置请求头

            response = httpClient.execute(httpPost);// 获取响应
            int statusCode = response.getStatusLine().getStatusCode();
            System.out.println("HTTP Status Code:" + statusCode);
            if (statusCode != HttpStatus.SC_OK) {
                System.out.println("HTTP请求未成功!HTTP Status Code:"
                        + response.getStatusLine());
            }
            HttpEntity httpEntity = response.getEntity();
            String reponseContent = EntityUtils.toString(httpEntity);
            EntityUtils.consume(httpEntity);
            System.out.println("响应内容:" + reponseContent);
        } catch (Exception e) {
            e.printStackTrace();
        }

    }
}

{"result":"success","responseData":[{"tid":5668549,"userId":1852621,"fid":1079,"typeId":2,"domainId":14,"topicId":0,"nickName":"李***","subject":"进贤一中乱收费","content":"您好:\n我是江西省南昌市进贤县进贤一中的学生家长,前段时间进贤一中收取学生600元,校方称其为改卷费。近段时间,自一月26日后,学校开始强制补课,并在1月27日收取学生400元补课费,一月31日结束强...","dateline":1548659737,"ip":"223.104.170.*","stateInfo":"未回复","traceInfo":"未回复","processInfo":"未交办","hasReported":0,"favNum":0,"forumName":"南昌市委书记殷美根","typeName":"投诉","domainName":"教育","sourceName":"手机浏览器","answerId":null,"answerContent":null,"answerDateline":null,"answerOrganization":null},{"tid":5664590,"userId":1850390,"fid":1079,"typeId":2,"domainId":14,"topicId":0,"nickName":"手***","subject":"主题公园别墅无证幼儿园难道就关不了吗?","content":"书记你好!自从别墅区内有智慧树及未来星幼儿园开办以来,两家无证幼儿园相隔不到10米,幼儿园在7月份已经上报了都是现场,为什么一直取缔不了?而且小区内部存在极大的安全隐患,严重扰民,从早到晚都是孩子们的...","dateline":1548487073,"ip":"59.53.7.*","stateInfo":"未回复","traceInfo":"未回复","processInfo":"未交办","hasReported":0,"favNum":0,"forumName":"南昌市委书记殷美根","typeName":"投诉","domainName":"教育","sourceName":"手机浏览器","answerId":null,"answerContent":null,"answerDateline":null,"answerOrganization":null},{"tid":5659972,"userId":1803445,"fid":1079,"typeId":6,"domainId":7,"topicId":0,"nickName":"2***","subject":"拖欠工程材料款","content":"尊敬的省委书记您好:南昌市丽阳装饰工程有限公司拖欠2012年-2017年期间(红谷凯旋办公大楼,汉港凯旋城,宜春御湖城老年公寓)等几个项目工程材料款合计30多万。2017年底审核结算后承诺付款无数次,...","dateline":1548322912,"ip":"117.43.68.*","stateInfo":"已回复","traceInfo":"已回复","processInfo":"未交办","hasReported":0,"favNum":0,"forumName":"南昌市委书记殷美根","typeName":"求助","domainName":"城建","sourceName":"客户端","answerId":3372083,"answerContent":"谢谢你反映情况。建议你通过司法途径追回欠款。未尽之处,敬请谅解。","answerDateline":1548645084,"answerOrganization":"南昌市委民声通道"},{"tid":5658741,"userId":1846587,"fid":1079,"typeId":6,"domainId":12,"topicId":0,"nickName":"圆***","subject":"救救我可怜的娃16岁患上白血病一年多","content":"市委书记你好,帮帮我这个苦难的家庭吧,儿子十四岁多患上急性白血病一年多,在南大一附院治疗把家里积蓄全部花光,还欠钱,没钱治疗就会复发,我想办个贫困户可以多报销点医药费,这次去医院化疗医生说化疗打下去都...","dateline":1548301402,"ip":"59.63.151.*","stateInfo":"已回复","traceInfo":"已回复","processInfo":"未交办","hasReported":0,"favNum":0,"forumName":"南昌市委书记殷美根","typeName":"求助","domainName":"医疗","sourceName":"手机浏览器","answerId":3372022,"answerContent":"谢谢你反映情况。请你到当地县区民政部门咨询相关救助政策。未尽之处,敬请谅解。","answerDateline":1548644492,"answerOrganization":"南昌市委民声通道"},{"tid":5656180,"userId":1840290,"fid":1079,"typeId":6,"domainId":9,"topicId":0,"nickName":"x***","subject":"第二次:乐化农民建房为什么不能“拆旧建新“?","content":"我家老房是42年前建的砖木结构的旧式瓦房,村子其他村民都建了楼房,也是全村唯一独剩的旧房,已是非常陈旧破败,特别是屋顶漏水,年年要检修屋顶瓦片,现在生活条件改善了,想住得舒心一点,向村委会申请不占用耕...","dateline":1548218791,"ip":"116.25.94.*","stateInfo":"未回复","traceInfo":"未回复","processInfo":"未交办","hasReported":0,"favNum":0,"forumName":"南昌市委书记殷美根","typeName":"求助","domainName":"三农","sourceName":"PC浏览器","answerId":null,"answerContent":null,"answerDateline":null,"answerOrganization":null},{"tid":5648308,"userId":1223889,"fid":1079,"typeId":2,"domainId":7,"topicId":0,"nickName":"天***","subject":"朱桥东路佛塔村段商贩在路中间摆摊","content":"每逢农历的初二 初五 初八 十二 十五 十八 廿二 廿五 廿八 是佛塔村当街的日子,以前只在村里的农贸市场摆摊,后来慢慢摆到朱桥东路上的非机动车道,现在摆到路中间了 不仅影响市容,也...","dateline":1547952618,"ip":"59.53.30.*","stateInfo":"未回复","traceInfo":"未回复","processInfo":"未交办","hasReported":0,"favNum":0,"forumName":"南昌市委书记殷美根","typeName":"投诉","domainName":"城建","sourceName":"PC浏览器","answerId":null,"answerContent":null,"answerDateline":null,"answerOrganization":null},{"tid":5642495,"userId":1139396,"fid":1079,"typeId":6,"domainId":9,"topicId":0,"nickName":"8***","subject":"百姓苦,苦于求助无门","content":"敬爱的书记您好,我是您治下的南昌市湾里区罗亭镇义坪村松山组村民,是这样的,由于我母亲是残疾,天生小儿麻痹症,有残疾证,我想说我母亲怎么就没有享受国家低保政策的,那么我母亲没有资格的话,那么我想问,在我...","dateline":1547732335,"ip":"115.148.224.*","stateInfo":"未回复","traceInfo":"未回复","processInfo":"未交办","hasReported":0,"favNum":0,"forumName":"南昌市委书记殷美根","typeName":"求助","domainName":"三农","sourceName":"客户端","answerId":null,"answerContent":null,"answerDateline":null,"answerOrganization":null},{"tid":5639297,"userId":1835490,"fid":1079,"typeId":6,"domainId":7,"topicId":0,"nickName":"中***","subject":"关于南昌新力龙湾项目违规收取价外款的举报材料","content":"关于南昌新力龙湾项目违规收取价外款的举报材料尊敬的各位领导:南昌宝葫芦农庄有限公司开发的“新力龙湾”项目违规收取价位款,无视中央三令五申调控房价的精神和省市关于禁止收取“价外款”的文件规定,顶风违规收...","dateline":1547637410,"ip":"182.97.215.*","stateInfo":"未回复","traceInfo":"未回复","processInfo":"未交办","hasReported":0,"favNum":0,"forumName":"南昌市委书记殷美根","typeName":"求助","domainName":"城建","sourceName":"客户端","answerId":null,"answerContent":null,"answerDateline":null,"answerOrganization":null},{"tid":5629109,"userId":1778124,"fid":1079,"typeId":5,"domainId":7,"topicId":18,"nickName":",***","subject":"安置房问题","content":"希望国家对于安置房问题应该高度集中,加强监管力度。i地方的办事效率应该提高了。","dateline":1547274215,"ip":"117.166.80.*","stateInfo":"已回复","traceInfo":"已回复","processInfo":"未交办","hasReported":0,"favNum":0,"forumName":"南昌市委书记殷美根","typeName":"建言","domainName":"城建","sourceName":"客户端","answerId":3361266,"answerContent":"谢谢你所提的建议。我室会将你的建议转达给相关部门。未尽之处,敬请谅解。","answerDateline":1547438837,"answerOrganization":"南昌市委民声通道"},{"tid":5627829,"userId":1827474,"fid":1079,"typeId":6,"domainId":20,"topicId":0,"nickName":"你***","subject":"享骑出行公司不退押金","content":"市委书记您好,自2018年12月10日起,申请享骑出行退押金299元到现在为止,押金迟迟未退,已经超过约定时间,客服电话一直打不通,公司地址不详,不知道怎么办,希望市委书记协助我退押金,万分感谢,祝市...","dateline":1547211208,"ip":"223.104.172.*","stateInfo":"已回复","traceInfo":"已回复","processInfo":"未交办","hasReported":0,"favNum":0,"forumName":"南昌市委书记殷美根","typeName":"求助","domainName":"金融","sourceName":"客户端","answerId":3361083,"answerContent":"谢谢你反映情况。关于消费者维权问题,请拨打市市场监管局举报热线12315,向有关人员反映。另外,请你最好提供公司注册地址,以便确认是否属于我市市场监管局管辖范围。未尽之处,敬请谅解。","answerDateline":1547431066,"answerOrganization":"南昌市委民声通道"}]}

  • 点赞
  • 写回答
  • 关注问题
  • 收藏
  • 复制链接分享

4条回答 默认 最新

  • devmiao devmiao 2019-01-29 23:51
    点赞 评论 复制链接分享
  • u012976158 强化脑细胞 2019-01-30 09:49

    就是对json串的解析呀, JSONArray.parseArray(jsonString, clazz);转成对象集合

    点赞 评论 复制链接分享
  • qq_21741673 洗洗脚泡泡澡 2019-01-30 10:31

    把获取到的JSON字符串转成实体入库就好了啊、加载更多就是分页咯。

    点赞 评论 复制链接分享
  • wojiushiwo945you 毕小宝 2019-01-30 14:16

    可以看下Java爬虫轻量级框webmagic,可能满足您的要求。

    点赞 评论 复制链接分享

相关推荐