MapReducer 写入到数据库 报错

【 DBUserWritable 类 】

package org.neworigin.com.Database;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.lib.db.DBWritable;

public class DBUserWritable implements DBWritable,WritableComparable{

private String name="";
private String sex="";
private int age=0;
private int num=0;
private String department="";
private String tables="";

@Override
public String toString() {
    return "DBUserWritable [name=" + name + ", sex=" + sex + ", age=" + age + ", department=" + department + "]";
}


public DBUserWritable(DBUserWritable d){
    this.name=d.getName();
    this.sex=d.getSex();
    this.age=d.getAge();
    this.num=d.getNum();
    this.department=d.getDepartment();
    this.tables=d.getTables();
}
public String getName() {
    return name;
}

public void setName(String name) {
    this.name = name;
}

public String getSex() {
    return sex;
}

public void setSex(String sex) {
    this.sex = sex;
}

public int getAge() {
    return age;
}

public void setAge(int age) {
    this.age = age;
}

public int getNum() {
    return num;
}

public void setNum(int num) {
    this.num = num;
}

public String getDepartment() {
    return department;
}

public void setDepartment(String department) {
    this.department = department;
}

public String getTables() {
    return tables;
}

public void setTables(String tables) {
    this.tables = tables;
}

public DBUserWritable(String name, String sex, int age, int num, String department, String tables) {
    super();
    this.name = name;
    this.sex = sex;
    this.age = age;
    this.num = num;
    this.department = department;
    this.tables = tables;
}

public DBUserWritable() {
    super();
    // TODO Auto-generated constructor stub
}

public void write(DataOutput out) throws IOException {
    // TODO Auto-generated method stub
    out.writeUTF(name);
    out.writeUTF(sex);
    out.writeInt(age);
out.writeInt(num);
    out.writeUTF(department);
out.writeUTF(tables);
}

public void readFields(DataInput in) throws IOException {
    // TODO Auto-generated method stub
    name = in.readUTF();
    sex=in.readUTF();
    age=in.readInt();
    num=in.readInt();
    department=in.readUTF();
    tables=in.readUTF();
}

public int compareTo(Object o) {
    // TODO Auto-generated method stub
    return 0;
}

public void write(PreparedStatement statement) throws SQLException {
    // TODO Auto-generated method stub
    statement.setString(1, this.getName());
    statement.setString(2, this.getSex());
    statement.setInt(3, this.getAge());
    statement.setString(4, this.getDepartment());
}

public void readFields(ResultSet resultSet) throws SQLException {
    // TODO Auto-generated method stub
     this.name=resultSet.getString(1);
     this.sex=resultSet.getString(2);
     this.age=resultSet.getInt(3);
     this.department=resultSet.getString(4);
}

}

【mapper】

package org.neworigin.com.Database;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class UserDBMapper extends Mapper {
DBUserWritable DBuser= new DBUserWritable();
@Override
protected void map(LongWritable key, Text value, Mapper.Context context)
throws IOException, InterruptedException {
String[] values=value.toString().split(" ");

    if(values.length==4){

     DBuser.setName(values[0]);
     DBuser.setSex(values[1]);
     DBuser.setAge(Integer.parseInt(values[2]));
     DBuser.setNum(Integer.parseInt(values[3]));
     DBuser.setTables("t1");
     System.out.println("mapper---t1---------------"+DBuser);
     context.write(new Text(values[3]),DBuser);

     }


        if(values.length==2){

             DBuser.setNum(Integer.parseInt(values[0]));
             DBuser.setDepartment(values[1]);   
             DBuser.setTables("t2");
             context.write(new Text(values[0]),DBuser);
             //System.out.println("mapper --t2"+"--"+values[0]+"----"+DBuser);
        }

}

}

【reducer 】

package org.neworigin.com.Database;

import java.io.IOException;
import java.util.LinkedList;
import java.util.List;

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class UserDBReducer extends Reducer {
// public DBUserWritable db= new DBUserWritable();

@Override
protected void reduce(Text k2, Iterable<DBUserWritable> v2,
        Reducer<Text, DBUserWritable, NullWritable,DBUserWritable>.Context context) throws IOException, InterruptedException {
    String Name="";
 List<DBUserWritable> list=new LinkedList<DBUserWritable>();
 for(DBUserWritable val : v2){
     list.add(new DBUserWritable(val));//new 一个对象 给list

// System.out.println("[table]"+val.getTables()+"----key"+k2+"---"+val);
if(val.getTables().equals("t2")){
Name=val.getDepartment();

     }
 }
 //键是  num  
for(DBUserWritable join : list){
    System.out.println("[table]"+join.getTables()+"----key"+k2+"---"+join);
if(join.getTables().equals("t1")){

    join.setDepartment(Name);
System.out.println("db-----"+join);
    context.write(NullWritable.get(), join);
    }
}

}

}

【app】

package org.neworigin.com.Database;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.db.DBConfiguration;
import org.apache.hadoop.mapreduce.lib.db.DBOutputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class UserDBAPP {

public static void main(String[] args) throws Exception, URISyntaxException {
    // TODO Auto-generated method stub
   String INPUT_PATH="file:///E:/BigData_eclipse_database/Database/data/table1";
   String INPUT_PATH1="file:///E:/BigData_eclipse_database/Database/data/table2";

// String OUTPUT_PARH="file:///E:/BigData_eclipse_database/Database/data/output";
Configuration conf = new Configuration();
// FileSystem fs=FileSystem.get(new URI(OUTPUT_PARH),conf);
// if(fs.exists(new Path(OUTPUT_PARH))){
// fs.delete(new Path(OUTPUT_PARH));
// }

   Job job = new Job(conf,"mydb");
   //设置数据库配置

   DBConfiguration.configureDB(conf, "com.mysql.jdbc.Driver", "jdbc:mysql://localhost/hadoop", "root", "123456");


   FileInputFormat.addInputPaths(job,INPUT_PATH);
   FileInputFormat.addInputPaths(job,INPUT_PATH1);

   job.setMapperClass(UserDBMapper.class);
   job.setMapOutputKeyClass(Text.class);
   job.setMapOutputValueClass(DBUserWritable.class);

   job.setReducerClass(UserDBReducer.class);
   job.setOutputKeyClass(NullWritable.class);
   job.setOutputValueClass(DBUserWritable.class);

// FileOutputFormat.setOutputPath(job, new Path(OUTPUT_PARH));
//设置输出路径
DBOutputFormat.setOutput(job,"user_tables", "name","sex","age","department");
job.setOutputFormatClass(DBOutputFormat.class);
boolean re = job.waitForCompletion(true);
System.out.println(re);
}

}

【报错】ps 表链接 ,写到本地没问题 写到数据库 就报错;
17/11/10 11:39:11 WARN output.FileOutputCommitter: Output Path is null in cleanupJob()
17/11/10 11:39:11 WARN mapred.LocalJobRunner: job_local1812680657_0001
java.lang.Exception: java.io.IOException
at org.apache.hadoop.mapred.LocalJobRunner$Job.runTasks(LocalJobRunner.java:462)
at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:529)
Caused by: java.io.IOException
at org.apache.hadoop.mapreduce.lib.db.DBOutputFormat.getRecordWriter(DBOutputFormat.java:185)
at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.(ReduceTask.java:541)
at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:614)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389)
at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:319)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
17/11/10 11:39:12 INFO mapreduce.Job: Job job_local1812680657_0001 running in uber mode : false
17/11/10 11:39:12 INFO mapreduce.Job: map 100% reduce 0%
17/11/10 11:39:12 INFO mapreduce.Job: Job job_local1812680657_0001 failed with state FAILED due to: NA
17/11/10 11:39:12 INFO mapreduce.Job: Counters: 35

0

2个回答

会不会少了端口号3306,jdbc:mysql://localhost/hadoop

0
qq_38200425
Ziv2U DBConfiguration.configureDB(conf, "com.mysql.jdbc.Driver", "jdbc:mysql://localhost/hadoop", "root", "123456"); 这个主函数里面写的有
接近 2 年之前 回复

DBConfiguration.configureDB(conf, "com.mysql.jdbc.Driver", "jdbc:mysql://localhost/hadoop", "root", "123456");

这个主函数里面写的有

0
Csdn user default icon
上传中...
上传图片
插入图片
抄袭、复制答案,以达到刷声望分或其他目的的行为,在CSDN问答是严格禁止的,一经发现立刻封号。是时候展现真正的技术了!
其他相关推荐
sql数据库写入报错
Line 1:Incorrect syntax near 'http:'.rnrn请教这句话的意思是什么啊
单词统计,MapReducer处理数据,写入HBase数据库(案例)
WordCountDemo package com.word; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUt...
MapReducer多表查询
一个例子晚上例子,别处看来的,但是有些代码看不懂 表1 factoryname addressed Beijing Red Star 1 Shenzhen Thunder 3 Guangzhou Honda 2 Beijing Rising 1 Guangzhou Development Bank 2 Tencent 3 Back of Beijing 1 表2 addr
写入数据库总报错怎么回事?
怎么弄都是错误,只得在讨教了!rn什么叫做“标准表达式中数据类型不匹配”?rnrst.Open "update table set reply='"&request("content")&"',retime='"now"' where lid ='"&session("id")&"'",cn,2,3 ' 25 行rnrnrnMicrosoft VBScript 编译器错误 错误 '800a03f6' rnrn缺少 'End' rnrn/iisHelp/common/500-100.asp,行242 rnrnMicrosoft OLE DB Provider for ODBC Drivers 错误 '80040e07' rnrn[Microsoft][ODBC Microsoft Access Driver] 标准表达式中数据类型不匹配。 rnrn/hj/board/reply.asp,行25 rnrnrnrn
datagrid内容写入数据库报错
For i = 0 To DataGrid1.ApproxCount - 1rn DataGrid1.Row = irn For j = 0 To DataGrid1.Columns.Count - 1rn rs.AddNewrn rs.Fields(j).Value = DataGrid1.Columns(j).Valuern Next jrn rs.Updatern Next irn报多步操作产生错误,请检查每一步的状态值rn按下调试显示问题出在 rs.Fields(j).Value = DataGrid1.Columns(j).Value
Mapreducer任务启动
一、Streaming 创建start.sh文件,内容如下 #!/bin/bash hadoop fs -rm -r/data/apps/zhangwenchao/mapreduce/streaming/wordcount/output   hadoop jar /data/tools/hadoop/hadoop-2.6.2/share/hadoop/tools/lib/hadoop
mapreducer工作流程
hadoop mapreduce中对split的处理 分片: 1. 在job.waitForCompletion(true)后使用job.submit() 提交job,之后调用 submitter.submitJobInternal(Job.this, cluster); 2. 在submitJobInternal()函数中 会给job创建分片 int maps = writeSplits(j...
MapReducer的执行流程
MapReducer工作流程图: MapReducer工作流程: ------------------------------------------------mapper的工作-------------------------------------------- 1.mapper开始运行,调用InputFormat组件读取文件逻辑切片(逻辑切片不是block块,切片大小默认和block块大...
MapReducer中文编码
如果输入文件有中文,输出文件可能会出现乱码。乱码问题的话一般都是编解码错误。 本文的最后有一篇参考文档,那篇文档已经解决了问题了,但是,可是自己是小白,那篇文档没做解释的话,有的我理解不了,所以就把那篇文章解释了一遍。基础好的可以直接看那篇文章。编码的问题首先看一下这段代码String s = "中国"; Text t1 = new Text(); Text t2 = new Text();t1.
mapReducer的测试案例①
需求: 实现统计 每个手机号的 上行包 下行包  总包   案例资源和文件:http://pan.baidu.com/s/1eSMmpkm 首先定义了一类接收数据处理过程中map阶段输出的value. package com.vampire.taobao;   import java.io.DataInput; import java.io.DataOutput; imp
MapReducer流量统计
1、对流量日志中的用户统计总上、下行流量 技术点:自定义javaBean用来在mapreduce中充当value 注意: javaBean要实现Writable接口,实现两个方法 package com.mr.flowsum; import org.apache.hadoop.io.WritableComparable; import java.io.DataInput; import j
MapReducer程序调试技巧
  写过程序分布式代码的人都知道,分布式的程序是比较难以调试的,但是也不是不可以调试,对于Hadoop分布式集群来说,在其上面运行的是mapreduce程序,因此,有时候写好了mapreduce程序之后,执行结果发现跟自己想要的结果不一样,但是有没有报错,此时就很难发现问题,查找问题的方法之一就是对程序进行调试,跟踪代码的执行,找出问题的所在。那么对于Hadoop的Mapreduce是如...
log4j 日志内容写入数据库报错。
配置文件如下:log4j.rootLogger=DEBUG,DATABASErn rnlog4j.appender.DATABASE=org.apache.log4j.jdbc.JDBCAppenderrnlog4j.appender.DATABASE.URL=jdbc:oracle:thin:@localhost:1521:cnyxlxwmrnlog4j.appender.database.BufferSize=40 rnlog4j.appender.DATABASE.driver=oracle.jdbc.driver.OracleDriverrnlog4j.appender.DATABASE.user=vio_proofrnlog4j.appender.DATABASE.password=vio_proofrnlog4j.appender.DATABASE.sql=INSERT INTO PROOF_EXCEPTION(MESSAGE,DATE_HAPPEN) values('%c-%-4r[%t]%-5p%c%x-%m%n',sysdate)rnlog4j.appender.DATABASE.layout=org.apache.log4j.PatternLayout rnlog4j.appender.DATABASE.layout.ConversionPattern=-%c-%-4r[%t]%-5p%c%x-%m%nrnrn错误信息如下:rnlog4j:ERROR Failed to excute sqlrnjava.sql.SQLException: ORA-00917: 缺少逗号rnrn at oracle.jdbc.dbaccess.DBError.throwSqlException(DBError.java:187)rn at oracle.jdbc.ttc7.TTIoer.processError(TTIoer.java:241)rn at oracle.jdbc.ttc7.Oall7.receive(Oall7.java:543)rn at oracle.jdbc.ttc7.TTC7Protocol.doOall7(TTC7Protocol.java:1477)rn at oracle.jdbc.ttc7.TTC7Protocol.parseExecuteFetch(TTC7Protocol.java:888)rn at oracle.jdbc.driver.OracleStatement.executeNonQuery(OracleStatement.java:2030)rn at oracle.jdbc.driver.OracleStatement.doExecuteOther(OracleStatement.java:1950)rn at oracle.jdbc.driver.OracleStatement.doExecuteWithTimeout(OracleStatement.java:2591)rn at oracle.jdbc.driver.OracleStatement.executeUpdate(OracleStatement.java:999)rn at org.apache.log4j.jdbc.JDBCAppender.execute(JDBCAppender.java:178)rn at org.apache.log4j.jdbc.JDBCAppender.flushBuffer(JDBCAppender.java:250)rn at org.apache.log4j.jdbc.JDBCAppender.append(JDBCAppender.java:146)rn at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:230)rn at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:65)rn at org.apache.log4j.Category.callAppenders(Category.java:203)rn at org.apache.log4j.Category.forcedLog(Category.java:388)rn at org.apache.log4j.Category.log(Category.java:853)rn at org.apache.commons.logging.impl.Log4JLogger.info(Log4JLogger.java:133)rn at org.quartz.impl.StdSchedulerFactory.instantiate(StdSchedulerFactory.java:1014)rn at org.quartz.impl.StdSchedulerFactory.getScheduler(StdSchedulerFactory.java:1152)rn at org.quartz.impl.StdSchedulerFactory.getDefaultScheduler(StdSchedulerFactory.java:1168)rn at com.citic.util.quartz.OnLineCountListener.contextInitialized(OnLineCountListener.java:55)rn at com.caucho.server.webapp.Application.start(Application.java:1597)rn at com.caucho.server.deploy.DeployController.startImpl(DeployController.java:621)rn at com.caucho.server.deploy.StartAutoRedeployAutoStrategy.startOnInit(StartAutoRedeployAutoStrategy.java:72)rn at com.caucho.server.deploy.DeployController.startOnInit(DeployController.java:509)rn at com.caucho.server.deploy.DeployContainer.start(DeployContainer.java:158)rn at com.caucho.server.webapp.ApplicationContainer.start(ApplicationContainer.java:652)rn at com.caucho.server.host.Host.start(Host.java:385)rn at com.caucho.server.deploy.DeployController.startImpl(DeployController.java:621)rn at com.caucho.server.deploy.StartAutoRedeployAutoStrategy.startOnInit(StartAutoRedeployAutoStrategy.java:72)rn at com.caucho.server.deploy.DeployController.startOnInit(DeployController.java:509)rn at com.caucho.server.deploy.DeployContainer.start(DeployContainer.java:158)rn at com.caucho.server.host.HostContainer.start(HostContainer.java:501)rn at com.caucho.server.resin.ServletServer.start(ServletServer.java:977)rn at com.caucho.server.deploy.DeployController.startImpl(DeployController.java:621)rn at com.caucho.server.deploy.AbstractDeployControllerStrategy.start(AbstractDeployControllerStrategy.java:56)rn at com.caucho.server.deploy.DeployController.start(DeployController.java:517)rn at com.caucho.server.resin.ResinServer.start(ResinServer.java:485)rn at com.caucho.server.resin.Resin.init(Resin.java)rn at com.caucho.server.resin.Resin.main(Resin.java:624)rnrn检查数据库 发现日志内容一直在写入。
Python 使用多线程写入数据库,报错
#问题描述,定义了一个函数,用于数据库写入表,直接使用函数是可以运行,但我创建了一个n多线程,在多线程里面使用运行函数,然后后错nimport mysql.connectornfrom mysql.connector import errorcodenimport threading,timenn#数据库连接参数nndef sqlconn():n dbconfig='user': 'qrcodeadmin','password': 'careyes!@#','host': '119.23.8.62','port':3308,'database': 'qrcodes','raise_on_warnings': Truen try:n cnx = mysql.connector.connect(**dbconfig)n print('数据库连接成功')n except mysql.connector.Error as err:n if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:n print("Something is wrong with your user name or password")n elif err.errno == errorcode.ER_BAD_DB_ERROR:n print("Database does not exist")n else:n print(err.encode)n return cnxnndef handle():n while(True):n global UID,IDn for i in range(50):n print(i)n sqlinsert="INSERT INTO qrcode (URL,QrID) VALUES (%s,%s)",(UID,ID)n cur.execute(sqlinsert[0],sqlinsert[1])n cur.execute(sqlinsert[0],sqlinsert[1])n dbcon.commit()nnnif __name__=='__main__':n global curn dbcon=sqlconn()n cur=dbcon.cursor()n UID=' MjAxNzEwMTExMjQyNDQ'n ID='201710111242420111'n#直接使用handle可以运行n handle()n#-------------n sql_thread=threading.Thread(target=handle)n#如果使用线程就会报错n sql_thread.start()n dbcon.close()n print(dbcon)n n #报错内容n Exception in thread Thread-1:nTraceback (most recent call last):n File "C:\Python\lib\site-packages\mysql\connector\connection.py", line 280, inn _send_cmdn self._socket.send(nAttributeError: 'NoneType' object has no attribute 'send'nnDuring handling of the above exception, another exception occurred:nnTraceback (most recent call last):n File "C:\Python\lib\threading.py", line 914, in _bootstrap_innern self.run()n File "C:\Python\lib\threading.py", line 862, in runn self._target(*self._args, **self._kwargs)n File "sqltestV0.2_Thread.py", line 34, in handlen cur.execute(sqlinsert[0],sqlinsert[1])n File "C:\Python\lib\site-packages\mysql\connector\cursor.py", line 561, in exencuten self._handle_result(self._connection.cmd_query(stmt))n File "C:\Python\lib\site-packages\mysql\connector\connection.py", line 514, inn cmd_queryn result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query))n File "C:\Python\lib\site-packages\mysql\connector\connection.py", line 284, inn _send_cmdn raise errors.OperationalError("MySQL Connection not available.")nmysql.connector.errors.OperationalError: MySQL Connection not available.
insert不能写入数据库,不报错,求救
前几天就没有问题,今天再测试就不行了。昏死了。。。rn是个插入留言的按钮。部分代码:rnrnOleDbConnection con = new OleDbConnection();rnrnprivate void OpenDatabase()rnrn con.ConnectionString = "Provider=Microsoft.Jet.OLEDB.4.0;Data Source=" + Server.MapPath("book.mdb");rn con.Open();rnrnrnprivate void Page_Load(object sender, System.EventArgs e)rnrn // 在此处放置用户代码以初始化页面rn OpenDatabase();rnrnrnprivate void Buttonok_Click(object sender, System.EventArgs e)rnrn OleDbConnection myConnection = con;rn OleDbCommand cmd;rn string sql = "insert into [f_bbs] (f_name,head,f_content,f_oicq,f_email,f_homepage) values ('0','1','2','3','4','5')";rn sql = string.Format(sql,this.TextBoxname.Text,this.Rblist.SelectedItem.Value,this.nr.Text,this.TextBoxoicq.Text,this.TextBoxmail.Text,this.TextBoxurl.Text);rnrn cmd = new OleDbCommand(sql,myConnection);rnrn char[] str=TextBoxoicq.Text.ToCharArray();rn foreach (char c in str)rn rn if (c<48||c>57)rn rn Response.Write(" ");rn return;rn rn else if(this.TextBoxoicq.Text.Length<5)rn rn Response.Write(" ");rn return;rn rn rnrn if( this.TextBoxname.Text != "" && this.nr.Text != "" )rn rn tryrn rn cmd.ExecuteScalar();rn rn finallyrn rn myConnection.Close();rn Response.Redirect("default.aspx");rn rn rn elsern rn Response.Write(" ");rn rn
[MapReducer]求平均数
这是Udacity的课程 intro to hadoop and mapReduce里面Lesson4的练习 只说一点,求平均不能在combiner中进行 算法依然是计数,记录总和,最后输出 平均值 #Mapper #!/usr/bin/python import sys from datetime import datetime for line in sys.stdin: data
mapreducer入门代码之单词计数
package com.wc.hadoop; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache
Mapreducer实现两表关联
address: 1    Beijing 2    Guangzhou         3    Shenzhen     4    Xian factory: Beijing Red Star    1 Shenzhen Thunder    3 Guangzhou Honda    2 Beijing Rising    1 Guangzhou Development Bank    ...
MapReducer去掉重复的数据
file1: 2012-3-1 a 2012-3-2 b 2012-3-3 c 2012-3-4 d 2012-3-5 a 2012-3-6 b 2012-3-7 c 2012-3-3 c file2: 2012-3-1 b 2012-3-2 a 2012-3-3 b 2012-3-4 d 2012-3-5 a 2012-3-6 c 2012-3-7 d 2012-3-3 c 结果: 2...
MapReducer工作流程小结
自己画了一个Mapreduce的工作流程,比较简易~~
MapReducer实现单词计算
package com.zjs.mr; import java.io.IOException; import java.util.StringTokenizer; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Tex
javascipt写入XML报错!!!
rnrnrn rn 在线预订rn rnrnrn rn rnrn rn rn rn Quick Reservation: 快速预定 rn rn Your E-mail 预定邮箱 rn rn Your Name 姓名: rn rn Items of the menu: rn 项目 Please select an item rn : - - - - - - - - - - - - - - - - rn rn Number of persons:人数 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 rn rn rn rn rn rnrn rnemail: rn name: rn item: rn year: rn month: rn day: rn time: rnlength: rnrnrn
HBase大批量写入报错~~
刚刚开始学习用JAVA写HBASE,小批量写入没问题。rn大批量写的时候就报错,HBASE的配置和代码都是以前同事写的,现在两眼一抹黑,不知道怎么弄。rnrn下面是错误日志rnrnjava.util.concurrent.ExecutionException: java.net.SocketTimeoutException: Call to 11.124.1.120:60020 failed on socket timeout exception: java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=11.124.1.120:56559 remote=11.124.1.120:60020]rn at java.util.concurrent.FutureTask$Sync.innerGet(FutureTask.java:222)rn at java.util.concurrent.FutureTask.get(FutureTask.java:83)rn at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.processBatchCallback(HConnectionManager.java:1540)rn at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.processBatch(HConnectionManager.java:1392)rn at org.apache.hadoop.hbase.client.HTable.flushCommits(HTable.java:918)rn at org.apache.hadoop.hbase.client.HTable.doPut(HTable.java:770)rn at org.apache.hadoop.hbase.client.HTable.put(HTable.java:757)rn at PS2HBASEIMP.ReadWriteHbase.addRecord(ReadWriteHbase.java:92)rn at PS2HBASEIMP.ReadWriteHbase.addPutList(ReadWriteHbase.java:75)rn at PS2HBASEIMP.TestImpHbase.main(TestImpHbase.java:139)rn at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)rn at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)rn at java.lang.reflect.Method.invoke(Method.java:597)rn at org.apache.hadoop.util.RunJar.main(RunJar.java:156)rnCaused by: java.net.SocketTimeoutException: Call to 11.124.1.120:60020 failed on socket timeout exception: java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=11.124.1.120:56559 remote=11.124.1.120:60020]rn at org.apache.hadoop.hbase.ipc.HBaseClient.wrapException(HBaseClient.java:1052)rn at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:1025)rn at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:150)rn at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$3$1.call(HConnectionManager.java:1369)rn at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$3$1.call(HConnectionManager.java:1367)rn at org.apache.hadoop.hbase.client.ServerCallable.withoutRetries(ServerCallable.java:210)rn at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$3.call(HConnectionManager.java:1376)rn at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$3.call(HConnectionManager.java:1364)rn at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)rn at java.util.concurrent.FutureTask.run(FutureTask.java:138)rn at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)rn at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)rn at java.lang.Thread.run(Thread.java:662)rn
C#写入报错
OleDbConnection zx = new OleDbConnection();rn zx.ConnectionString = "Provider=Microsoft.jet.oledb.4.0;Data Source=" + ("sicc.mdb");rn zx.Open();rn OleDbCommand zc = new OleDbCommand("select * from zc where username='"+nametextBox.Text+"'",zx);rn int zv = Convert.ToInt32(zc.ExecuteScalar());rn if (zv > 0)rn rn MessageBox.Show("注册失败此用户名存在", "提示");rn zx.Close();rn rn elsern rnrn OleDbConnection xc = new OleDbConnection();rn xc.ConnectionString = "Provider=Microsoft.jet.oledb.4.0;Data Source=" + ("sicc.mdb");rn xc.Open();rn OleDbCommand zb = new OleDbCommand("insert into zc values ('" + nametextBox.Text + "','" + pwdtextBox.Text + "','" + cspwdtextBox.Text + "','" + pwdtstextBox.Text + "','" + pwddatextBox.Text + "','" + mailtextBox.Text + "','" + naemmntextBox.Text + "','" + xmtextBox.Text + "','" + xbcomboBox.Text + "','" + sjhmtextBox.Text + "','" + dhtextBox.Text + "','" + zxqxcomboBox.Text + "')", xc);rn int xv = Convert.ToInt32(zb.ExecuteNonQuery());rn if (xv > 0)rn rn this.Hide();rn MessageBox.Show("注册成功!", "提示");rn xc.Close();rn rn elsern rn MessageBox.Show("注册失败!", "提示");rn rnrn报错原因:查询值的数目与目标字段中数目不相同rnrn其中:数据库的ID为自动编号,
文件目录写入报错
我用php建立一个文件夹。设置0777权限。用file_put_content写入文件报错:rnfailed to open stream: Permission denied'rnrn1、首先想到的就是权限问题,我用函数判断创建的文件夹是否有可写权限。我创建test.txt测试,可以成功。rn还有奇怪的是,程序往里面写入十几个文件都成功,但写到某个文件就报错了。如果没有可写权限,应该是一个文件都没有创建成功才是呀。rnrn2、路径是正确的,这点检查很多遍了。rnrn此外我想不到其他可能性了。麻烦高手指点一下,谢谢。
sqoop2写入hdfs报错
1.Caused by: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.SafeModeException): Cannot delete /output/.aa5fadaa-873e-4b6e-aa7c-42302c5f8452. Name node is in safe mode.关闭h
触发器写入txt文件报错
我在某个表写了一个触发器,然后通过C#往表里插入数据:rn触发器大致如下:rnrnALTER trigger [dbo].[testTrigger] on 表rnfor insert rnas rnexec master..xp_cmdshell 'bcp 表 out d:\test.txt /c /U sa /P 密码 'rnrn不知道为什么每次插入数据总是会报错。rnrn但是rnexec master..xp_cmdshell 'bcp 表 out d:\test.txt /c /U sa /P 密码 'rn这句话单独执行是可以的rnrnrnbcp 表 out d:\test.txt /c /U sa /P 密码 rn这句话在命令行里也可以执行rnrn非常想知道怎么才能让数据库不报错?急,求教高人啊~~~~~
php写入xml文件报错
我在写入xml文件时,如果有中文,保存xml文件时,提示报错,如下:rnrnWarning: DOMDocument::load() [domdocument.load]: Document is empty in file:///E:/Web/htdocs/xml/top.xml, line: 1 in E:\Web\htdocs\back\admin_news_top.php on line 6rnrnWarning: DOMDocument::load() [domdocument.load]: Start tag expected, '<' not found in file:///E:/Web/htdocs/xml/top.xml, line: 1 in E:\Web\htdocs\back\admin_news_top.php on line 6rnrnadmin_news_top.php 代码:rn[code=PHP]rnload('../xml/top.xml'); //第6行rn[/code]rnrn如果输入为中文,报错后直接清空xml文件里的所有内容rnrnrn这是我的XML文件内容:rn[code=XML]rnrnrnrnrn[/code]
写入数据库
我用jbuiter编一个小程序.想把一个文本的内容写入数据库中..rn要怎么写呢...rnrn我用这个rnString sql3="insert into diy (name,fen) values('"+jTextField1.getText()+"','"+jTextField2.getText()+"')";rnrnString pstmt = conn.prepareStatement(sql3);rnrnrs=stmt.executeUpdate(pstmt);rn但是不行啊..
写入数据库的
入数据库提示:rnMicrosoft OLE DB Provider for ODBC Drivers (0x80004005)rn[Microsoft][ODBC Microsoft Access Driver] INSERT INTO 语句的语法错误。rn/news1/save.asp, 第 17 行rnrnrn代码:rnrn<%rntypeid=request.form("typeid")rntitle=request.form("txttitle")rntxt=request.form("text")rntp=request.form("tp2")rnk=daternset rs=server.createobject("adodb.recordset") rnsql="select * from news"rnrs.open sql,conn,0,2rnrs.addnewrnrs("typeid")=typeidrnrs("title")=titlernrs("text")=txtrnrs("pic")=picrnrs("tp")=tprnrs.updatern%>rnrnrn
写入数据库!
我循环写进上百条记录,但是我想当表里已经有相同的记录时,就不写那一条,没有时就写进去?在SQL2000 server 里可以实现吗?
MapReducer 计算定位时间问题-面试题
/**数据 * userA,locationA,2018-01-01 08:00:00,60 userA,locationA,2018-01-01 09:00:00,60 userA,locationB,2018-01-01 10:00:00,60 userA,locationA,2018-01-01 11:00:00,60 * * 最终结果 user...
MapReducer扩展案例---TopN案例
TopN案例 1.需求 对需求2.3输出结果进行加工,输出流量使用量在前10的用户信息 (1)输入数据 13470253144 180 180 360 13509468723 7335 110349 117684 13560439638 918 4938 5856 13568436656 3597 25635 29232 13590439668 1116 954 2070 13630577991 ...
MapReducer中获取输入文件路径
正常情况: 在社区版的hadoop版本0.19/0.20中,当使用普通的输入的时候,比如: job.setInputFormatClass(TextInputFormat.class); 在mapper运行的时候,可以用如下的方法得到对应的filesplit,也就能拿到对应的&quot;输入路径&quot;等信息 FileSplit fileSplit = (FileSplit)(reporter.getInput...
Hadoop实战之MapReducer项目结构分析
一.MapReducer项目结构分析 1.前言 参考本例子前: 1.需要确保搭建好了hadoop集群环境。 2.安装了eclipse开发hadoop的环境。 3.这是关于MapReducer中的Mapper、Reduce、Job三个类的学习分析。 2.hadoop的MapReducer模型结构 (1)在eclipse中做hadoop开发: 在系统上打开eclipse,新建一个Map...
编写我的第一个MapReducer程序
环境 :    hadoop集群   1、编写Mapper类 package com.xue.mapreducer; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Mapper; import java.io....
hadoop之MapReducer作业的提交执行过程
debug信息 列出要切片的所有文件:[DeprecatedRawLocalFileStatus{path=file:/D:/a.txt; isDirectory=false; length=58; replication=1; blocksize=33554432; modification_time=1481531168562; access_time=0; owner=; group=;
hadoop本地调试模式,测试 mapreducer
这次使用的是  win下开发 MR代码,可以进行debug模式进行调试,当没有问题的时候,可以修改一下源文件名 ,然后上传到linux运行 基本数据 dept文件内容: 10,ACCOUNTING,NEW YORK 20,RESEARCH,DALLAS 30,SALES,CHICAGO 40,OPERATIONS,BOSTON   emp文件内容: 7369,SMITH,CL
jar包形式运行MapReducer代码
在我们平时运行MapRuducer代码中,往往更倾向于直接在eclipse中直接运行,而在很多业务生产过程中,需要我们以jar包的形式在集群中运行。所以,今天笔者就讲解一下如何打jar包来运行程序。 以一个小的案例来进行说明。 MapReducer实现WordCount 其中的代码笔者就不赘述了,在后面的学习博客中,笔者会详细讲代码分享出来,这里只是讲解如何打jar包: 1.选择对应的代码包 2...
MapReducer纠错之序列化反序列化问题
在MapReducer编程中经常会出现序列化反序列化的说法 在这里解释一下 序列化:序列化的意思就是把内存中的对象变成字节存储到磁盘上 反序列化:反序列化的意思就是把磁盘上的对象活化到内存中 说到这里也许还是没有说清楚序列化的意思,在大数据中,假设有一个非常大的集群,也就是很多计算机,这一群计算机叫做集群,MapReducer程序就是一个计算数据的程序,当A这台机器计算完毕之后,A要把工作...
相关热词 c#串口测试应用程序 c# 匹配 正则表达式 c#防止窗体重绘 c#读写txt文件 c#挖地雷源代码 c#弹框选项 c# 移除 队列 c# 密码写入配置文件 c# 获取可用内存大小 c# 嵌入excel编辑