mapreduce编程出现错误:Job running in uber mode : false?

eclipse编写mapreduce程序,实现统计多个目录下所有文件的文件行数总数。

代码如下:
package ZhangBo;

import java.io.IOException;
//import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
//import org.apache.hadoop.mapreduce.lib.input.MultipleInputs;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
//import org.apache.hadoop.io.LongWritable;
//import org.apache.hadoop.util.GenericOptionsParser;
//import org.junit.Test;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
//import org.apache.log4j.MDC;
//import org.apache.log4j.Logger;

//import firstnight.testlog4jj;

public class WordCount {
//////log
static Log logger = LogFactory.getLog(WordCount.class);

public static class TokenizerMapper 
   extends Mapper<LongWritable, Text, Text, IntWritable>{
               //一行的起始位置

private final static IntWritable one = new IntWritable(1);
private Text word = new Text();
public void map(LongWritable key, Text value, Context context
                ) throws IOException, InterruptedException {
  //进行分割
    logger.debug("This is debug message.");
    logger.error("This is error message."); 
    logger.info("ni hao ");
    context.write(word,one);

}
}
//对输入的key value值进行处理 转换成新的 key value值 进行一系列框架自带的处理 排序 合并 分句等等
public static class IntSumReducer
extends Reducer {
private IntWritable result = new IntWritable();
public void reduce(Text key, Iterable values,
Context context
) throws IOException, InterruptedException {
int sum = 0;
//循环遍历 IntWritable
for (IntWritable val : values) {
sum += val.get();//get 转 int
}
result.set(sum);
context.write(key, result);
}

  }

private static FileSystem hdfs;
private static String args1;
private static String args2;
private static String args3;
Configuration conf1 = new Configuration();

private static void showDir(FileStatus fs) throws Exception
{
Path path = fs.getPath();

  if (fs.isDir())
  {
      args2 = path.toString();
      if (args2 != args3)
          args1 = args1 +","+args2;
      System.out.println("目录:" + path);
      FileStatus[] f = hdfs.listStatus(path);
      if (f.length > 0)
      {
          for (FileStatus file:f)
          {
              showDir(file);
              args3 = file.getPath().toString();
          }
      }
  }
  else
  {
      System.out.println("文件:" + path); 
  }
  System.out.println("目录之和: "+args1);

}

public void test() throws Exception
{
hdfs = FileSystem.get(conf1);
FileStatus[] fs = hdfs.listStatus(new Path("hdfs://C15/zbo"));
args1 = "hdfs://C15/zbo";
if (fs.length > 0)
{
for (FileStatus f : fs)
{
showDir(f);
}
}

else
{
System.out.println("没有什么好遍历的....");
}
}

public static void main(String[] args) throws Exception
{
new WordCount().test();
Configuration conf = new Configuration();
//////////////////////

     Job job = new Job(conf,"word count");
     job.setJarByClass(WordCount.class);
     job.setMapperClass(TokenizerMapper.class);
     job.setCombinerClass(IntSumReducer.class);
     job.setReducerClass(IntSumReducer.class);
     FileInputFormat.addInputPaths(job,args1);
     FileOutputFormat.setOutputPath(job, new Path("outbo"));
     job.setOutputKeyClass(Text.class);
     job.setOutputValueClass(IntWritable.class);
     System.exit(job.waitForCompletion(true)?0:1);
  }

}

在目录hdfs://C15/zbo下创建目录和文件
在eclipse上打jar包ho.jar
执行程序 :hadoop jar ZhangBo.WordCount
执行结果如下所示:

[root@c15m1 /]# hadoop jar ho.jar ZhangBo.WordCount
文件:hdfs://C15/zbo/file1.txt
目录之和: hdfs://C15/zbo
文件:hdfs://C15/zbo/file2.txt
目录之和: hdfs://C15/zbo
目录:hdfs://C15/zbo/zbo
文件:hdfs://C15/zbo/zbo/file1.txt
目录之和: hdfs://C15/zbo,hdfs://C15/zbo/zbo
文件:hdfs://C15/zbo/zbo/file2.txt
目录之和: hdfs://C15/zbo,hdfs://C15/zbo/zbo
目录:hdfs://C15/zbo/zbo/zbo
文件:hdfs://C15/zbo/zbo/zbo/file1.txt
目录之和: hdfs://C15/zbo,hdfs://C15/zbo/zbo,hdfs://C15/zbo/zbo/zbo
文件:hdfs://C15/zbo/zbo/zbo/file2.txt
目录之和: hdfs://C15/zbo,hdfs://C15/zbo/zbo,hdfs://C15/zbo/zbo/zbo
目录之和: hdfs://C15/zbo,hdfs://C15/zbo/zbo,hdfs://C15/zbo/zbo/zbo
目录之和: hdfs://C15/zbo,hdfs://C15/zbo/zbo,hdfs://C15/zbo/zbo/zbo
15/11/17 08:55:37 INFO client.RMProxy: Connecting to ResourceManager at c15m1.ecld.com/132.121.94.213:8050
15/11/17 08:55:38 INFO hdfs.DFSClient: Created HDFS_DELEGATION_TOKEN token 72820725 for hdfs on ha-hdfs:C15
15/11/17 08:55:38 INFO security.TokenCache: Got dt for hdfs://C15; Kind: HDFS_DELEGATION_TOKEN, Service: ha-hdfs:C15, Ident: (HDFS_DELEGATION_TOKEN token 72820725 for hdfs)
15/11/17 08:55:38 WARN mapreduce.JobSubmitter: Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this.
15/11/17 08:55:38 INFO input.FileInputFormat: Total input paths to process : 8
15/11/17 08:55:38 INFO lzo.GPLNativeCodeLoader: Loaded native gpl library
15/11/17 08:55:38 INFO lzo.LzoCodec: Successfully loaded & initialized native-lzo library [hadoop-lzo rev dbd51f0fb61f5347228a7a23fe0765ac1242fcdf]
15/11/17 08:55:47 INFO mapreduce.JobSubmitter: number of splits:8
15/11/17 08:55:48 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1440641224751_187144
15/11/17 08:55:48 INFO mapreduce.JobSubmitter: Kind: HDFS_DELEGATION_TOKEN, Service: ha-hdfs:C15, Ident: (HDFS_DELEGATION_TOKEN token 72820725 for hdfs)
15/11/17 08:55:48 INFO impl.YarnClientImpl: Submitted application application_1440641224751_187144
15/11/17 08:55:49 INFO mapreduce.Job: The url to track the job: http://c15m1.ecld.com:8088/proxy/application_1440641224751_187144/
15/11/17 08:55:49 INFO mapreduce.Job: Running job: job_1440641224751_187144
15/11/17 08:55:51 INFO mapreduce.Job: Job job_1440641224751_187144 running in uber mode : false
15/11/17 08:55:51 INFO mapreduce.Job: map 0% reduce 0%
15/11/17 08:55:51 INFO mapreduce.Job: Job job_1440641224751_187144 failed with state FAILED due to: Application application_1440641224751_187144 failed 2 times due to AM Container for appattempt_1440641224751_187144_000002 exited with exitCode: -1000 due to: Application application_1440641224751_187144 initialization failed (exitCode=255) with output: Requested user hdfs is banned

.Failing this attempt.. Failing the application.
15/11/17 08:55:51 INFO mapreduce.Job: Counters: 0

问题:running in uber mode : false是什么问题?

1个回答

你好,我也遇到了这个问题请问你是怎么解决的?谢谢!

Csdn user default icon
上传中...
上传图片
插入图片
抄袭、复制答案,以达到刷声望分或其他目的的行为,在CSDN问答是严格禁止的,一经发现立刻封号。是时候展现真正的技术了!
其他相关推荐
cdh hadoop mapreduce 运行时的问题:(有时候会出现,有时候不出现,急求大神帮助)
15/10/08 08:49:13 INFO mapreduce.Job: Job job_1419225162729_18465 running in uber mode : false 15/10/08 08:49:13 INFO mapreduce.Job: map 0% reduce 0% 15/10/08 08:49:13 INFO mapreduce.Job: Job job_1419225162729_18465 failed with state FAILED due to: Application application_1419225162729_18465 failed 1 times due to AM Container for appattempt_1419225162729_18465_000001 exited with exitCode: -1000 due to: java.io.IOException: Not able to initialize app-log directories in any of the configured local directories for app application_1419225162729_18465 at org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.createAppLogDirs(DefaultContainerExecutor.java:459) at org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.startLocalizer(DefaultContainerExecutor.java:91) at org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService$LocalizerRunner.run(ResourceLocalizationService.java:861) .Failing this attempt.. Failing the application. 15/10/08 08:49:13 INFO mapreduce.Job: Counters: 0 Moved: 'hdfs://oiddhnode02:8020/user/nmger/worktemp/2015100408' to trash at: hdfs://oiddhnode02:8020/user/nmger/.Trash/Current
使用sqoop2把hive表导出道pgsql时报错,数据只导出了一部分,如何解决?
18/10/11 16:30:39 INFO mapreduce.Job: Running job: job_1539239753824_0018 18/10/11 16:30:45 INFO mapreduce.Job: Job job_1539239753824_0018 running in uber mode : false 18/10/11 16:30:45 INFO mapreduce.Job: map 0% reduce 0% 18/10/11 16:30:56 INFO mapreduce.Job: map 19% reduce 0% 18/10/11 16:30:59 INFO mapreduce.Job: map 27% reduce 0% 18/10/11 16:31:01 INFO mapreduce.Job: map 100% reduce 0% 18/10/11 16:31:02 INFO mapreduce.Job: Job job_1539239753824_0018 failed with state FAILED due to: Task failed task_1539239753824_0018_m_000000 Job failed as tasks failed. failedMaps:1 failedReduces:0 18/10/11 16:31:02 INFO mapreduce.Job: Counters: 8 Job Counters Failed map tasks=1 Launched map tasks=1 Rack-local map tasks=1 Total time spent by all maps in occupied slots (ms)=13204 Total time spent by all reduces in occupied slots (ms)=0 Total time spent by all map tasks (ms)=13204 Total vcore-milliseconds taken by all map tasks=13204 Total megabyte-milliseconds taken by all map tasks=13520896 18/10/11 16:31:02 WARN mapreduce.Counters: Group FileSystemCounters is deprecated. Use org.apache.hadoop.mapreduce.FileSystemCounter instead 18/10/11 16:31:02 INFO mapreduce.ExportJobBase: Transferred 0 bytes in 24.5676 seconds (0 bytes/sec) 18/10/11 16:31:02 WARN mapreduce.Counters: Group org.apache.hadoop.mapred.Task$Counter is deprecated. Use org.apache.hadoop.mapreduce.TaskCounter instead 18/10/11 16:31:02 INFO mapreduce.ExportJobBase: Exported 0 records. 18/10/11 16:31:02 ERROR mapreduce.ExportJobBase: Export job failed! 18/10/11 16:31:02 ERROR tool.ExportTool: Error during export: Export job failed! at org.apache.sqoop.mapreduce.ExportJobBase.runExport(ExportJobBase.java:445) at org.apache.sqoop.manager.SqlManager.exportTable(SqlManager.java:931) at org.apache.sqoop.tool.ExportTool.exportTable(ExportTool.java:80) at org.apache.sqoop.tool.ExportTool.run(ExportTool.java:99) at org.apache.sqoop.Sqoop.run(Sqoop.java:147) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70) at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:183) at org.apache.sqoop.Sqoop.runTool(Sqoop.java:234) at org.apache.sqoop.Sqoop.runTool(Sqoop.java:243) at org.apache.sqoop.Sqoop.main(Sqoop.java:252)
hadoop中wordcount使用IKsegmenter分词器后本地编译运行通过,打成jar包后报IKsegmenter类无法找到
使用eclipse在本地上编译运行的时候没有问题,output能够获得正常结果。 但是使用hadoop运行时就会报错,之前没有使用过ecipse,在网上找了半天,export里面的选项来回试了一下也没成功..以下是报错信息: ``` 19/11/19 20:57:39 INFO client.RMProxy: Connecting to ResourceManager at bigdata-senior01.chybinmy.com/192.168.100.10:8032 19/11/19 20:57:40 INFO input.FileInputFormat: Total input files to process : 1 19/11/19 20:57:40 INFO mapreduce.JobSubmitter: number of splits:1 19/11/19 20:57:41 INFO Configuration.deprecation: yarn.resourcemanager.system-metrics-publisher.enabled is deprecated. Instead, use yarn.system-metrics-publisher.enabled 19/11/19 20:57:41 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1574165004078_0005 19/11/19 20:57:41 INFO conf.Configuration: resource-types.xml not found 19/11/19 20:57:41 INFO resource.ResourceUtils: Unable to find 'resource-types.xml'. 19/11/19 20:57:41 INFO resource.ResourceUtils: Adding resource type - name = memory-mb, units = Mi, type = COUNTABLE 19/11/19 20:57:41 INFO resource.ResourceUtils: Adding resource type - name = vcores, units = , type = COUNTABLE 19/11/19 20:57:42 INFO impl.YarnClientImpl: Submitted application application_1574165004078_0005 19/11/19 20:57:42 INFO mapreduce.Job: The url to track the job: http://bigdata-senior01.chybinmy.com:8088/proxy/application_1574165004078_0005/ 19/11/19 20:57:42 INFO mapreduce.Job: Running job: job_1574165004078_0005 19/11/19 20:57:49 INFO mapreduce.Job: Job job_1574165004078_0005 running in uber mode : false 19/11/19 20:57:49 INFO mapreduce.Job: map 0% reduce 0% 19/11/19 20:57:53 INFO mapreduce.Job: Task Id : attempt_1574165004078_0005_m_000000_0, Status : FAILED Error: java.lang.ClassNotFoundException: org.wltea.analyzer.core.IKSegmenter at java.net.URLClassLoader$1.run(URLClassLoader.java:366) at java.net.URLClassLoader$1.run(URLClassLoader.java:355) at java.security.AccessController.doPrivileged(Native Method) at java.net.URLClassLoader.findClass(URLClassLoader.java:354) at java.lang.ClassLoader.loadClass(ClassLoader.java:425) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308) at java.lang.ClassLoader.loadClass(ClassLoader.java:358) at com.example.test.ChineseWordCount$TokenizerMapper.map(ChineseWordCount.java:36) at com.example.test.ChineseWordCount$TokenizerMapper.map(ChineseWordCount.java:1) at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:146) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:793) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:177) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1893) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:171) 19/11/19 20:57:58 INFO mapreduce.Job: Task Id : attempt_1574165004078_0005_m_000000_1, Status : FAILED Error: java.lang.ClassNotFoundException: org.wltea.analyzer.core.IKSegmenter at java.net.URLClassLoader$1.run(URLClassLoader.java:366) at java.net.URLClassLoader$1.run(URLClassLoader.java:355) at java.security.AccessController.doPrivileged(Native Method) at java.net.URLClassLoader.findClass(URLClassLoader.java:354) at java.lang.ClassLoader.loadClass(ClassLoader.java:425) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308) at java.lang.ClassLoader.loadClass(ClassLoader.java:358) at com.example.test.ChineseWordCount$TokenizerMapper.map(ChineseWordCount.java:36) at com.example.test.ChineseWordCount$TokenizerMapper.map(ChineseWordCount.java:1) at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:146) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:793) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:177) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1893) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:171) 19/11/19 20:58:03 INFO mapreduce.Job: Task Id : attempt_1574165004078_0005_m_000000_2, Status : FAILED Error: java.lang.ClassNotFoundException: org.wltea.analyzer.core.IKSegmenter at java.net.URLClassLoader$1.run(URLClassLoader.java:366) at java.net.URLClassLoader$1.run(URLClassLoader.java:355) at java.security.AccessController.doPrivileged(Native Method) at java.net.URLClassLoader.findClass(URLClassLoader.java:354) at java.lang.ClassLoader.loadClass(ClassLoader.java:425) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308) at java.lang.ClassLoader.loadClass(ClassLoader.java:358) at com.example.test.ChineseWordCount$TokenizerMapper.map(ChineseWordCount.java:36) at com.example.test.ChineseWordCount$TokenizerMapper.map(ChineseWordCount.java:1) at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:146) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:793) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:177) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1893) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:171) 19/11/19 20:58:11 INFO mapreduce.Job: map 100% reduce 100% 19/11/19 20:58:12 INFO mapreduce.Job: Job job_1574165004078_0005 failed with state FAILED due to: Task failed task_1574165004078_0005_m_000000 Job failed as tasks failed. failedMaps:1 failedReduces:0 19/11/19 20:58:12 INFO mapreduce.Job: Counters: 13 Job Counters Failed map tasks=4 Killed reduce tasks=1 Launched map tasks=4 Other local map tasks=3 Data-local map tasks=1 Total time spent by all maps in occupied slots (ms)=12897 Total time spent by all reduces in occupied slots (ms)=0 Total time spent by all map tasks (ms)=12897 Total vcore-milliseconds taken by all map tasks=12897 Total megabyte-milliseconds taken by all map tasks=13206528 Map-Reduce Framework CPU time spent (ms)=0 Physical memory (bytes) snapshot=0 Virtual memory (bytes) snapshot=0 [hadoop@bigdata-senior01 hadoop-2.10.0]$ bin/hdfs dfs -ls / Found 4 items drwxr-xr-x - hadoop supergroup 0 2019-11-19 20:13 /input drwxr-xr-x - hadoop supergroup 0 2019-11-19 20:54 /output drwxr-xr-x - hadoop supergroup 0 2019-11-19 20:58 /output2 drwx------ - hadoop supergroup 0 2019-11-19 20:16 /tmp ```
mapreduce,貌似是overide,求助
报错: zxy@zxy-virtual-machine:/usr/hadoop/hadoop-2.4.0$ hadoop jar WordCount.jar WordCount /input /output 15/04/23 07:12:49 INFO Configuration.deprecation: session.id is deprecated. Instead, use dfs.metrics.session-id 15/04/23 07:12:49 INFO jvm.JvmMetrics: Initializing JVM Metrics with processName=JobTracker, sessionId= 15/04/23 07:12:50 INFO input.FileInputFormat: Total input paths to process : 1 15/04/23 07:12:50 INFO mapreduce.JobSubmitter: number of splits:1 15/04/23 07:12:51 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_local168934583_0001 15/04/23 07:12:51 WARN conf.Configuration: file:/home/zxy/hadoop_tmp/mapred/staging/zxy168934583/.staging/job_local168934583_0001/job.xml:an attempt to override final parameter: mapreduce.job.end-notification.max.retry.interval; Ignoring. 15/04/23 07:12:51 WARN conf.Configuration: file:/home/zxy/hadoop_tmp/mapred/staging/zxy168934583/.staging/job_local168934583_0001/job.xml:an attempt to override final parameter: mapreduce.job.end-notification.max.attempts; Ignoring. 15/04/23 07:12:52 WARN conf.Configuration: file:/home/zxy/hadoop_tmp/mapred/local/localRunner/zxy/job_local168934583_0001/job_local168934583_0001.xml:an attempt to override final parameter: mapreduce.job.end-notification.max.retry.interval; Ignoring. 15/04/23 07:12:52 WARN conf.Configuration: file:/home/zxy/hadoop_tmp/mapred/local/localRunner/zxy/job_local168934583_0001/job_local168934583_0001.xml:an attempt to override final parameter: mapreduce.job.end-notification.max.attempts; Ignoring. 15/04/23 07:12:52 INFO mapreduce.Job: The url to track the job: http://localhost:8080/ 15/04/23 07:12:52 INFO mapreduce.Job: Running job: job_local168934583_0001 15/04/23 07:12:52 INFO mapred.LocalJobRunner: OutputCommitter set in config null 15/04/23 07:12:52 INFO mapred.LocalJobRunner: OutputCommitter is org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter 15/04/23 07:12:52 INFO mapred.LocalJobRunner: Waiting for map tasks 15/04/23 07:12:52 INFO mapred.LocalJobRunner: Starting task: attempt_local168934583_0001_m_000000_0 15/04/23 07:12:52 INFO mapred.Task: Using ResourceCalculatorProcessTree : [ ] 15/04/23 07:12:52 INFO mapred.MapTask: Processing split: hdfs://localhost:9000/input/data.txt:0+57 15/04/23 07:12:52 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer 15/04/23 07:12:53 INFO mapreduce.Job: Job job_local168934583_0001 running in uber mode : false 15/04/23 07:12:53 INFO mapreduce.Job: map 0% reduce 0% 15/04/23 07:12:55 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584) 15/04/23 07:12:55 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 100 15/04/23 07:12:55 INFO mapred.MapTask: soft limit at 83886080 15/04/23 07:12:55 INFO mapred.MapTask: bufstart = 0; bufvoid = 104857600 15/04/23 07:12:55 INFO mapred.MapTask: kvstart = 26214396; length = 6553600 15/04/23 07:12:55 INFO mapred.MapTask: Starting flush of map output 15/04/23 07:12:55 INFO mapred.MapTask: Spilling map output 15/04/23 07:12:55 INFO mapred.MapTask: bufstart = 0; bufend = 36; bufvoid = 104857600 15/04/23 07:12:55 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 26214376(104857504); length = 21/6553600 15/04/23 07:12:55 INFO mapred.MapTask: Finished spill 0 15/04/23 07:12:55 INFO mapred.LocalJobRunner: map task executor complete. 15/04/23 07:12:55 WARN mapred.LocalJobRunner: job_local168934583_0001 java.lang.Exception: java.lang.ArrayIndexOutOfBoundsException: 3 at org.apache.hadoop.mapred.LocalJobRunner$Job.runTasks(LocalJobRunner.java:462) at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:522) Caused by: java.lang.ArrayIndexOutOfBoundsException: 3 at WordCount$TokenizerMapper.map(WordCount.java:35) at WordCount$TokenizerMapper.map(WordCount.java:1) at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:145) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:764) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:340) at org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run(LocalJobRunner.java:243) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 15/04/23 07:12:55 INFO mapreduce.Job: Job job_local168934583_0001 failed with state FAILED due to: NA 15/04/23 07:12:55 INFO mapreduce.Job: Counters: 0 zxy@zxy-virtual-machine:/usr/hadoop/hadoop-2.4.0$ hadoop fs -ls /output
Hadoop2.7.3本地调试reduce失败
2017-04-28 17:13:23,238 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1360)) - Job job_local1046505697_0023 running in uber mode : false 2017-04-28 17:13:23,238 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1367)) - map 100% reduce 0% 2017-04-28 17:13:23,239 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1380)) - Job job_local1046505697_0023 failed with state FAILED due to: NA 2017-04-28 17:13:23,242 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1385)) - Counters: 22 之前运行成功了,今天再次运行报了这个错误,求指导。
通过sqoop 导入hive错误
[root@orep apps]# sqoop import --connect jdbc:mysql://192.168.3.6:3306/hive --username root --password mysql123 --table VERSION --hive-table VERSION --hive-import 17/03/09 05:45:34 INFO sqoop.Sqoop: Running Sqoop version: 1.4.6-cdh5.10.0 17/03/09 05:45:34 WARN tool.BaseSqoopTool: Setting your password on the command-line is insecure. Consider using -P instead. 17/03/09 05:45:34 INFO tool.BaseSqoopTool: Using Hive-specific delimiters for output. You can override 17/03/09 05:45:34 INFO tool.BaseSqoopTool: delimiters with --fields-terminated-by, etc. 17/03/09 05:45:34 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset. 17/03/09 05:45:34 INFO tool.CodeGenTool: Beginning code generation 17/03/09 05:45:34 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `VERSION` AS t LIMIT 1 17/03/09 05:45:34 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `VERSION` AS t LIMIT 1 17/03/09 05:45:34 INFO orm.CompilationManager: HADOOP_MAPRED_HOME is /apps/hadoop-2.7.2 Note: /tmp/sqoop-root/compile/4b54b295fba470d9743716efe53e0d48/VERSION.java uses or overrides a deprecated API. Note: Recompile with -Xlint:deprecation for details. 17/03/09 05:45:36 INFO orm.CompilationManager: Writing jar file: /tmp/sqoop-root/compile/4b54b295fba470d9743716efe53e0d48/VERSION.jar 17/03/09 05:45:36 WARN manager.MySQLManager: It looks like you are importing from mysql. 17/03/09 05:45:36 WARN manager.MySQLManager: This transfer can be faster! Use the --direct 17/03/09 05:45:36 WARN manager.MySQLManager: option to exercise a MySQL-specific fast path. 17/03/09 05:45:36 INFO manager.MySQLManager: Setting zero DATETIME behavior to convertToNull (mysql) 17/03/09 05:45:36 INFO mapreduce.ImportJobBase: Beginning import of VERSION SLF4J: Class path contains multiple SLF4J bindings. SLF4J: Found binding in [jar:file:/apps/hadoop-2.7.2/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: Found binding in [jar:file:/apps/hbase/lib/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation. SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory] 17/03/09 05:45:36 INFO Configuration.deprecation: mapred.jar is deprecated. Instead, use mapreduce.job.jar 17/03/09 05:45:37 INFO Configuration.deprecation: mapred.map.tasks is deprecated. Instead, use mapreduce.job.maps 17/03/09 05:45:37 INFO client.RMProxy: Connecting to ResourceManager at /192.168.3.6:8032 17/03/09 05:45:41 INFO db.DBInputFormat: Using read commited transaction isolation 17/03/09 05:45:41 INFO db.DataDrivenDBInputFormat: BoundingValsQuery: SELECT MIN(`VER_ID`), MAX(`VER_ID`) FROM `VERSION` 17/03/09 05:45:41 INFO db.IntegerSplitter: Split size: 0; Num splits: 4 from: 1 to: 1 17/03/09 05:45:41 INFO mapreduce.JobSubmitter: number of splits:1 17/03/09 05:45:41 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1488979391372_0005 17/03/09 05:45:42 INFO impl.YarnClientImpl: Submitted application application_1488979391372_0005 17/03/09 05:45:42 INFO mapreduce.Job: The url to track the job: http://Kylin01:8088/proxy/application_1488979391372_0005/ 17/03/09 05:45:42 INFO mapreduce.Job: Running job: job_1488979391372_0005 17/03/09 05:45:50 INFO mapred.ClientServiceDelegate: Application state is completed. FinalApplicationStatus=FAILED. Redirecting to job history server 17/03/09 05:45:50 INFO mapreduce.Job: Job job_1488979391372_0005 running in uber mode : false 17/03/09 05:45:50 INFO mapreduce.Job: map 0% reduce 100% 17/03/09 05:45:50 INFO mapreduce.Job: Job job_1488979391372_0005 failed with state FAILED due to: 17/03/09 05:45:50 INFO mapreduce.ImportJobBase: The MapReduce job has already been retired. Performance 17/03/09 05:45:50 INFO mapreduce.ImportJobBase: counters are unavailable. To get this information, 17/03/09 05:45:50 INFO mapreduce.ImportJobBase: you will need to enable the completed job store on 17/03/09 05:45:50 INFO mapreduce.ImportJobBase: the jobtracker with: 17/03/09 05:45:50 INFO mapreduce.ImportJobBase: mapreduce.jobtracker.persist.jobstatus.active = true 17/03/09 05:45:50 INFO mapreduce.ImportJobBase: mapreduce.jobtracker.persist.jobstatus.hours = 1 17/03/09 05:45:50 INFO mapreduce.ImportJobBase: A jobtracker restart is required for these settings 17/03/09 05:45:50 INFO mapreduce.ImportJobBase: to take effect. 17/03/09 05:45:50 ERROR tool.ImportTool: Error during import: Import job failed! 烦请给我大神帮我诊断一下,谢谢
eclipse运行hadoop mapreduce程序如下错误
2017-09-06 15:48:42,677 INFO [LocalJobRunner Map Task Executor #0] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(591)) - 2017-09-06 15:48:42,686 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:flush(1460)) - Starting flush of map output 2017-09-06 15:48:42,686 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:flush(1482)) - Spilling map output 2017-09-06 15:48:42,686 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:flush(1483)) - bufstart = 0; bufend = 108; bufvoid = 104857600 2017-09-06 15:48:42,686 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:flush(1485)) - kvstart = 26214396(104857584); kvend = 26214352(104857408); length = 45/6553600 2017-09-06 15:48:42,733 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:sortAndSpill(1667)) - Finished spill 0 2017-09-06 15:48:42,743 INFO [LocalJobRunner Map Task Executor #0] mapred.Task (Task.java:done(1038)) - Task:attempt_local1469942249_0001_m_000000_0 is done. And is in the process of committing 2017-09-06 15:48:42,751 INFO [Thread-19] mapred.LocalJobRunner (LocalJobRunner.java:runTasks(456)) - map task executor complete. 2017-09-06 15:48:42,783 WARN [Thread-19] mapred.LocalJobRunner (LocalJobRunner.java:run(560)) - job_local1469942249_0001 java.lang.Exception: java.lang.NoSuchMethodError: org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree.getRssMemorySize()J at org.apache.hadoop.mapred.LocalJobRunner$Job.runTasks(LocalJobRunner.java:462) at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:522) Caused by: java.lang.NoSuchMethodError: org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree.getRssMemorySize()J at org.apache.hadoop.mapred.Task.updateResourceCounters(Task.java:872) at org.apache.hadoop.mapred.Task.updateCounters(Task.java:1021) at org.apache.hadoop.mapred.Task.done(Task.java:1040) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:345) at org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run(LocalJobRunner.java:243) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 2017-09-06 15:48:43,333 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1360)) - Job job_local1469942249_0001 running in uber mode : false 2017-09-06 15:48:43,335 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1367)) - map 0% reduce 0% 2017-09-06 15:48:43,337 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1380)) - Job job_local1469942249_0001 failed with state FAILED due to: NA 2017-09-06 15:48:43,352 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1385)) - Counters: 10 Map-Reduce Framework Map input records=12 Map output records=12 Map output bytes=108 Map output materialized bytes=0 Input split bytes=104 Combine input records=0 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 File Input Format Counters Bytes Read=132 Finished
利用sqoop把数据从Oracle导出到hive报错
![图片说明](https://img-ask.csdn.net/upload/201504/16/1429180711_592161.png) bash-4.1$ sqoop import --connect jdbc:oracle:thin:@192.168.1.169:1521:orcl --username HADOOP --password hadoop2015 --table CALC_UPAY_DATE_HADOOP_HDFS --split-by UPAYID --hive-import Warning: /usr/lib/sqoop/../accumulo does not exist! Accumulo imports will fail. Please set $ACCUMULO_HOME to the root of your Accumulo installation. find: paths must precede expression: ant-eclipse-1.0-jvm1.2.jar Usage: find [-H] [-L] [-P] [-Olevel] [-D help|tree|search|stat|rates|opt|exec] [path...] [expression] 15/04/16 03:28:13 INFO sqoop.Sqoop: Running Sqoop version: 1.4.4-cdh5.0.2 15/04/16 03:28:13 WARN tool.BaseSqoopTool: Setting your password on the command-line is insecure. Consider using -P instead. 15/04/16 03:28:13 INFO tool.BaseSqoopTool: Using Hive-specific delimiters for output. You can override 15/04/16 03:28:13 INFO tool.BaseSqoopTool: delimiters with --fields-terminated-by, etc. 15/04/16 03:28:13 INFO manager.SqlManager: Using default fetchSize of 1000 15/04/16 03:28:13 INFO tool.CodeGenTool: Beginning code generation 15/04/16 03:28:13 INFO manager.OracleManager: Time zone has been set to GMT 15/04/16 03:28:13 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM CALC_UPAY_DATE_HADOOP_HDFS t WHERE 1=0 15/04/16 03:28:14 INFO orm.CompilationManager: HADOOP_MAPRED_HOME is /usr/lib/hadoop-mapreduce Note: /tmp/sqoop-hdfs/compile/e9286bf0e7d796ba396d3155210012b0/CALC_UPAY_DATE_HADOOP_HDFS.java uses or overrides a deprecated API. Note: Recompile with -Xlint:deprecation for details. 15/04/16 03:28:15 INFO orm.CompilationManager: Writing jar file: /tmp/sqoop-hdfs/compile/e9286bf0e7d796ba396d3155210012b0/CALC_UPAY_DATE_HADOOP_HDFS.jar 15/04/16 03:28:15 INFO mapreduce.ImportJobBase: Beginning import of CALC_UPAY_DATE_HADOOP_HDFS 15/04/16 03:28:15 INFO Configuration.deprecation: mapred.jar is deprecated. Instead, use mapreduce.job.jar 15/04/16 03:28:15 INFO manager.OracleManager: Time zone has been set to GMT 15/04/16 03:28:16 INFO Configuration.deprecation: mapred.map.tasks is deprecated. Instead, use mapreduce.job.maps 15/04/16 03:28:16 INFO client.RMProxy: Connecting to ResourceManager at hadoop01/192.168.1.201:8032 15/04/16 03:28:18 INFO db.DBInputFormat: Using read commited transaction isolation 15/04/16 03:28:18 INFO db.DataDrivenDBInputFormat: BoundingValsQuery: SELECT MIN(UPAYID), MAX(UPAYID) FROM CALC_UPAY_DATE_HADOOP_HDFS 15/04/16 03:28:19 INFO mapreduce.JobSubmitter: number of splits:4 15/04/16 03:28:20 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1429145594985_0020 15/04/16 03:28:20 INFO impl.YarnClientImpl: Submitted application application_1429145594985_0020 15/04/16 03:28:20 INFO mapreduce.Job: The url to track the job: http://hadoop01:8088/proxy/application_1429145594985_0020/ 15/04/16 03:28:20 INFO mapreduce.Job: Running job: job_1429145594985_0020 15/04/16 03:28:31 INFO mapreduce.Job: Job job_1429145594985_0020 running in uber mode : false 15/04/16 03:28:31 INFO mapreduce.Job: map 0% reduce 0% 15/04/16 03:28:59 INFO mapreduce.Job: Task Id : attempt_1429145594985_0020_m_000000_0, Status : FAILED Error: oracle.jdbc.driver.T4CPreparedStatement.isClosed()Z 15/04/16 03:29:00 INFO mapreduce.Job: Task Id : attempt_1429145594985_0020_m_000002_0, Status : FAILED Error: oracle.jdbc.driver.T4CPreparedStatement.isClosed()Z 15/04/16 03:29:01 INFO mapreduce.Job: Task Id : attempt_1429145594985_0020_m_000001_0, Status : FAILED Error: oracle.jdbc.driver.T4CPreparedStatement.isClosed()Z 我用sqoop把数据从hive导出到oracle一切正常
hadoop wordcount报错192.168.79.172 to  :54895 拒绝连接
各位大神,我是hadoop新手,在自己电脑安装好hadoop后,运行wordcount报错,报错已困扰两天,希望有大神可以帮解决。 报错日志如下: [hadoop@hadoop0 mapreduce]$ hadoop jar hadoop-mapreduce-examples-2.7.1.jar wordcount /input /output 17/04/14 13:49:46 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 17/04/14 13:49:48 INFO client.RMProxy: Connecting to ResourceManager at hadoop0/192.168.79.172:8032 17/04/14 13:49:50 INFO input.FileInputFormat: Total input paths to process : 2 17/04/14 13:49:50 INFO mapreduce.JobSubmitter: number of splits:2 17/04/14 13:49:50 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1492077890345_0003 17/04/14 13:49:51 INFO impl.YarnClientImpl: Submitted application application_1492077890345_0003 17/04/14 13:49:51 INFO mapreduce.Job: The url to track the job: http://hadoop0:8088/proxy/application_1492077890345_0003/ 17/04/14 13:49:51 INFO mapreduce.Job: Running job: job_1492077890345_0003 17/04/14 13:50:12 INFO mapreduce.Job: Job job_1492077890345_0003 running in uber mode : false 17/04/14 13:50:12 INFO mapreduce.Job: map 0% reduce 0% 17/04/14 13:50:12 INFO mapreduce.Job: Job job_1492077890345_0003 failed with state FAILED due to: Application application_1492077890345_0003 failed 2 times due to Error launching appattempt_1492077890345_0003_000002. Got exception: java.net.ConnectException: Call From hadoop0/192.168.79.172 to  :54895 failed on connection exception: java.net.ConnectException: 拒绝连接; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:792) at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:732) at org.apache.hadoop.ipc.Client.call(Client.java:1480) at org.apache.hadoop.ipc.Client.call(Client.java:1407) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:229) at com.sun.proxy.$Proxy83.startContainers(Unknown Source) at org.apache.hadoop.yarn.api.impl.pb.client.ContainerManagementProtocolPBClientImpl.startContainers(ContainerManagementProtocolPBClientImpl.java:96) at org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher.launch(AMLauncher.java:119) at org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher.run(AMLauncher.java:254) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.net.ConnectException: 拒绝连接 at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717) at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:531) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:495) at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:609) at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:707) at org.apache.hadoop.ipc.Client$Connection.access$2800(Client.java:370) at org.apache.hadoop.ipc.Client.getConnection(Client.java:1529) at org.apache.hadoop.ipc.Client.call(Client.java:1446) ... 9 more . Failing the application. 17/04/14 13:50:12 INFO mapreduce.Job: Counters: 0 metrics文件内容如下: ![图片说明](https://img-ask.csdn.net/upload/201704/14/1492150334_860585.png) 另外,在报错日志中发下这么句话Call From hadoop0/192.168.79.172 to  :54895 failed ,为什么54895只有端口前面没有IP,是不是环境没有配置正确?
执行jar报错 Hadoop java.io.IOException
[img=http://img.bbs.csdn.net/upload/201703/15/1489518401_142809.png][/img] Error: java.io.IOException: Initialization of all the collectors failed. Error in last collector was :interface javax.xml.soap.Text hadoop jar Hadoop_Demo1.jar /user/myData/ /user/out/ 执行简单jar包 17/03/15 02:52:37 INFO client.RMProxy: Connecting to ResourceManager at s0/192.168.253.130:8032 17/03/15 02:52:37 WARN mapreduce.JobResourceUploader: Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 17/03/15 02:52:38 INFO input.FileInputFormat: Total input paths to process : 2 17/03/15 02:52:38 INFO mapreduce.JobSubmitter: number of splits:2 17/03/15 02:52:38 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1489512856623_0004 17/03/15 02:52:39 INFO impl.YarnClientImpl: Submitted application application_1489512856623_0004 17/03/15 02:52:39 INFO mapreduce.Job: The url to track the job: http://s0:8088/proxy/application_1489512856623_0004/ 17/03/15 02:52:39 INFO mapreduce.Job: Running job: job_1489512856623_0004 17/03/15 02:52:50 INFO mapreduce.Job: Job job_1489512856623_0004 running in uber mode : false 17/03/15 02:52:50 INFO mapreduce.Job: map 0% reduce 0% 17/03/15 02:55:18 INFO mapreduce.Job: map 50% reduce 0% 17/03/15 02:55:18 INFO mapreduce.Job: Task Id : attempt_1489512856623_0004_m_000001_0, Status : FAILED Error: java.io.IOException: Initialization of all the collectors failed. Error in last collector was :interface javax.xml.soap.Text at org.apache.hadoop.mapred.MapTask.createSortingCollector(MapTask.java:414) at org.apache.hadoop.mapred.MapTask.access$100(MapTask.java:81) at org.apache.hadoop.mapred.MapTask$NewOutputCollector.<init>(MapTask.java:698) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:770) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158) Caused by: java.lang.ClassCastException: interface javax.xml.soap.Text at java.lang.Class.asSubclass(Class.java:3404) at org.apache.hadoop.mapred.JobConf.getOutputKeyComparator(JobConf.java:887) at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.init(MapTask.java:1004) at org.apache.hadoop.mapred.MapTask.createSortingCollector(MapTask.java:402) ... 9 more Container killed by the ApplicationMaster. 17/03/15 02:55:18 INFO mapreduce.Job: Task Id : attempt_1489512856623_0004_m_000000_0, Status : FAILED Error: java.io.IOException: Initialization of all the collectors failed. Error in last collector was :interface javax.xml.soap.Text at org.apache.hadoop.mapred.MapTask.createSortingCollector(MapTask.java:414) at org.apache.hadoop.mapred.MapTask.access$100(MapTask.java:81) at org.apache.hadoop.mapred.MapTask$NewOutputCollector.<init>(MapTask.java:698) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:770) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158) Caused by: java.lang.ClassCastException: interface javax.xml.soap.Text at java.lang.Class.asSubclass(Class.java:3404) at org.apache.hadoop.mapred.JobConf.getOutputKeyComparator(JobConf.java:887) at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.init(MapTask.java:1004) at org.apache.hadoop.mapred.MapTask.createSortingCollector(MapTask.java:402) ... 9 more 17/03/15 02:55:19 INFO mapreduce.Job: map 0% reduce 0% 17/03/15 02:55:31 INFO mapreduce.Job: Task Id : attempt_1489512856623_0004_m_000000_1, Status : FAILED Error: java.io.IOException: Initialization of all the collectors failed. Error in last collector was :interface javax.xml.soap.Text at org.apache.hadoop.mapred.MapTask.createSortingCollector(MapTask.java:414) at org.apache.hadoop.mapred.MapTask.access$100(MapTask.java:81) at org.apache.hadoop.mapred.MapTask$NewOutputCollector.<init>(MapTask.java:698) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:770) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
solaris部署hadoop集群跑wordcount报错
solaris部署hadoop集群跑wordcount报错, 信息如下: [admin@4bf635fa-5f3e-4b47-b42d-7558a6f0bbff ~]$ hadoop jar /opt/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.1.jar wordcount /input /output 15/08/20 00:48:09 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 15/08/20 00:48:10 INFO client.RMProxy: Connecting to ResourceManager at master/192.168.77.27:8032 15/08/20 00:48:11 INFO input.FileInputFormat: Total input paths to process : 3 15/08/20 00:48:11 INFO mapreduce.JobSubmitter: number of splits:3 15/08/20 00:48:11 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1439973008617_0002 15/08/20 00:48:13 INFO impl.YarnClientImpl: Submitted application application_1439973008617_0002 15/08/20 00:48:13 INFO mapreduce.Job: The url to track the job: http://master:8088/proxy/application_1439973008617_0002/ 15/08/20 00:48:13 INFO mapreduce.Job: Running job: job_1439973008617_0002 15/08/20 00:48:35 INFO mapreduce.Job: Job job_1439973008617_0002 running in uber mode : false 15/08/20 00:48:35 INFO mapreduce.Job: map 0% reduce 0% 15/08/20 00:48:35 INFO mapreduce.Job: Job job_1439973008617_0002 failed with state FAILED due to: Application application_1439973008617_0002 failed 2 times due to Error launching appattempt_1439973008617_0002_000002. Got exception: java.net.ConnectException: Call From localhost/127.0.0.1 to localhost:37524 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:422) at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:792) at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:732) at org.apache.hadoop.ipc.Client.call(Client.java:1480) at org.apache.hadoop.ipc.Client.call(Client.java:1407) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:229) at com.sun.proxy.$Proxy83.startContainers(Unknown Source) at org.apache.hadoop.yarn.api.impl.pb.client.ContainerManagementProtocolPBClientImpl.startContainers(ContainerManagementProtocolPBClientImpl.java:96) at org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher.launch(AMLauncher.java:119) at org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher.run(AMLauncher.java:254) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717) at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:531) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:495) at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:609) at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:707) at org.apache.hadoop.ipc.Client$Connection.access$2800(Client.java:370) at org.apache.hadoop.ipc.Client.getConnection(Client.java:1529) at org.apache.hadoop.ipc.Client.call(Client.java:1446) ... 9 more . Failing the application. 15/08/20 00:48:35 INFO mapreduce.Job: Counters: 0 hadoop dfsadmin -report 统计信息如下: [admin@4bf635fa-5f3e-4b47-b42d-7558a6f0bbff ~]$ hadoop dfsadmin -report DEPRECATED: Use of this script to execute hdfs command is deprecated. Instead use the hdfs command for it. 15/08/20 00:49:35 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable Configured Capacity: 344033951744 (320.41 GB) Present Capacity: 342854029952 (319.31 GB) DFS Remaining: 342853630464 (319.31 GB) DFS Used: 399488 (390.13 KB) DFS Used%: 0.00% Under replicated blocks: 7 Blocks with corrupt replicas: 0 Missing blocks: 0 Missing blocks (with replication factor 1): 0 ------------------------------------------------- Live datanodes (1): Name: 192.168.77.28:50010 (slave1) Hostname: localhost Decommission Status : Normal Configured Capacity: 344033951744 (320.41 GB) DFS Used: 399488 (390.13 KB) Non DFS Used: 1179921792 (1.10 GB) DFS Remaining: 342853630464 (319.31 GB) DFS Used%: 0.00% DFS Remaining%: 99.66% Configured Cache Capacity: 0 (0 B) Cache Used: 0 (0 B) Cache Remaining: 0 (0 B) Cache Used%: 100.00% Cache Remaining%: 0.00% Xceivers: 1 Last contact: Thu Aug 20 00:49:36 UTC 2015 192.168.77.27(master)跑ResourceManager和NameNode,192.168.77.28(slave)跑DataNode和NodeManger,貌似hadoop把DataNode和NodeManger的hostname都认成localhost了,所以才找不到报错了。 同样的配置在centos一切正常,在solaris就报错 ![NodeManger信息](https://img-ask.csdn.net/upload/201508/20/1440038080_659898.png) ![DataNode信息](https://img-ask.csdn.net/upload/201508/20/1440038161_998191.png)
hadoop2.5.2 mapreduce作业失败
``` 16/06/14 03:26:45 INFO client.RMProxy: Connecting to ResourceManager at centos1/192.168.6.132:8032 16/06/14 03:26:47 WARN mapreduce.JobSubmitter: Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 16/06/14 03:26:47 INFO input.FileInputFormat: Total input paths to process : 1 16/06/14 03:26:48 INFO mapreduce.JobSubmitter: number of splits:1 16/06/14 03:26:48 INFO Configuration.deprecation: mapred.job.tracker is deprecated. Instead, use mapreduce.jobtracker.address 16/06/14 03:26:48 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1465885546873_0002 16/06/14 03:26:49 INFO impl.YarnClientImpl: Submitted application application_1465885546873_0002 16/06/14 03:26:49 INFO mapreduce.Job: The url to track the job: http://centos1:8088/proxy/application_1465885546873_0002/ 16/06/14 03:26:49 INFO mapreduce.Job: Running job: job_1465885546873_0002 16/06/14 03:27:10 INFO mapreduce.Job: Job job_1465885546873_0002 running in uber mode : false 16/06/14 03:27:10 INFO mapreduce.Job: map 0% reduce 0% 16/06/14 03:27:10 INFO mapreduce.Job: Job job_1465885546873_0002 failed with state FAILED due to: Application application_1465885546873_0002 failed 2 times due to Error launching appattempt_1465885546873_0002_000002. Got exception: java.net.ConnectException: Call From local.localdomain/127.0.0.1 to local:50334 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:783) at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:730) at org.apache.hadoop.ipc.Client.call(Client.java:1415) at org.apache.hadoop.ipc.Client.call(Client.java:1364) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206) at com.sun.proxy.$Proxy32.startContainers(Unknown Source) at org.apache.hadoop.yarn.api.impl.pb.client.ContainerManagementProtocolPBClientImpl.startContainers(ContainerManagementProtocolPBClientImpl.java:96) at org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher.launch(AMLauncher.java:118) at org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher.run(AMLauncher.java:249) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) ``` 然后错误日志如下 ``` 2016-06-14 03:26:49,936 INFO org.apache.hadoop.yarn.server.resourcemanager.amlaucher.AMLauncher: Setting up container Container: [ContainerId: container_1465885546873_0002_01_000001, NodeId: local:42709, NodeHttpAddress: local:8042, Resource: <memory:2048, vCores:1>, Priority: 0, Token: Token { kind: ContainerToken, service: 127.0.0.1:42709 }, ] for AM appattempt_1465885546873_0002_000001 2016-06-14 03:26:49,936 INFO org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher: Command to launch container container_1465885546873_0002_01_000001 : $JAVA_HOME/bin/java -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=<LOG_DIR> -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Xmx1024m org.apache.hadoop.mapreduce.v2.app.MRAppMaster 1><LOG_DIR>/stdout 2><LOG_DIR>/stderr 2016-06-14 03:26:50,948 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: local/127.0.0.1:42709. Already tried 0 time(s); retry policy is RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) 2016-06-14 03:26:51,950 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: local/127.0.0.1:42709. Already tried 1 time(s); retry policy is RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) 2016-06-14 03:26:52,951 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: local/127.0.0.1:42709. Already tried 2 time(s); retry policy is RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) 2016-06-14 03:26:53,952 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: local/127.0.0.1:42709. Already tried 3 time(s); retry policy is RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) 2016-06-14 03:26:54,953 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: local/127.0.0.1:42709. Already tried 4 time(s); retry policy is RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) 2016-06-14 03:26:55,954 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: local/127.0.0.1:42709. Already tried 5 time(s); retry policy is RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) 2016-06-14 03:26:56,956 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: local/127.0.0.1:42709. Already tried 6 time(s); retry policy is RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) 2016-06-14 03:26:57,957 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: local/127.0.0.1:42709. Already tried 7 time(s); retry policy is RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) 2016-06-14 03:26:58,959 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: local/127.0.0.1:42709. Already tried 8 time(s); retry policy is RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) 2016-06-14 03:26:59,960 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: local/127.0.0.1:42709. Already tried 9 time(s); retry policy is RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) 2016-06-14 03:26:59,962 INFO org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher: Error launching appattempt_1465885546873_0002_000001. Got exception: java.net.ConnectException: Call From local.localdomain/127.0.0.1 to local:42709 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ``` core-site.xml如下 ``` <configuration> <property> <name>ha.zookeeper.quorum</name> <value>centos1:2181,centos2:2181,centos3:2181</value> </property> <property> <name>hadoop.tmp.dir</name> <value>/opt/hadoop2.5</value> </property> <property> <name>fs.defaultFS</name> <value>hdfs://mycluster</value> </property> </configuration> ``` hdfs-site.xml如下 ``` <configuration> <property> <name>dfs.nameservices</name> <value>mycluster</value> </property> <property> <name>dfs.ha.namenodes.mycluster</name> <value>centos1,centos2</value> </property> <property> <name>dfs.namenode.rpc-address.mycluster.centos1</name> <value>centos1:8020</value> </property> <property> <name>dfs.namenode.rpc-address.mycluster.centos2</name> <value>centos2:8020</value> </property> <property> <name>dfs.namenode.http-address.mycluster.centos1</name> <value>centos1:50070</value> </property> <property> <name>dfs.namenode.http-address.mycluster.centos2</name> <value>centos2:50070</value> </property> <property> <name>dfs.namenode.shared.edits.dir</name> <value>qjournal://centos2:8485;centos3:8485;centos4:8485/mycluster</value> </property> <property> <name>dfs.client.failover.proxy.provider.mycluster</name> <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value> </property> <property> <name>dfs.ha.fencing.methods</name> <value>sshfence</value> </property> <property> <name>dfs.ha.fencing.ssh.private-key-files</name> <value>/root/.ssh/id_dsa</value> </property> <property> <name>dfs.journalnode.edits.dir</name> <value>/home/hadoop-data</value> </property> <property> <name>dfs.ha.automatic-failover.enabled</name> <value>true</value> </property> </configuration> ``` yarn-site.xml如下 ``` <configuration> <property> <name>yarn.nodemanager.aux-services</name> <value>mapreduce_shuffle</value> </property> <property> <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name> <value>org.apache.hadoop.mapred.ShuffleHandler</value> </property> <property> <name>yarn.resourcemanager.hostname</name> <value>centos1</value> </property> <property> <name>yarn.resourcemanager.address</name> <value>centos1:8032</value> </property> <property> <name>yarn.resourcemanager.admin.address</name> <value>centos1:8033</value> </property> </configuration> ``` mapred-site.xml如下 ``` <configuration> <property> <name>mapreduce.framework.name</name> <value>yarn</value> </property> </configuration> ``` slaves如下 ``` centos2 centos3 centos4 ``` hosts如下 ``` 127.0.0.1 local local.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 192.168.6.132 centos1 192.168.6.133 centos2 192.168.6.134 centos3 192.168.6.135 centos4 ```
在hadoop上用mahout的随机森林算法训练机器学习分类器
我搭了三台CentOS7虚拟机,安装好了hadoop-3.0.0和mahout-distirution-0.9。 准备用随机森林算法建立模型。第一步生成好了描述文件(/KDDdes.info);第二步建立好了模型(/user/hadoop/nsl-forest);但第三步就会报错。 这是HDFS: ``` [hadoop@hadoop1 ~]$ hadoop fs -ls / Found 5 items -rw-r--r-- 1 hadoop supergroup 3365886 2018-05-02 23:58 /KDDTest+.arff -rw-r--r-- 1 hadoop supergroup 18742306 2018-05-02 23:40 /KDDTrain+.arff -rw-r--r-- 1 hadoop supergroup 2795 2018-05-02 23:42 /KDDdes.info drwx------ - hadoop supergroup 0 2018-04-29 20:05 /tmp drwxr-xr-x - hadoop supergroup 0 2018-04-29 20:05 /user ``` 这是第三步 ``` [hadoop@hadoop1 ~]$ hadoop jar /opt/mahout-distribution-0.9/mahout-examples-0.9-job.jar org.apache.mahout.classifier.df.mapreduce.TestForest -i /KDDTest+.arff -ds /KDDdes.info -m /user/hadoop/nsl-forest -a -mr -o nsl-prediction 2018-05-03 00:01:28,469 INFO mapreduce.Classifier: Adding the dataset to the DistributedCache 2018-05-03 00:01:28,471 INFO mapreduce.Classifier: Adding the decision forest to the DistributedCache 2018-05-03 00:01:28,475 INFO mapreduce.Classifier: Configuring the job... 2018-05-03 00:01:28,479 INFO mapreduce.Classifier: Running the job... 2018-05-03 00:01:28,568 INFO client.RMProxy: Connecting to ResourceManager at hadoop1/192.168.80.100:8032 2018-05-03 00:01:28,960 INFO mapreduce.JobResourceUploader: Disabling Erasure Coding for path: /tmp/hadoop-yarn/staging/hadoop/.staging/job_1525056498669_0005 2018-05-03 00:01:29,462 INFO input.FileInputFormat: Total input files to process : 1 2018-05-03 00:01:29,968 INFO mapreduce.JobSubmitter: number of splits:1 2018-05-03 00:01:30,080 INFO Configuration.deprecation: yarn.resourcemanager.system-metrics-publisher.enabled is deprecated. Instead, use yarn.system-metrics-publisher.enabled 2018-05-03 00:01:30,685 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1525056498669_0005 2018-05-03 00:01:30,686 INFO mapreduce.JobSubmitter: Executing with tokens: [] 2018-05-03 00:01:31,277 INFO conf.Configuration: resource-types.xml not found 2018-05-03 00:01:31,278 INFO resource.ResourceUtils: Unable to find 'resource-types.xml'. 2018-05-03 00:01:31,330 INFO impl.YarnClientImpl: Submitted application application_1525056498669_0005 2018-05-03 00:01:31,358 INFO mapreduce.Job: The url to track the job: http://hadoop1:8088/proxy/application_1525056498669_0005/ 2018-05-03 00:01:31,359 INFO mapreduce.Job: Running job: job_1525056498669_0005 2018-05-03 00:01:38,470 INFO mapreduce.Job: Job job_1525056498669_0005 running in uber mode : false 2018-05-03 00:01:38,470 INFO mapreduce.Job: map 0% reduce 0% 2018-05-03 00:01:42,604 INFO mapreduce.Job: Task Id : attempt_1525056498669_0005_m_000000_0, Status : FAILED Error: java.lang.ArrayIndexOutOfBoundsException: 946827879 at org.apache.mahout.classifier.df.node.Node.read(Node.java:58) at org.apache.mahout.classifier.df.DecisionForest.readFields(DecisionForest.java:197) at org.apache.mahout.classifier.df.DecisionForest.read(DecisionForest.java:203) at org.apache.mahout.classifier.df.DecisionForest.load(DecisionForest.java:225) at org.apache.mahout.classifier.df.mapreduce.Classifier$CMapper.setup(Classifier.java:209) at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:143) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:794) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:342) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:174) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1962) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:168) 2018-05-03 00:01:46,641 INFO mapreduce.Job: Task Id : attempt_1525056498669_0005_m_000000_1, Status : FAILED Error: java.lang.ArrayIndexOutOfBoundsException: 946827879 at org.apache.mahout.classifier.df.node.Node.read(Node.java:58) at org.apache.mahout.classifier.df.DecisionForest.readFields(DecisionForest.java:197) at org.apache.mahout.classifier.df.DecisionForest.read(DecisionForest.java:203) at org.apache.mahout.classifier.df.DecisionForest.load(DecisionForest.java:225) at org.apache.mahout.classifier.df.mapreduce.Classifier$CMapper.setup(Classifier.java:209) at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:143) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:794) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:342) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:174) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1962) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:168) 2018-05-03 00:01:51,687 INFO mapreduce.Job: Task Id : attempt_1525056498669_0005_m_000000_2, Status : FAILED Error: java.lang.ArrayIndexOutOfBoundsException: 946827879 at org.apache.mahout.classifier.df.node.Node.read(Node.java:58) at org.apache.mahout.classifier.df.DecisionForest.readFields(DecisionForest.java:197) at org.apache.mahout.classifier.df.DecisionForest.read(DecisionForest.java:203) at org.apache.mahout.classifier.df.DecisionForest.load(DecisionForest.java:225) at org.apache.mahout.classifier.df.mapreduce.Classifier$CMapper.setup(Classifier.java:209) at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:143) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:794) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:342) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:174) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1962) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:168) 2018-05-03 00:01:55,745 INFO mapreduce.Job: map 100% reduce 0% 2018-05-03 00:01:57,766 INFO mapreduce.Job: Job job_1525056498669_0005 failed with state FAILED due to: Task failed task_1525056498669_0005_m_000000 Job failed as tasks failed. failedMaps:1 failedReduces:0 killedMaps:0 killedReduces: 0 2018-05-03 00:01:57,844 INFO mapreduce.Job: Counters: 9 Job Counters Failed map tasks=4 Launched map tasks=4 Other local map tasks=3 Data-local map tasks=1 Total time spent by all maps in occupied slots (ms)=10027 Total time spent by all reduces in occupied slots (ms)=0 Total time spent by all map tasks (ms)=10027 Total vcore-milliseconds taken by all map tasks=10027 Total megabyte-milliseconds taken by all map tasks=10267648 Exception in thread "main" java.lang.IllegalStateException: Job failed! at org.apache.mahout.classifier.df.mapreduce.Classifier.run(Classifier.java:127) at org.apache.mahout.classifier.df.mapreduce.TestForest.mapreduce(TestForest.java:188) at org.apache.mahout.classifier.df.mapreduce.TestForest.testForest(TestForest.java:174) at org.apache.mahout.classifier.df.mapreduce.TestForest.run(TestForest.java:146) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) at org.apache.mahout.classifier.df.mapreduce.TestForest.main(TestForest.java:315) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.util.RunJar.run(RunJar.java:239) at org.apache.hadoop.util.RunJar.main(RunJar.java:153) ``` 这是什么问题,求助大神!!!
用eclipse连接虚拟机hadoop集群执行MapReduce程序,但是报以下错误,请问如何解决?
# 说明:eclipse中关于hadoop的各项advanced parameter参数均已按配置文件进行配置。但在执行过程中还是报如下错误,请问如何解决。 # 执行日志: 2018-09-22 22:59:11,429 INFO [org.apache.commons.beanutils.FluentPropertyBeanIntrospector] - Error when creating PropertyDescriptor for public final void org.apache.commons.configuration2.AbstractConfiguration.setProperty(java.lang.String,java.lang.Object)! Ignoring this property. 2018-09-22 22:59:11,443 WARN [org.apache.hadoop.metrics2.impl.MetricsConfig] - Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2018-09-22 22:59:11,496 INFO [org.apache.hadoop.metrics2.impl.MetricsSystemImpl] - Scheduled Metric snapshot period at 10 second(s). 2018-09-22 22:59:11,496 INFO [org.apache.hadoop.metrics2.impl.MetricsSystemImpl] - JobTracker metrics system started 2018-09-22 22:59:20,863 WARN [org.apache.hadoop.mapreduce.JobResourceUploader] - Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2018-09-22 22:59:20,879 WARN [org.apache.hadoop.mapreduce.JobResourceUploader] - No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2018-09-22 22:59:20,928 INFO [org.apache.hadoop.mapreduce.lib.input.FileInputFormat] - Total input files to process : 1 2018-09-22 22:59:20,984 INFO [org.apache.hadoop.mapreduce.JobSubmitter] - number of splits:1 2018-09-22 22:59:21,072 INFO [org.apache.hadoop.mapreduce.JobSubmitter] - Submitting tokens for job: job_local1513265977_0001 2018-09-22 22:59:21,074 INFO [org.apache.hadoop.mapreduce.JobSubmitter] - Executing with tokens: [] 2018-09-22 22:59:21,950 INFO [org.apache.hadoop.mapred.LocalDistributedCacheManager] - Creating symlink: \tmp\hadoop-启政先生\mapred\local\1537628361150\movies.csv <- G:\java_workspace\MapReduce_DEMO/movies.csv 2018-09-22 22:59:21,995 WARN [org.apache.hadoop.fs.FileUtil] - Command 'E:\hadoop-3.0.0\bin\winutils.exe symlink G:\java_workspace\MapReduce_DEMO\movies.csv \tmp\hadoop-启政先生\mapred\local\1537628361150\movies.csv' failed 1 with: CreateSymbolicLink error (1314): ??????????? 2018-09-22 22:59:21,995 WARN [org.apache.hadoop.mapred.LocalDistributedCacheManager] - Failed to create symlink: \tmp\hadoop-启政先生\mapred\local\1537628361150\movies.csv <- G:\java_workspace\MapReduce_DEMO/movies.csv 2018-09-22 22:59:21,996 INFO [org.apache.hadoop.mapred.LocalDistributedCacheManager] - Localized hdfs://192.168.5.110:9000/temp/input/movies.csv as file:/tmp/hadoop-启政先生/mapred/local/1537628361150/movies.csv 2018-09-22 22:59:22,046 INFO [org.apache.hadoop.mapreduce.Job] - The url to track the job: http://localhost:8080/ 2018-09-22 22:59:22,047 INFO [org.apache.hadoop.mapreduce.Job] - Running job: job_local1513265977_0001 2018-09-22 22:59:22,047 INFO [org.apache.hadoop.mapred.LocalJobRunner] - OutputCommitter set in config null 2018-09-22 22:59:22,051 INFO [org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter] - File Output Committer Algorithm version is 2 2018-09-22 22:59:22,051 INFO [org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter] - FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false 2018-09-22 22:59:22,052 INFO [org.apache.hadoop.mapred.LocalJobRunner] - OutputCommitter is org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter 2018-09-22 22:59:22,100 INFO [org.apache.hadoop.mapred.LocalJobRunner] - Waiting for map tasks 2018-09-22 22:59:22,101 INFO [org.apache.hadoop.mapred.LocalJobRunner] - Starting task: attempt_local1513265977_0001_m_000000_0 2018-09-22 22:59:22,120 INFO [org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter] - File Output Committer Algorithm version is 2 2018-09-22 22:59:22,120 INFO [org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter] - FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false 2018-09-22 22:59:22,128 INFO [org.apache.hadoop.yarn.util.ProcfsBasedProcessTree] - ProcfsBasedProcessTree currently is supported only on Linux. 2018-09-22 22:59:22,169 INFO [org.apache.hadoop.mapred.Task] - Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@7ef907ef 2018-09-22 22:59:22,172 INFO [org.apache.hadoop.mapred.MapTask] - Processing split: hdfs://192.168.5.110:9000/temp/input/ratings.csv:0+2438233 ----------cachePath=/temp/input/movies.csv---------- 2018-09-22 22:59:22,226 INFO [org.apache.hadoop.mapred.LocalJobRunner] - map task executor complete. 2018-09-22 22:59:22,233 WARN [org.apache.hadoop.mapred.LocalJobRunner] - job_local1513265977_0001 java.lang.Exception: java.io.FileNotFoundException: \temp\input\movies.csv (系统找不到指定的路径。) at org.apache.hadoop.mapred.LocalJobRunner$Job.runTasks(LocalJobRunner.java:492) at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:552) Caused by: java.io.FileNotFoundException: \temp\input\movies.csv (系统找不到指定的路径。) at java.io.FileInputStream.open0(Native Method) at java.io.FileInputStream.open(Unknown Source) at java.io.FileInputStream.<init>(Unknown Source) at java.io.FileInputStream.<init>(Unknown Source) at java.io.FileReader.<init>(Unknown Source) at MovieJoinExercise1.MovieJoin$MovieJoinMapper.setup(MovieJoin.java:79) at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:143) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:794) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:342) at org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run(LocalJobRunner.java:271) at java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.util.concurrent.FutureTask.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) 2018-09-22 22:59:23,051 INFO [org.apache.hadoop.mapreduce.Job] - Job job_local1513265977_0001 running in uber mode : false 2018-09-22 22:59:23,052 INFO [org.apache.hadoop.mapreduce.Job] - map 0% reduce 0% 2018-09-22 22:59:23,053 INFO [org.apache.hadoop.mapreduce.Job] - Job job_local1513265977_0001 failed with state FAILED due to: NA 2018-09-22 22:59:23,058 INFO [org.apache.hadoop.mapreduce.Job] - Counters: 0
map 和 reduce0%就失败了,能帮我看看是什么原因吗?
17/06/13 15:30:29 INFO client.RMProxy: Connecting to ResourceManager at master/192.168.1.100:8032 17/06/13 15:30:29 INFO input.FileInputFormat: Total input paths to process : 1 17/06/13 15:30:29 INFO mapreduce.JobSubmitter: number of splits:1 17/06/13 15:30:30 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1497335254625_0003 17/06/13 15:30:30 INFO impl.YarnClientImpl: Submitted application application_1497335254625_0003 17/06/13 15:30:30 INFO mapreduce.Job: The url to track the job: http://master:8088/proxy/application_1497335254625_0003/ 17/06/13 15:30:30 INFO mapreduce.Job: Running job: job_1497335254625_0003 17/06/13 15:30:34 INFO mapreduce.Job: Job job_1497335254625_0003 running in uber mode : false 17/06/13 15:30:34 INFO mapreduce.Job: map 0% reduce 0% 17/06/13 15:30:39 INFO mapreduce.Job: Task Id : attempt_1497335254625_0003_m_000000_0, Status : FAILED Exception from container-launch. Container id: container_1497335254625_0003_01_000002 Exit code: 1 Exception message: /usr/hadoop/hadoop-2.6.0/tmp/hadoopData /nm-local-dir/usercache/hadoop/appcache/application_1497335254625_0003/container_1497335254625_0003_01_000002/default_container_executor_session.sh: 行 3: /usr/hadoop/hadoop-2.6.0/tmp/hadoopData: 是一个目录 /bin/mv: 目标"/nm-local-dir/nmPrivate/application_1497335254625_0003/container_1497335254625_0003_01_000002/container_1497335254625_0003_01_000002.pid" 不是目录 Stack trace: ExitCodeException exitCode=1: /usr/hadoop/hadoop-2.6.0/tmp/hadoopData /nm-local-dir/usercache/hadoop/appcache/application_1497335254625_0003/container_1497335254625_0003_01_000002/default_container_executor_session.sh: 行 3: /usr/hadoop/hadoop-2.6.0/tmp/hadoopData: 是一个目录 /bin/mv: 目标"/nm-local-dir/nmPrivate/application_1497335254625_0003/container_1497335254625_0003_01_000002/container_1497335254625_0003_01_000002.pid" 不是目录 at org.apache.hadoop.util.Shell.runCommand(Shell.java:538) at org.apache.hadoop.util.Shell.run(Shell.java:455) at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:715) at org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.launchContainer(DefaultContainerExecutor.java:211) at org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:302) at org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:82) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Container exited with a non-zero exit code 1 17/06/13 15:30:45 INFO mapreduce.Job: Task Id : attempt_1497335254625_0003_m_000000_1, Status : FAILED Exception from container-launch. Container id: container_1497335254625_0003_01_000003 Exit code: 1 Exception message: /usr/hadoop/hadoop-2.6.0/tmp/hadoopData /nm-local-dir/usercache/hadoop/appcache/application_1497335254625_0003/container_1497335254625_0003_01_000003/default_container_executor_session.sh: 行 3: /usr/hadoop/hadoop-2.6.0/tmp/hadoopData: 是一个目录 /bin/mv: 目标"/nm-local-dir/nmPrivate/application_1497335254625_0003/container_1497335254625_0003_01_000003/container_1497335254625_0003_01_000003.pid" 不是目录 Stack trace: ExitCodeException exitCode=1: /usr/hadoop/hadoop-2.6.0/tmp/hadoopData /nm-local-dir/usercache/hadoop/appcache/application_1497335254625_0003/container_1497335254625_0003_01_000003/default_container_executor_session.sh: 行 3: /usr/hadoop/hadoop-2.6.0/tmp/hadoopData: 是一个目录 /bin/mv: 目标"/nm-local-dir/nmPrivate/application_1497335254625_0003/container_1497335254625_0003_01_000003/container_1497335254625_0003_01_000003.pid" 不是目录 at org.apache.hadoop.util.Shell.runCommand(Shell.java:538) at org.apache.hadoop.util.Shell.run(Shell.java:455) at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:715) at org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.launchContainer(DefaultContainerExecutor.java:211) at org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:302) at org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:82) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Container exited with a non-zero exit code 1 17/06/13 15:30:51 INFO mapreduce.Job: Task Id : attempt_1497335254625_0003_m_000000_2, Status : FAILED Exception from container-launch. Container id: container_1497335254625_0003_01_000004 Exit code: 1 Exception message: /usr/hadoop/hadoop-2.6.0/tmp/hadoopData /nm-local-dir/usercache/hadoop/appcache/application_1497335254625_0003/container_1497335254625_0003_01_000004/default_container_executor_session.sh: 行 3: /usr/hadoop/hadoop-2.6.0/tmp/hadoopData: 是一个目录 /bin/mv: 目标"/nm-local-dir/nmPrivate/application_1497335254625_0003/container_1497335254625_0003_01_000004/container_1497335254625_0003_01_000004.pid" 不是目录 Stack trace: ExitCodeException exitCode=1: /usr/hadoop/hadoop-2.6.0/tmp/hadoopData /nm-local-dir/usercache/hadoop/appcache/application_1497335254625_0003/container_1497335254625_0003_01_000004/default_container_executor_session.sh: 行 3: /usr/hadoop/hadoop-2.6.0/tmp/hadoopData: 是一个目录 /bin/mv: 目标"/nm-local-dir/nmPrivate/application_1497335254625_0003/container_1497335254625_0003_01_000004/container_1497335254625_0003_01_000004.pid" 不是目录 at org.apache.hadoop.util.Shell.runCommand(Shell.java:538) at org.apache.hadoop.util.Shell.run(Shell.java:455) at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:715) at org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.launchContainer(DefaultContainerExecutor.java:211) at org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:302) at org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:82) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Container exited with a non-zero exit code 1 17/06/13 15:30:57 INFO mapreduce.Job: map 100% reduce 100% 17/06/13 15:30:57 INFO mapreduce.Job: Job job_1497335254625_0003 failed with state FAILED due to: Task failed task_1497335254625_0003_m_000000 Job failed as tasks failed. failedMaps:1 failedReduces:0 17/06/13 15:30:57 INFO mapreduce.Job: Counters: 12 Job Counters Failed map tasks=4 Launched map tasks=4 Other local map tasks=3 Data-local map tasks=1 Total time spent by all maps in occupied slots (ms)=14994 Total time spent by all reduces in occupied slots (ms)=0 Total time spent by all map tasks (ms)=14994 Total vcore-seconds taken by all map tasks=14994 Total megabyte-seconds taken by all map tasks=15353856 Map-Reduce Framework CPU time spent (ms)=0 Physical memory (bytes) snapshot=0 Virtual memory (bytes) snapshot=0 hadoop@master:/usr/hadoop/hadoop-2.6.0$
Sqoop将数据从hive导入mysql报错,各位帮我看看
这是运行的命令: liuyanbing@ubuntu:/opt/sqoop$ bin/sqoop export --connect jdbc:mysql://localhost:3306/dbtaobao --username root --password root --table user_log --export-dir '/user/hive/warehouse/dbtaobao.db/inner_user_log' --fields-terminated-by ','; 报错内容: Warning: /opt/sqoop/../hcatalog does not exist! HCatalog jobs will fail. Please set $HCAT_HOME to the root of your HCatalog installation. Warning: /opt/sqoop/../accumulo does not exist! Accumulo imports will fail. Please set $ACCUMULO_HOME to the root of your Accumulo installation. Warning: /opt/sqoop/../zookeeper does not exist! Accumulo imports will fail. Please set $ZOOKEEPER_HOME to the root of your Zookeeper installation. 2019-06-11 16:05:04,541 INFO sqoop.Sqoop: Running Sqoop version: 1.4.6 2019-06-11 16:05:04,573 WARN tool.BaseSqoopTool: Setting your password on the command-line is insecure. Consider using -P instead. 2019-06-11 16:05:04,678 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset. 2019-06-11 16:05:04,678 INFO tool.CodeGenTool: Beginning code generation Tue Jun 11 16:05:04 CST 2019 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification. 2019-06-11 16:05:05,241 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `user_log` AS t LIMIT 1 2019-06-11 16:05:05,379 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `user_log` AS t LIMIT 1 2019-06-11 16:05:05,392 INFO orm.CompilationManager: HADOOP_MAPRED_HOME is /bigdata/hadoop-3.1.1 Note: /tmp/sqoop-liuyanbing/compile/990c7e516f6811ff0f7c264686938932/user_log.java uses or overrides a deprecated API. Note: Recompile with -Xlint:deprecation for details. 2019-06-11 16:05:09,951 INFO orm.CompilationManager: Writing jar file: /tmp/sqoop-liuyanbing/compile/990c7e516f6811ff0f7c264686938932/user_log.jar 2019-06-11 16:05:09,960 INFO mapreduce.ExportJobBase: Beginning export of user_log 2019-06-11 16:05:09,960 INFO Configuration.deprecation: mapred.job.tracker is deprecated. Instead, use mapreduce.jobtracker.address 2019-06-11 16:05:10,093 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2019-06-11 16:05:10,131 INFO Configuration.deprecation: mapred.jar is deprecated. Instead, use mapreduce.job.jar 2019-06-11 16:05:11,220 INFO Configuration.deprecation: mapred.reduce.tasks.speculative.execution is deprecated. Instead, use mapreduce.reduce.speculative 2019-06-11 16:05:11,224 INFO Configuration.deprecation: mapred.map.tasks.speculative.execution is deprecated. Instead, use mapreduce.map.speculative 2019-06-11 16:05:11,225 INFO Configuration.deprecation: mapred.map.tasks is deprecated. Instead, use mapreduce.job.maps 2019-06-11 16:05:11,399 INFO client.RMProxy: Connecting to ResourceManager at localhost/127.0.0.1:8032 2019-06-11 16:05:12,478 INFO mapreduce.JobResourceUploader: Disabling Erasure Coding for path: /tmp/hadoop-yarn/staging/liuyanbing/.staging/job_1560238973821_0003 2019-06-11 16:05:15,272 WARN hdfs.DataStreamer: Caught exception java.lang.InterruptedException at java.lang.Object.wait(Native Method) at java.lang.Thread.join(Thread.java:1252) at java.lang.Thread.join(Thread.java:1326) at org.apache.hadoop.hdfs.DataStreamer.closeResponder(DataStreamer.java:986) at org.apache.hadoop.hdfs.DataStreamer.endBlock(DataStreamer.java:640) at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:810) 2019-06-11 16:05:18,771 INFO input.FileInputFormat: Total input files to process : 1 2019-06-11 16:05:18,780 INFO input.FileInputFormat: Total input files to process : 1 2019-06-11 16:05:19,285 INFO mapreduce.JobSubmitter: number of splits:4 2019-06-11 16:05:19,352 INFO Configuration.deprecation: mapred.map.tasks.speculative.execution is deprecated. Instead, use mapreduce.map.speculative 2019-06-11 16:05:19,353 INFO Configuration.deprecation: yarn.resourcemanager.system-metrics-publisher.enabled is deprecated. Instead, use yarn.system-metrics-publisher.enabled 2019-06-11 16:05:19,472 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1560238973821_0003 2019-06-11 16:05:19,473 INFO mapreduce.JobSubmitter: Executing with tokens: [] 2019-06-11 16:05:19,959 INFO conf.Configuration: resource-types.xml not found 2019-06-11 16:05:19,959 INFO resource.ResourceUtils: Unable to find 'resource-types.xml'. 2019-06-11 16:05:20,049 INFO impl.YarnClientImpl: Submitted application application_1560238973821_0003 2019-06-11 16:05:20,105 INFO mapreduce.Job: The url to track the job: http://ubuntu:8088/proxy/application_1560238973821_0003/ 2019-06-11 16:05:20,106 INFO mapreduce.Job: Running job: job_1560238973821_0003 2019-06-11 16:05:29,273 INFO mapreduce.Job: Job job_1560238973821_0003 running in uber mode : false 2019-06-11 16:05:29,286 INFO mapreduce.Job: map 0% reduce 0% 2019-06-11 16:05:42,450 INFO mapreduce.Job: Task Id : attempt_1560238973821_0003_m_000002_0, Status : FAILED [2019-06-11 16:05:39.558]Container [pid=22666,containerID=container_1560238973821_0003_01_000004] is running 318323200B beyond the 'VIRTUAL' memory limit. Current usage: 125.2 MB of 1 GB physical memory used; 2.4 GB of 2.1 GB virtual memory used. Killing container. Dump of the process-tree for container_1560238973821_0003_01_000004 : |- PID PPID PGRPID SESSID CMD_NAME USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) RSSMEM_USAGE(PAGES) FULL_CMD_LINE |- 22910 22666 22666 22666 (java) 302 45 2558558208 31405 /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000004/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000004 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000002_0 4 |- 22666 22656 22666 22666 (bash) 0 0 14622720 634 /bin/bash -c /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000004/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000004 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000002_0 4 1>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000004/stdout 2>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000004/stderr [2019-06-11 16:05:40.618]Container killed on request. Exit code is 143 [2019-06-11 16:05:40.619]Container exited with a non-zero exit code 143. 2019-06-11 16:05:42,479 INFO mapreduce.Job: Task Id : attempt_1560238973821_0003_m_000001_0, Status : FAILED [2019-06-11 16:05:39.558]Container [pid=22651,containerID=container_1560238973821_0003_01_000003] is running 320690688B beyond the 'VIRTUAL' memory limit. Current usage: 127.5 MB of 1 GB physical memory used; 2.4 GB of 2.1 GB virtual memory used. Killing container. Dump of the process-tree for container_1560238973821_0003_01_000003 : |- PID PPID PGRPID SESSID CMD_NAME USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) RSSMEM_USAGE(PAGES) FULL_CMD_LINE |- 22955 22651 22651 22651 (java) 296 49 2560925696 32025 /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000003/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000003 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000001_0 3 |- 22651 22649 22651 22651 (bash) 0 0 14622720 627 /bin/bash -c /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000003/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000003 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000001_0 3 1>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000003/stdout 2>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000003/stderr [2019-06-11 16:05:40.618]Container killed on request. Exit code is 143 [2019-06-11 16:05:40.621]Container exited with a non-zero exit code 143. 2019-06-11 16:05:42,480 INFO mapreduce.Job: Task Id : attempt_1560238973821_0003_m_000003_0, Status : FAILED [2019-06-11 16:05:38.617]Container [pid=22749,containerID=container_1560238973821_0003_01_000005] is running 320125440B beyond the 'VIRTUAL' memory limit. Current usage: 126.4 MB of 1 GB physical memory used; 2.4 GB of 2.1 GB virtual memory used. Killing container. Dump of the process-tree for container_1560238973821_0003_01_000005 : |- PID PPID PGRPID SESSID CMD_NAME USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) RSSMEM_USAGE(PAGES) FULL_CMD_LINE |- 22987 22749 22749 22749 (java) 324 37 2560360448 31709 /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000005/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000005 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000003_0 5 |- 22749 22720 22749 22749 (bash) 0 1 14622720 640 /bin/bash -c /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000005/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000005 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000003_0 5 1>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000005/stdout 2>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000005/stderr [2019-06-11 16:05:40.620]Container killed on request. Exit code is 143 [2019-06-11 16:05:40.622]Container exited with a non-zero exit code 143. 2019-06-11 16:05:42,482 INFO mapreduce.Job: Task Id : attempt_1560238973821_0003_m_000000_0, Status : FAILED [2019-06-11 16:05:39.558]Container [pid=22675,containerID=container_1560238973821_0003_01_000002] is running 319543808B beyond the 'VIRTUAL' memory limit. Current usage: 125.4 MB of 1 GB physical memory used; 2.4 GB of 2.1 GB virtual memory used. Killing container. Dump of the process-tree for container_1560238973821_0003_01_000002 : |- PID PPID PGRPID SESSID CMD_NAME USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) RSSMEM_USAGE(PAGES) FULL_CMD_LINE |- 22937 22675 22675 22675 (java) 316 38 2559778816 31497 /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000002/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000002 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000000_0 2 |- 22675 22670 22675 22675 (bash) 0 0 14622720 612 /bin/bash -c /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000002/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000002 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000000_0 2 1>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000002/stdout 2>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000002/stderr [2019-06-11 16:05:40.619]Container killed on request. Exit code is 143 [2019-06-11 16:05:40.622]Container exited with a non-zero exit code 143. 2019-06-11 16:05:52,546 INFO mapreduce.Job: Task Id : attempt_1560238973821_0003_m_000002_1, Status : FAILED [2019-06-11 16:05:50.910]Container [pid=23116,containerID=container_1560238973821_0003_01_000006] is running 282286592B beyond the 'VIRTUAL' memory limit. Current usage: 68.6 MB of 1 GB physical memory used; 2.4 GB of 2.1 GB virtual memory used. Killing container. Dump of the process-tree for container_1560238973821_0003_01_000006 : |- PID PPID PGRPID SESSID CMD_NAME USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) RSSMEM_USAGE(PAGES) FULL_CMD_LINE |- 23194 23116 23116 23116 (java) 85 29 2522521600 16852 /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000006/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000006 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000002_1 6 |- 23116 23115 23116 23116 (bash) 0 1 14622720 710 /bin/bash -c /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000006/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000006 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000002_1 6 1>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000006/stdout 2>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000006/stderr [2019-06-11 16:05:50.970]Container killed on request. Exit code is 143 [2019-06-11 16:05:51.012]Container exited with a non-zero exit code 143. 2019-06-11 16:05:55,561 INFO mapreduce.Job: Task Id : attempt_1560238973821_0003_m_000000_1, Status : FAILED [2019-06-11 16:05:54.193]Container [pid=23396,containerID=container_1560238973821_0003_01_000009] is running 313866752B beyond the 'VIRTUAL' memory limit. Current usage: 111.1 MB of 1 GB physical memory used; 2.4 GB of 2.1 GB virtual memory used. Killing container. Dump of the process-tree for container_1560238973821_0003_01_000009 : |- PID PPID PGRPID SESSID CMD_NAME USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) RSSMEM_USAGE(PAGES) FULL_CMD_LINE |- 23396 23394 23396 23396 (bash) 0 1 14622720 710 /bin/bash -c /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000009/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000009 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000000_1 9 1>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000009/stdout 2>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000009/stderr |- 23473 23396 23396 23396 (java) 245 40 2554101760 27743 /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000009/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000009 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000000_1 9 [2019-06-11 16:05:54.228]Container killed on request. Exit code is 143 [2019-06-11 16:05:54.263]Container exited with a non-zero exit code 143. 2019-06-11 16:05:55,563 INFO mapreduce.Job: Task Id : attempt_1560238973821_0003_m_000003_1, Status : FAILED [2019-06-11 16:05:54.332]Container [pid=23304,containerID=container_1560238973821_0003_01_000008] is running 314042880B beyond the 'VIRTUAL' memory limit. Current usage: 113.8 MB of 1 GB physical memory used; 2.4 GB of 2.1 GB virtual memory used. Killing container. Dump of the process-tree for container_1560238973821_0003_01_000008 : |- PID PPID PGRPID SESSID CMD_NAME USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) RSSMEM_USAGE(PAGES) FULL_CMD_LINE |- 23381 23304 23304 23304 (java) 265 51 2554277888 28423 /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000008/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000008 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000003_1 8 |- 23304 23302 23304 23304 (bash) 0 1 14622720 720 /bin/bash -c /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000008/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000008 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000003_1 8 1>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000008/stdout 2>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000008/stderr [2019-06-11 16:05:54.353]Container killed on request. Exit code is 143 [2019-06-11 16:05:54.381]Container exited with a non-zero exit code 143. 2019-06-11 16:05:55,565 INFO mapreduce.Job: Task Id : attempt_1560238973821_0003_m_000001_1, Status : FAILED [2019-06-11 16:05:54.408]Container [pid=23200,containerID=container_1560238973821_0003_01_000007] is running 314497536B beyond the 'VIRTUAL' memory limit. Current usage: 115.5 MB of 1 GB physical memory used; 2.4 GB of 2.1 GB virtual memory used. Killing container. Dump of the process-tree for container_1560238973821_0003_01_000007 : |- PID PPID PGRPID SESSID CMD_NAME USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) RSSMEM_USAGE(PAGES) FULL_CMD_LINE |- 23200 23198 23200 23200 (bash) 0 1 14622720 711 /bin/bash -c /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000007/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000007 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000001_1 7 1>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000007/stdout 2>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000007/stderr |- 23277 23200 23200 23200 (java) 257 60 2554732544 28852 /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000007/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000007 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000001_1 7 [2019-06-11 16:05:54.463]Container killed on request. Exit code is 143 [2019-06-11 16:05:54.482]Container exited with a non-zero exit code 143. 2019-06-11 16:06:01,619 INFO mapreduce.Job: Task Id : attempt_1560238973821_0003_m_000002_2, Status : FAILED [2019-06-11 16:06:00.584]Container [pid=23515,containerID=container_1560238973821_0003_01_000011] is running 337451520B beyond the 'VIRTUAL' memory limit. Current usage: 203.4 MB of 1 GB physical memory used; 2.4 GB of 2.1 GB virtual memory used. Killing container. Dump of the process-tree for container_1560238973821_0003_01_000011 : |- PID PPID PGRPID SESSID CMD_NAME USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) RSSMEM_USAGE(PAGES) FULL_CMD_LINE |- 23515 23513 23515 23515 (bash) 0 1 14622720 712 /bin/bash -c /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000011/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000011 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000002_2 11 1>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000011/stdout 2>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000011/stderr |- 23592 23515 23515 23515 (java) 456 89 2577686528 51352 /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000011/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000011 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000002_2 11 [2019-06-11 16:06:00.602]Container killed on request. Exit code is 143 [2019-06-11 16:06:00.659]Container exited with a non-zero exit code 143. 2019-06-11 16:06:05,651 INFO mapreduce.Job: Task Id : attempt_1560238973821_0003_m_000000_2, Status : FAILED [2019-06-11 16:06:03.816]Container [pid=23651,containerID=container_1560238973821_0003_01_000012] is running 331475456B beyond the 'VIRTUAL' memory limit. Current usage: 173.7 MB of 1 GB physical memory used; 2.4 GB of 2.1 GB virtual memory used. Killing container. Dump of the process-tree for container_1560238973821_0003_01_000012 : |- PID PPID PGRPID SESSID CMD_NAME USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) RSSMEM_USAGE(PAGES) FULL_CMD_LINE |- 23728 23651 23651 23651 (java) 418 39 2571710464 43768 /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000012/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000012 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000000_2 12 |- 23651 23649 23651 23651 (bash) 0 1 14622720 707 /bin/bash -c /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000012/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000012 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000000_2 12 1>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000012/stdout 2>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000012/stderr [2019-06-11 16:06:03.981]Container killed on request. Exit code is 143 [2019-06-11 16:06:03.986]Container exited with a non-zero exit code 143. 2019-06-11 16:06:08,677 INFO mapreduce.Job: Task Id : attempt_1560238973821_0003_m_000001_2, Status : FAILED [2019-06-11 16:06:07.127]Container [pid=23848,containerID=container_1560238973821_0003_01_000014] is running 335940096B beyond the 'VIRTUAL' memory limit. Current usage: 198.2 MB of 1 GB physical memory used; 2.4 GB of 2.1 GB virtual memory used. Killing container. Dump of the process-tree for container_1560238973821_0003_01_000014 : |- PID PPID PGRPID SESSID CMD_NAME USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) RSSMEM_USAGE(PAGES) FULL_CMD_LINE |- 23848 23847 23848 23848 (bash) 0 1 14622720 714 /bin/bash -c /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000014/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000014 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000001_2 14 1>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000014/stdout 2>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000014/stderr |- 23926 23848 23848 23848 (java) 408 59 2576175104 50032 /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000014/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000014 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000001_2 14 [2019-06-11 16:06:07.186]Container killed on request. Exit code is 143 [2019-06-11 16:06:07.201]Container exited with a non-zero exit code 143. 2019-06-11 16:06:08,678 INFO mapreduce.Job: Task Id : attempt_1560238973821_0003_m_000003_2, Status : FAILED [2019-06-11 16:06:07.227]Container [pid=23751,containerID=container_1560238973821_0003_01_000013] is running 337357312B beyond the 'VIRTUAL' memory limit. Current usage: 192.7 MB of 1 GB physical memory used; 2.4 GB of 2.1 GB virtual memory used. Killing container. Dump of the process-tree for container_1560238973821_0003_01_000013 : |- PID PPID PGRPID SESSID CMD_NAME USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) RSSMEM_USAGE(PAGES) FULL_CMD_LINE |- 23829 23751 23751 23751 (java) 463 52 2577592320 48632 /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000013/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000013 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000003_2 13 |- 23751 23749 23751 23751 (bash) 0 1 14622720 706 /bin/bash -c /opt/java/jdk1.8.0_181/bin/java -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx820m -Djava.io.tmpdir=/bigdata/hadoop-3.1.1/tmp/nm-local-dir/usercache/liuyanbing/appcache/application_1560238973821_0003/container_1560238973821_0003_01_000013/tmp -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000013 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog org.apache.hadoop.mapred.YarnChild 127.0.1.1 43655 attempt_1560238973821_0003_m_000003_2 13 1>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000013/stdout 2>/bigdata/hadoop-3.1.1/logs/userlogs/application_1560238973821_0003/container_1560238973821_0003_01_000013/stderr [2019-06-11 16:06:07.280]Container killed on request. Exit code is 143 [2019-06-11 16:06:07.360]Container exited with a non-zero exit code 143. 2019-06-11 16:06:12,703 INFO mapreduce.Job: map 100% reduce 0% 2019-06-11 16:06:12,711 INFO mapreduce.Job: Job job_1560238973821_0003 failed with state FAILED due to: Task failed task_1560238973821_0003_m_000002 Job failed as tasks failed. failedMaps:1 failedReduces:0 killedMaps:0 killedReduces: 0 2019-06-11 16:06:12,979 INFO mapreduce.Job: Counters: 13 Job Counters Failed map tasks=13 Killed map tasks=3 Launched map tasks=16 Other local map tasks=12 Data-local map tasks=4 Total time spent by all maps in occupied slots (ms)=124936 Total time spent by all reduces in occupied slots (ms)=0 Total time spent by all map tasks (ms)=124936 Total vcore-milliseconds taken by all map tasks=124936 Total megabyte-milliseconds taken by all map tasks=127934464 Map-Reduce Framework CPU time spent (ms)=0 Physical memory (bytes) snapshot=0 Virtual memory (bytes) snapshot=0 2019-06-11 16:06:12,986 WARN mapreduce.Counters: Group FileSystemCounters is deprecated. Use org.apache.hadoop.mapreduce.FileSystemCounter instead 2019-06-11 16:06:12,990 INFO mapreduce.ExportJobBase: Transferred 0 bytes in 61.7517 seconds (0 bytes/sec) 2019-06-11 16:06:12,999 INFO mapreduce.ExportJobBase: Exported 0 records. 2019-06-11 16:06:12,999 ERROR tool.ExportTool: Error during export: Export job failed! 新手找不到错误 求大神班帮我看看
Hadoop自定义分组和多ReductTask出现异常
我现在有三个节点 程序在windows下编写,并将Job提交到了集群的Yarn上去执行,出现异常.但是在Linux下使用Hadoop jar 执行是可以的.之前在执行WordCount和其他小程序时候, 并没有出错,我认为错误原因在于这个ReductTask.请大牛指导一下.万分感谢.. ``` 2015-12-04 15:33:43,100 INFO [main] client.RMProxy (RMProxy.java:createRMProxy(92)) - Connecting to ResourceManager at hadoop01/10.5.110.250:8032 2015-12-04 15:33:43,458 WARN [main] mapreduce.JobSubmitter (JobSubmitter.java:copyAndConfigureFiles(150)) - Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2015-12-04 15:33:43,478 WARN [main] mapreduce.JobSubmitter (JobSubmitter.java:copyAndConfigureFiles(259)) - No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2015-12-04 15:33:43,525 INFO [main] input.FileInputFormat (FileInputFormat.java:listStatus(280)) - Total input paths to process : 1 2015-12-04 15:33:43,573 INFO [main] mapreduce.JobSubmitter (JobSubmitter.java:submitJobInternal(396)) - number of splits:1 2015-12-04 15:33:43,655 INFO [main] mapreduce.JobSubmitter (JobSubmitter.java:printTokens(479)) - Submitting tokens for job: job_1449213919153_0002 2015-12-04 15:33:43,744 INFO [main] mapred.YARNRunner (YARNRunner.java:createApplicationSubmissionContext(369)) - Job jar is not present. Not adding any jar to the list of resources. 2015-12-04 15:33:43,778 INFO [main] impl.YarnClientImpl (YarnClientImpl.java:submitApplication(204)) - Submitted application application_1449213919153_0002 2015-12-04 15:33:43,807 INFO [main] mapreduce.Job (Job.java:submit(1289)) - The url to track the job: http://hadoop01:8088/proxy/application_1449213919153_0002/ 2015-12-04 15:33:43,808 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1334)) - Running job: job_1449213919153_0002 2015-12-04 15:33:46,823 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1355)) - Job job_1449213919153_0002 running in uber mode : false 2015-12-04 15:33:46,825 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1362)) - map 0% reduce 0% 2015-12-04 15:33:46,833 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1375)) - Job job_1449213919153_0002 failed with state FAILED due to: Application application_1449213919153_0002 failed 2 times due to AM Container for appattempt_1449213919153_0002_000002 exited with exitCode: -1000 due to: File file:/tmp/hadoop-yarn/staging/lixiwei/.staging/job_1449213919153_0002/job.splitmetainfo does not exist .Failing this attempt.. Failing the application. 2015-12-04 15:33:46,861 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1380)) - Counters: 0 ``` 程序如下: ``` public class FlowSumArea { public static class FlowSumAreaMapper extends Mapper<LongWritable, Text, Text, FlowBean> { @Override protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, FlowBean>.Context context) throws IOException, InterruptedException { String line = value.toString(); String[] fields = StringUtils.split(line, "\t"); String phoneNo = fields[1]; long upFlow = Long.parseLong(fields[7]); long downFLow = Long.parseLong(fields[8]); context.write(new Text(phoneNo), new FlowBean(phoneNo, upFlow, downFLow)); } } public static class FlowSumAreaReducer extends Reducer<Text, FlowBean, Text, FlowBean> { @Override protected void reduce(Text key, Iterable<FlowBean> values, Reducer<Text, FlowBean, Text, FlowBean>.Context context) throws IOException, InterruptedException { long upFlowCounter = 0; long downFlowCounter = 0; for (FlowBean bean : values) { upFlowCounter += bean.getUpFlow(); downFlowCounter += bean.getDownFlow(); } context.write(key, new FlowBean(key.toString(), upFlowCounter, downFlowCounter)); } } public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException { // 1.获取配置文件 Configuration conf = new Configuration(); // 2.设置Job Job job = Job.getInstance(); job.setJarByClass(FlowSumArea.class); job.setMapperClass(FlowSumAreaMapper.class); job.setReducerClass(FlowSumAreaReducer.class); job.setPartitionerClass(AreaPartitioner.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(FlowBean.class); // 设置Reduce的任务并发数,应该跟分组的数量保持一致 job.setNumReduceTasks(6); // 3.设置输入输出路径 FileInputFormat.setInputPaths(job, new Path("C:\\Users\\51195\\Desktop\\flow\\flowarea\\srcdata")); FileOutputFormat.setOutputPath(job, new Path("C:\\Users\\51195\\Desktop\\flow\\flowarea\\outputdata6")); // FileInputFormat.setInputPaths(job, new Path(args[0])); // FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true)?0:1); } } ``` 这个是分组程序 ``` public class AreaPartitioner<KEY, VALUE> extends Partitioner<KEY, VALUE>{ private static HashMap<String,Integer> areaMap = new HashMap<>(); static{ areaMap.put("135", 0); areaMap.put("136", 1); areaMap.put("137", 2); areaMap.put("138", 3); areaMap.put("139", 4); } @Override public int getPartition(KEY key, VALUE value, int numPartitions) { //从key中拿到手机号,查询手机归属地字典,不同的省份返回不同的组号 int areaCoder = areaMap.get(key.toString().substring(0, 3))==null?5:areaMap.get(key.toString().substring(0, 3)); return areaCoder; } } ```
新手,hadoop上运行wordcount程序报错
运行的环境是:Ubuntu14.04+hadoop2.6.1 用的是virtualBox虚拟机,然后安装了一个master和三个slave节点 hadoop是可以成功启动的,没有任何问题 在Ubuntu安装了eclipse,用java写了word count的程序,源码如下: ``` package wordcount; import java.io.IOException; import java.util.StringTokenizer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; /** * @author * @version 创建时间:2017年9月9日 上午8:50:51 类说明 */ public class Wordcount { public static class TokenizerMapper extends Mapper<LongWritable, Text, Text, IntWritable> { private final static IntWritable one = new IntWritable(1); private Text word = new Text(); protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, IntWritable>.Context context) throws IOException, InterruptedException { StringTokenizer line = new StringTokenizer(value.toString()); while (line.hasMoreTokens()) { word.set(line.nextToken()); context.write(word, one); } } } public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> { private IntWritable result = new IntWritable(); protected void reduce(Text key, Iterable<IntWritable> values, Reducer<Text, IntWritable, Text, IntWritable>.Context context) throws IOException, InterruptedException { int sum = 0; for (IntWritable obj : values) { sum += obj.get(); } result.set(sum); context.write(key, result); } } public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "word count"); job.setJarByClass(Wordcount.class); job.setMapperClass(TokenizerMapper.class); job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path("hdfs://master:9000/user/hduser/demo/test.txt")); FileOutputFormat.setOutputPath(job, new Path("hdfs://master:9000/user/hduser/demo/wordcount")); //FileInputFormat.addInputPath(job, new Path(args[0])); //FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); } } ``` 启动hadoop后,在eclipse中直接运行上面的程序,运行成功,生成了wordcount文件夹,里面有_SUCCESS文件,也有统计的结果文件 然后我想把程序打包成jar文件来运行,先把上面程序中的: ``` FileInputFormat.addInputPath(job, new Path("hdfs://master:9000/user/hduser/demo/test.txt")); FileOutputFormat.setOutputPath(job, new Path("hdfs://master:9000/user/hduser/demo/wordcount")); ``` 改成如下: ``` FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); ``` 就是通过终端输入这两个参数 用eclipse的export打包成jar文件,后在终端输入: ``` hadoop jar wordcount.jar wordcount.Wordcount hdfs://master:9000/user/hduser/demo/test.txt hdfs://master:9000/user/hduser/demo/wordcount ``` 运行就报错了,报错情况如下: ``` 17/09/09 11:18:53 INFO client.RMProxy: Connecting to ResourceManager at master/192.168.56.100:8050 17/09/09 11:18:54 WARN mapreduce.JobResourceUploader: Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 17/09/09 11:18:55 INFO input.FileInputFormat: Total input paths to process : 1 17/09/09 11:18:55 INFO mapreduce.JobSubmitter: number of splits:1 17/09/09 11:18:55 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1504926710828_0001 17/09/09 11:18:56 INFO impl.YarnClientImpl: Submitted application application_1504926710828_0001 17/09/09 11:18:56 INFO mapreduce.Job: The url to track the job: http://master:8088/proxy/application_1504926710828_0001/ 17/09/09 11:18:56 INFO mapreduce.Job: Running job: job_1504926710828_0001 17/09/09 11:19:14 INFO mapreduce.Job: Job job_1504926710828_0001 running in uber mode : false 17/09/09 11:19:14 INFO mapreduce.Job: map 0% reduce 0% 17/09/09 11:19:14 INFO mapreduce.Job: Job job_1504926710828_0001 failed with state FAILED due to: Application application_1504926710828_0001 failed 2 times due to AM Container for appattempt_1504926710828_0001_000002 exited with exitCode: 1 For more detailed output, check application tracking page:http://master:8088/proxy/application_1504926710828_0001/Then, click on links to logs of each attempt. Diagnostics: Exception from container-launch. Container id: container_1504926710828_0001_02_000001 Exit code: 1 Stack trace: ExitCodeException exitCode=1: at org.apache.hadoop.util.Shell.runCommand(Shell.java:538) at org.apache.hadoop.util.Shell.run(Shell.java:455) at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:715) at org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.launchContainer(DefaultContainerExecutor.java:211) at org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:302) at org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:82) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:622) at java.lang.Thread.run(Thread.java:748) Container exited with a non-zero exit code 1 Failing this attempt. Failing the application. 17/09/09 11:19:14 INFO mapreduce.Job: Counters: 0 ``` 去查了下日志文件, ``` 2017-09-09 11:18:55,869 INFO org.apache.hadoop.hdfs.StateChange: DIR* completeFile: /tmp/hadoop-yarn/staging/hduser/.staging/job_1504926710828_0001/job.xml is closed by DFSClient_NONMAPREDUCE_-1306163227_1 2017-09-09 11:18:59,502 INFO org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor: Rescanning after 30000 milliseconds 2017-09-09 11:18:59,503 INFO org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor: Scanned 0 directive(s) and 0 block(s) in 1 millisecond(s). 2017-09-09 11:19:12,241 INFO org.apache.hadoop.ipc.Server: IPC Server handler 2 on 9000, call org.apache.hadoop.hdfs.protocol.ClientProtocol.getBlockLocations from 192.168.56.102:53610 Call#7 Retry#0: java.io.FileNotFoundException: File does not exist: /tmp/hadoop-yarn/staging/hduser/.staging/job_1504926710828_0001/job_1504926710828_0001_1.jhist 2017-09-09 11:19:12,293 INFO org.apache.hadoop.ipc.Server: IPC Server handler 4 on 9000, call org.apache.hadoop.hdfs.protocol.ClientProtocol.getBlockLocations from 192.168.56.102:53610 Call#8 Retry#0: java.io.FileNotFoundException: File does not exist: /tmp/hadoop-yarn/staging/hduser/.staging/job_1504926710828_0001/job_1504926710828_0001_1.jhist 2017-09-09 11:19:29,502 INFO org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor: Rescanning after 30000 milliseconds 2017-09-09 11:19:29,502 INFO org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor: Scanned 0 directive(s) and 0 block(s) in 0 millisecond(s). 2017-09-09 11:19:42,634 INFO org.apache.hadoop.hdfs.server.namenode.FSNamesystem: Roll Edit Log from 192.168.56.100 2017-09-09 11:19:42,634 INFO org.apache.hadoop.hdfs.server.namenode.FSEditLog: Rolling edit logs 2017-09-09 11:19:42,634 INFO org.apache.hadoop.hdfs.server.namenode.FSEditLog: Ending log segment 29 2017-09-09 11:19:42,635 INFO org.apache.hadoop.hdfs.server.namenode.FSEditLog: Number of transactions: 40 Total time for transactions(ms): 6 Number of transactions batched in Syncs: 0 Number of syncs: 27 SyncTimes(ms): 545 2017-09-09 11:19:42,704 INFO org.apache.hadoop.hdfs.server.namenode.FSEditLog: Number of transactions: 40 Total time for transactions(ms): 6 Number of transactions batched in Syncs: 0 Number of syncs: 28 SyncTimes(ms): 613 2017-09-09 11:19:42,704 INFO org.apache.hadoop.hdfs.server.namenode.FileJournalManager: Finalizing edits file /usr/local/hadoop/hadoop_data/hdfs/namenode/current/edits_inprogress_0000000000000000029 -> /usr/local/hadoop/hadoop_data/hdfs/namenode/current/edits_0000000000000000029-0000000000000000068 2017-09-09 11:19:42,704 INFO org.apache.hadoop.hdfs.server.namenode.FSEditLog: Starting log segment at 69 2017-09-09 11:19:59,503 INFO org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor: Rescanning after 30001 milliseconds 2017-09-09 11:19:59,503 INFO org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor: Scanned 0 directive(s) and 0 block(s) in 0 millisecond(s). 2017-09-09 11:20:29,504 INFO org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor: Rescanning after 30001 milliseconds 2017-09-09 11:20:29,504 INFO org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor: Scanned 0 directive(s) and 0 block(s) in 0 millisecond(s). 2017-09-09 11:20:42,759 INFO org.apache.hadoop.hdfs.server.namenode.FSNamesystem: Roll Edit Log from 192.168.56.100 2017-09-09 11:20:42,759 INFO org.apache.hadoop.hdfs.server.namenode.FSEditLog: Rolling edit logs 2017-09-09 11:20:42,759 INFO org.apache.hadoop.hdfs.server.namenode.FSEditLog: Ending log segment 69 2017-09-09 11:20:42,759 INFO org.apache.hadoop.hdfs.server.namenode.FSEditLog: Number of transactions: 2 Total time for transactions(ms): 0 Number of transactions batched in Syncs: 0 Number of syncs: 2 SyncTimes(ms): 24 2017-09-09 11:20:42,791 INFO org.apache.hadoop.hdfs.server.namenode.FSEditLog: Number of transactions: 2 Total time for transactions(ms): 0 Number of transactions batched in Syncs: 0 Number of syncs: 3 SyncTimes(ms): 56 ``` 在这里面报了一个错误: ``` java.io.FileNotFoundException: File does not exist: /tmp/hadoop-yarn/staging/hduser/.staging/job_1504926710828_0001/job_1504926710828_0001_1.jhist ``` 新手,不知道怎么办了
MapReducer 写入到数据库 报错
## 【 DBUserWritable 类 】 package org.neworigin.com.Database; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.lib.db.DBWritable; public class DBUserWritable implements DBWritable,WritableComparable{ private String name=""; private String sex=""; private int age=0; private int num=0; private String department=""; private String tables=""; @Override public String toString() { return "DBUserWritable [name=" + name + ", sex=" + sex + ", age=" + age + ", department=" + department + "]"; } public DBUserWritable(DBUserWritable d){ this.name=d.getName(); this.sex=d.getSex(); this.age=d.getAge(); this.num=d.getNum(); this.department=d.getDepartment(); this.tables=d.getTables(); } public String getName() { return name; } public void setName(String name) { this.name = name; } public String getSex() { return sex; } public void setSex(String sex) { this.sex = sex; } public int getAge() { return age; } public void setAge(int age) { this.age = age; } public int getNum() { return num; } public void setNum(int num) { this.num = num; } public String getDepartment() { return department; } public void setDepartment(String department) { this.department = department; } public String getTables() { return tables; } public void setTables(String tables) { this.tables = tables; } public DBUserWritable(String name, String sex, int age, int num, String department, String tables) { super(); this.name = name; this.sex = sex; this.age = age; this.num = num; this.department = department; this.tables = tables; } public DBUserWritable() { super(); // TODO Auto-generated constructor stub } public void write(DataOutput out) throws IOException { // TODO Auto-generated method stub out.writeUTF(name); out.writeUTF(sex); out.writeInt(age); out.writeInt(num); out.writeUTF(department); out.writeUTF(tables); } public void readFields(DataInput in) throws IOException { // TODO Auto-generated method stub name = in.readUTF(); sex=in.readUTF(); age=in.readInt(); num=in.readInt(); department=in.readUTF(); tables=in.readUTF(); } public int compareTo(Object o) { // TODO Auto-generated method stub return 0; } public void write(PreparedStatement statement) throws SQLException { // TODO Auto-generated method stub statement.setString(1, this.getName()); statement.setString(2, this.getSex()); statement.setInt(3, this.getAge()); statement.setString(4, this.getDepartment()); } public void readFields(ResultSet resultSet) throws SQLException { // TODO Auto-generated method stub this.name=resultSet.getString(1); this.sex=resultSet.getString(2); this.age=resultSet.getInt(3); this.department=resultSet.getString(4); } } ## 【mapper】 package org.neworigin.com.Database; import java.io.IOException; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Mapper; public class UserDBMapper extends Mapper<LongWritable, Text, Text, DBUserWritable> { DBUserWritable DBuser= new DBUserWritable(); @Override protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, DBUserWritable>.Context context) throws IOException, InterruptedException { String[] values=value.toString().split(" "); if(values.length==4){ DBuser.setName(values[0]); DBuser.setSex(values[1]); DBuser.setAge(Integer.parseInt(values[2])); DBuser.setNum(Integer.parseInt(values[3])); DBuser.setTables("t1"); System.out.println("mapper---t1---------------"+DBuser); context.write(new Text(values[3]),DBuser); } if(values.length==2){ DBuser.setNum(Integer.parseInt(values[0])); DBuser.setDepartment(values[1]); DBuser.setTables("t2"); context.write(new Text(values[0]),DBuser); //System.out.println("mapper --t2"+"--"+values[0]+"----"+DBuser); } } } ## 【reducer 】 package org.neworigin.com.Database; import java.io.IOException; import java.util.LinkedList; import java.util.List; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Reducer; public class UserDBReducer extends Reducer<Text, DBUserWritable,NullWritable,DBUserWritable> { // public DBUserWritable db= new DBUserWritable(); @Override protected void reduce(Text k2, Iterable<DBUserWritable> v2, Reducer<Text, DBUserWritable, NullWritable,DBUserWritable>.Context context) throws IOException, InterruptedException { String Name=""; List<DBUserWritable> list=new LinkedList<DBUserWritable>(); for(DBUserWritable val : v2){ list.add(new DBUserWritable(val));//new 一个对象 给list // System.out.println("[table]"+val.getTables()+"----key"+k2+"---"+val); if(val.getTables().equals("t2")){ Name=val.getDepartment(); } } //键是 num for(DBUserWritable join : list){ System.out.println("[table]"+join.getTables()+"----key"+k2+"---"+join); if(join.getTables().equals("t1")){ join.setDepartment(Name); System.out.println("db-----"+join); context.write(NullWritable.get(), join); } } } } ## 【app】 package org.neworigin.com.Database; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.db.DBConfiguration; import org.apache.hadoop.mapreduce.lib.db.DBOutputFormat; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; public class UserDBAPP { public static void main(String[] args) throws Exception, URISyntaxException { // TODO Auto-generated method stub String INPUT_PATH="file:///E:/BigData_eclipse_database/Database/data/table1"; String INPUT_PATH1="file:///E:/BigData_eclipse_database/Database/data/table2"; // String OUTPUT_PARH="file:///E:/BigData_eclipse_database/Database/data/output"; Configuration conf = new Configuration(); // FileSystem fs=FileSystem.get(new URI(OUTPUT_PARH),conf); // if(fs.exists(new Path(OUTPUT_PARH))){ // fs.delete(new Path(OUTPUT_PARH)); // } Job job = new Job(conf,"mydb"); //设置数据库配置 DBConfiguration.configureDB(conf, "com.mysql.jdbc.Driver", "jdbc:mysql://localhost/hadoop", "root", "123456"); FileInputFormat.addInputPaths(job,INPUT_PATH); FileInputFormat.addInputPaths(job,INPUT_PATH1); job.setMapperClass(UserDBMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(DBUserWritable.class); job.setReducerClass(UserDBReducer.class); job.setOutputKeyClass(NullWritable.class); job.setOutputValueClass(DBUserWritable.class); // FileOutputFormat.setOutputPath(job, new Path(OUTPUT_PARH)); //设置输出路径 DBOutputFormat.setOutput(job,"user_tables", "name","sex","age","department"); job.setOutputFormatClass(DBOutputFormat.class); boolean re = job.waitForCompletion(true); System.out.println(re); } } 【报错】ps 表链接 ,写到本地没问题 写到数据库 就报错; 17/11/10 11:39:11 WARN output.FileOutputCommitter: Output Path is null in cleanupJob() 17/11/10 11:39:11 WARN mapred.LocalJobRunner: job_local1812680657_0001 java.lang.Exception: java.io.IOException at org.apache.hadoop.mapred.LocalJobRunner$Job.runTasks(LocalJobRunner.java:462) at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:529) Caused by: java.io.IOException at org.apache.hadoop.mapreduce.lib.db.DBOutputFormat.getRecordWriter(DBOutputFormat.java:185) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.<init>(ReduceTask.java:541) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:614) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:319) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 17/11/10 11:39:12 INFO mapreduce.Job: Job job_local1812680657_0001 running in uber mode : false 17/11/10 11:39:12 INFO mapreduce.Job: map 100% reduce 0% 17/11/10 11:39:12 INFO mapreduce.Job: Job job_local1812680657_0001 failed with state FAILED due to: NA 17/11/10 11:39:12 INFO mapreduce.Job: Counters: 35
爬虫福利二 之 妹子图网MM批量下载
爬虫福利一:27报网MM批量下载    点击 看了本文,相信大家对爬虫一定会产生强烈的兴趣,激励自己去学习爬虫,在这里提前祝:大家学有所成! 目标网站:妹子图网 环境:Python3.x 相关第三方模块:requests、beautifulsoup4 Re:各位在测试时只需要将代码里的变量 path 指定为你当前系统要保存的路径,使用 python xxx.py 或IDE运行即可。
字节跳动视频编解码面经
三四月份投了字节跳动的实习(图形图像岗位),然后hr打电话过来问了一下会不会opengl,c++,shador,当时只会一点c++,其他两个都不会,也就直接被拒了。 七月初内推了字节跳动的提前批,因为内推没有具体的岗位,hr又打电话问要不要考虑一下图形图像岗,我说实习投过这个岗位不合适,不会opengl和shador,然后hr就说秋招更看重基础。我当时想着能进去就不错了,管他哪个岗呢,就同意了面试...
Java学习的正确打开方式
在博主认为,对于入门级学习java的最佳学习方法莫过于视频+博客+书籍+总结,前三者博主将淋漓尽致地挥毫于这篇博客文章中,至于总结在于个人,实际上越到后面你会发现学习的最好方式就是阅读参考官方文档其次就是国内的书籍,博客次之,这又是一个层次了,这里暂时不提后面再谈。博主将为各位入门java保驾护航,各位只管冲鸭!!!上天是公平的,只要不辜负时间,时间自然不会辜负你。 何谓学习?博主所理解的学习,它是一个过程,是一个不断累积、不断沉淀、不断总结、善于传达自己的个人见解以及乐于分享的过程。
程序员必须掌握的核心算法有哪些?
由于我之前一直强调数据结构以及算法学习的重要性,所以就有一些读者经常问我,数据结构与算法应该要学习到哪个程度呢?,说实话,这个问题我不知道要怎么回答你,主要取决于你想学习到哪些程度,不过针对这个问题,我稍微总结一下我学过的算法知识点,以及我觉得值得学习的算法。这些算法与数据结构的学习大多数是零散的,并没有一本把他们全部覆盖的书籍。下面是我觉得值得学习的一些算法以及数据结构,当然,我也会整理一些看过
linux系列之常用运维命令整理笔录
本博客记录工作中需要的linux运维命令,大学时候开始接触linux,会一些基本操作,可是都没有整理起来,加上是做开发,不做运维,有些命令忘记了,所以现在整理成博客,当然vi,文件操作等就不介绍了,慢慢积累一些其它拓展的命令,博客不定时更新 free -m 其中:m表示兆,也可以用g,注意都要小写 Men:表示物理内存统计 total:表示物理内存总数(total=used+free) use...
Python 基础(一):入门必备知识
目录1 标识符2 关键字3 引号4 编码5 输入输出6 缩进7 多行8 注释9 数据类型10 运算符10.1 常用运算符10.2 运算符优先级 1 标识符 标识符是编程时使用的名字,用于给变量、函数、语句块等命名,Python 中标识符由字母、数字、下划线组成,不能以数字开头,区分大小写。 以下划线开头的标识符有特殊含义,单下划线开头的标识符,如:_xxx ,表示不能直接访问的类属性,需通过类提供
兼职程序员一般可以从什么平台接私活?
这个问题我进行了系统性的总结,以下将进行言简意赅的说明和渠道提供,希望对各位小猿/小媛们有帮助~ 根据我们的经验,程序员兼职主要分为三种:兼职职位众包、项目整包和自由职业者驻场。 所谓的兼职职位众包,指的是需求方这边有自有工程师配合,只需要某个职位的工程师开发某个模块的项目。比如开发一个 app,后端接口有人开发,但是缺少 iOS 前端开发工程师,那么他们就会发布一个职位招聘前端,来配合公司一...
程序员接私活怎样防止做完了不给钱?
首先跟大家说明一点,我们做 IT 类的外包开发,是非标品开发,所以很有可能在开发过程中会有这样那样的需求修改,而这种需求修改很容易造成扯皮,进而影响到费用支付,甚至出现做完了项目收不到钱的情况。 那么,怎么保证自己的薪酬安全呢? 我们在开工前,一定要做好一些证据方面的准备(也就是“讨薪”的理论依据),这其中最重要的就是需求文档和验收标准。一定要让需求方提供这两个文档资料作为开发的基础。之后开发
Python十大装B语法
Python 是一种代表简单思想的语言,其语法相对简单,很容易上手。不过,如果就此小视 Python 语法的精妙和深邃,那就大错特错了。本文精心筛选了最能展现 Python 语法之精妙的十个知识点,并附上详细的实例代码。如能在实战中融会贯通、灵活使用,必将使代码更为精炼、高效,同时也会极大提升代码B格,使之看上去更老练,读起来更优雅。 1. for - else 什么?不是 if 和 else 才
数据库优化 - SQL优化
前面一篇文章从实例的角度进行数据库优化,通过配置一些参数让数据库性能达到最优。但是一些“不好”的SQL也会导致数据库查询变慢,影响业务流程。本文从SQL角度进行数据库优化,提升SQL运行效率。 判断问题SQL 判断SQL是否有问题时可以通过两个表象进行判断: 系统级别表象 CPU消耗严重 IO等待严重 页面响应时间过长
2019年11月中国大陆编程语言排行榜
2019年11月2日,我统计了某招聘网站,获得有效程序员招聘数据9万条。针对招聘信息,提取编程语言关键字,并统计如下: 编程语言比例 rank pl_ percentage 1 java 33.62% 2 c/c++ 16.42% 3 c_sharp 12.82% 4 javascript 12.31% 5 python 7.93% 6 go 7.25% 7
写了很久,这是一份最适合/贴切普通大众/科班/非科班的『学习路线』
说实话,对于学习路线这种文章我一般是不写的,大家看我的文章也知道,我是很少写建议别人怎么样怎么样的文章,更多的是,写自己的真实经历,然后供大家去参考,这样子,我内心也比较踏实,也不怕误导他人。 但是,最近好多人问我学习路线,而且很多大一大二的,说自己很迷茫,看到我那篇 普普通通,我的三年大学 之后很受激励,觉得自己也能行,(是的,别太浪,你一定能行)希望我能给他个学习路线,说
经典算法(5)杨辉三角
写在前面: 我是 扬帆向海,这个昵称来源于我的名字以及女朋友的名字。我热爱技术、热爱开源、热爱编程。技术是开源的、知识是共享的。 这博客是对自己学习的一点点总结及记录,如果您对 Java、算法 感兴趣,可以关注我的动态,我们一起学习。 用知识改变命运,让我们的家人过上更好的生活。 目录一、杨辉三角的介绍二、杨辉三角的算法思想三、代码实现1.第一种写法2.第二种写法 一、杨辉三角的介绍 百度
腾讯算法面试题:64匹马8个跑道需要多少轮才能选出最快的四匹?
昨天,有网友私信我,说去阿里面试,彻底的被打击到了。问了为什么网上大量使用ThreadLocal的源码都会加上private static?他被难住了,因为他从来都没有考虑过这个问题。无独有偶,今天笔者又发现有网友吐槽了一道腾讯的面试题,我们一起来看看。 腾讯算法面试题:64匹马8个跑道需要多少轮才能选出最快的四匹? 在互联网职场论坛,一名程序员发帖求助到。二面腾讯,其中一个算法题:64匹
面试官:你连RESTful都不知道我怎么敢要你?
面试官:了解RESTful吗? 我:听说过。 面试官:那什么是RESTful? 我:就是用起来很规范,挺好的 面试官:是RESTful挺好的,还是自我感觉挺好的 我:都挺好的。 面试官:… 把门关上。 我:… 要干嘛?先关上再说。 面试官:我说出去把门关上。 我:what ?,夺门而去 文章目录01 前言02 RESTful的来源03 RESTful6大原则1. C-S架构2. 无状态3.统一的接
为啥国人偏爱Mybatis,而老外喜欢Hibernate/JPA呢?
关于SQL和ORM的争论,永远都不会终止,我也一直在思考这个问题。昨天又跟群里的小伙伴进行了一番讨论,感触还是有一些,于是就有了今天这篇文。 声明:本文不会下关于Mybatis和JPA两个持久层框架哪个更好这样的结论。只是摆事实,讲道理,所以,请各位看官勿喷。 一、事件起因 关于Mybatis和JPA孰优孰劣的问题,争论已经很多年了。一直也没有结论,毕竟每个人的喜好和习惯是大不相同的。我也看
SQL-小白最佳入门sql查询一
一 说明 如果是初学者,建议去网上寻找安装Mysql的文章安装,以及使用navicat连接数据库,以后的示例基本是使用mysql数据库管理系统; 二 准备前提 需要建立一张学生表,列分别是id,名称,年龄,学生信息;本示例中文章篇幅原因SQL注释略; 建表语句: CREATE TABLE `student` ( `id` int(11) NOT NULL AUTO_INCREMENT, `
项目中的if else太多了,该怎么重构?
介绍 最近跟着公司的大佬开发了一款IM系统,类似QQ和微信哈,就是聊天软件。我们有一部分业务逻辑是这样的 if (msgType = "文本") { // dosomething } else if(msgType = "图片") { // doshomething } else if(msgType = "视频") { // doshomething } else { // dosho
致 Python 初学者
文章目录1. 前言2. 明确学习目标,不急于求成,不好高骛远3. 在开始学习 Python 之前,你需要做一些准备2.1 Python 的各种发行版2.2 安装 Python2.3 选择一款趁手的开发工具3. 习惯使用IDLE,这是学习python最好的方式4. 严格遵从编码规范5. 代码的运行、调试5. 模块管理5.1 同时安装了py2/py35.2 使用Anaconda,或者通过IDE来安装模
“狗屁不通文章生成器”登顶GitHub热榜,分分钟写出万字形式主义大作
一、垃圾文字生成器介绍 最近在浏览GitHub的时候,发现了这样一个骨骼清奇的雷人项目,而且热度还特别高。 项目中文名:狗屁不通文章生成器 项目英文名:BullshitGenerator 根据作者的介绍,他是偶尔需要一些中文文字用于GUI开发时测试文本渲染,因此开发了这个废话生成器。但由于生成的废话实在是太过富于哲理,所以最近已经被小伙伴们给玩坏了。 他的文风可能是这样的: 你发现,
程序员:我终于知道post和get的区别
IT界知名的程序员曾说:对于那些月薪三万以下,自称IT工程师的码农们,其实我们从来没有把他们归为我们IT工程师的队伍。他们虽然总是以IT工程师自居,但只是他们一厢情愿罢了。 此话一出,不知激起了多少(码农)程序员的愤怒,却又无可奈何,于是码农问程序员。 码农:你知道get和post请求到底有什么区别? 程序员:你看这篇就知道了。 码农:你月薪三万了? 程序员:嗯。 码农:你是怎么做到的? 程序员:
《程序人生》系列-这个程序员只用了20行代码就拿了冠军
你知道的越多,你不知道的越多 点赞再看,养成习惯GitHub上已经开源https://github.com/JavaFamily,有一线大厂面试点脑图,欢迎Star和完善 前言 这一期不算《吊打面试官》系列的,所有没前言我直接开始。 絮叨 本来应该是没有这期的,看过我上期的小伙伴应该是知道的嘛,双十一比较忙嘛,要值班又要去帮忙拍摄年会的视频素材,还得搞个程序员一天的Vlog,还要写BU
加快推动区块链技术和产业创新发展,2019可信区块链峰会在京召开
11月8日,由中国信息通信研究院、中国通信标准化协会、中国互联网协会、可信区块链推进计划联合主办,科技行者协办的2019可信区块链峰会将在北京悠唐皇冠假日酒店开幕。   区块链技术被认为是继蒸汽机、电力、互联网之后,下一代颠覆性的核心技术。如果说蒸汽机释放了人类的生产力,电力解决了人类基本的生活需求,互联网彻底改变了信息传递的方式,区块链作为构造信任的技术有重要的价值。   1...
程序员把地府后台管理系统做出来了,还有3.0版本!12月7号最新消息:已在开发中有github地址
第一幕:缘起 听说阎王爷要做个生死簿后台管理系统,我们派去了一个程序员…… 996程序员做的梦: 第一场:团队招募 为了应对地府管理危机,阎王打算找“人”开发一套地府后台管理系统,于是就在地府总经办群中发了项目需求。 话说还是中国电信的信号好,地府都是满格,哈哈!!! 经常会有外行朋友问:看某网站做的不错,功能也简单,你帮忙做一下? 而这次,面对这样的需求,这个程序员
网易云6亿用户音乐推荐算法
网易云音乐是音乐爱好者的集聚地,云音乐推荐系统致力于通过 AI 算法的落地,实现用户千人千面的个性化推荐,为用户带来不一样的听歌体验。 本次分享重点介绍 AI 算法在音乐推荐中的应用实践,以及在算法落地过程中遇到的挑战和解决方案。 将从如下两个部分展开: AI 算法在音乐推荐中的应用 音乐场景下的 AI 思考 从 2013 年 4 月正式上线至今,网易云音乐平台持续提供着:乐屏社区、UGC
8年经验面试官详解 Java 面试秘诀
    作者 | 胡书敏 责编 | 刘静 出品 | CSDN(ID:CSDNnews) 本人目前在一家知名外企担任架构师,而且最近八年来,在多家外企和互联网公司担任Java技术面试官,前后累计面试了有两三百位候选人。在本文里,就将结合本人的面试经验,针对Java初学者、Java初级开发和Java开发,给出若干准备简历和准备面试的建议。   Java程序员准备和投递简历的实
面试官如何考察你的思维方式?
1.两种思维方式在求职面试中,经常会考察这种问题:北京有多少量特斯拉汽车? 某胡同口的煎饼摊一年能卖出多少个煎饼? 深圳有多少个产品经理? 一辆公交车里能装下多少个乒乓球? 一
碎片化的时代,如何学习
今天周末,和大家聊聊学习这件事情。 在如今这个社会,我们的时间被各类 APP 撕的粉碎。 刷知乎、刷微博、刷朋友圈; 看论坛、看博客、看公号; 等等形形色色的信息和知识获取方式一个都不错过。 貌似学了很多,但是却感觉没什么用。 要解决上面这些问题,首先要分清楚一点,什么是信息,什么是知识。 那什么是信息呢? 你一切听到的、看到的,都是信息,比如微博上的明星出轨、微信中的表情大战、抖音上的...
so easy! 10行代码写个"狗屁不通"文章生成器
前几天,GitHub 有个开源项目特别火,只要输入标题就可以生成一篇长长的文章。 背后实现代码一定很复杂吧,里面一定有很多高深莫测的机器学习等复杂算法 不过,当我看了源代码之后 这程序不到50行 尽管我有多年的Python经验,但我竟然一时也没有看懂 当然啦,原作者也说了,这个代码也是在无聊中诞生的,平时撸码是不写中文变量名的, 中文...
知乎高赞:中国有什么拿得出手的开源软件产品?(整理自本人原创回答)
知乎高赞:中国有什么拿得出手的开源软件产品? 在知乎上,有个问题问“中国有什么拿得出手的开源软件产品(在 GitHub 等社区受欢迎度较好的)?” 事实上,还不少呢~ 本人于2019.7.6进行了较为全面的回答,对这些受欢迎的 Github 开源项目分类整理如下: 分布式计算、云平台相关工具类 1.SkyWalking,作者吴晟、刘浩杨 等等 仓库地址: apache/skywalking 更...
相关热词 如何提升c#开发能力 矩阵乘法c# c#调用谷歌浏览器 c# 去空格去转义符 c#用户登录窗体代码 c# 流 c# linux 可视化 c# mvc 返回图片 c# 像素空间 c# 日期 最后一天
立即提问