自己写的hadoop ，MapReduce程序不能并行

** 学习hadoop有一段时间了，在写hadoop 程序时，尽管是根据官方例子，套着模板写出的，但是不能达到真正意义上的并行，也就是说，各分机没有任务运行。 ------------------------------------------------------------------------ ** 运行环境如下： 操作系统： centOS6.3 32位， jdk1.7， hadoop-1.0.3， 1台master,3台worker。 为了具体说明问题，程序如下： package campus; import java.io.IOException; import java.net.URI; import java.util.Set; import java.util.TreeSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.GzipCodec; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.Reducer.Context; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.GenericOptionsParser; public class TestSmsMR { // map public static class TSmsMap extends Mapper<Object, Text, Text, Text> { private static Text keyWord = new Text(); //第一个结点 private static Text valueWord = new Text(); //第二个结点 public void map(Object key, Text value, Context context) { // value: tag: 0 1 u String line = value.toString(); String[] arr = line.split(" |\u0009|\\|"); // 通过 空格、 \t、 | 分割字符串 if ( !(arr[0].equals(arr[1])) ) { try { String tmpKey = arr[0]; String tmpValue = ""; for(int i = 1; i < arr.length; i ++){ tmpValue += arr[i] + " "; } keyWord.set(tmpKey); valueWord.set(tmpValue); // 数据是非对称的，这就需要使用一次 write context.write(keyWord, valueWord); // context.write(valueWord, keyWord); //添加这句的话，必须先看图文件,如果重复则不需要这一行 } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } // 这种方法可行 } } } //reduce write<Text,Set<Text>> public static class TSmsReduce extends Reducer<Text, Text, Text, Text> { private static Text keyStr = new Text(); private static Text valueStr = new Text(); public void reduce(Text key, Iterable<Text> values,Context context) { String writeKey = key.toString(); String writeValues = ""; for (Text val : values) { writeValues += val.toString() + "\t"; } keyStr.set(writeKey); valueStr.set(writeValues); // System.out.println("writeKey: " + writeKey + "\twriteValues: " + writeValues); try { context.write(keyStr, valueStr); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } public static void preExectue(String inputPath, String outputPath) throws Exception { Configuration conf = new Configuration(); // conf.setBoolean("mapred.compress.map.output", true); conf.setBoolean("mapred.output.compress", true); // conf.setIfUnset("mapred.map.output.compression.type", "BLOCK"); conf.setClass("mapred.map.output.compression.codec", GzipCodec.class, CompressionCodec.class); conf.addResource(new Path("/usr/hadoop/conf/core-site.xml")); conf.addResource(new Path("/usr/hadoop/conf/hdfs-site.xml")); // 如果 outputPath 存在，那么先删除 Path outPutPath = new Path(outputPath); FileSystem fs = FileSystem.get(URI.create(outputPath), conf); fs.delete(outPutPath); // 自己添加路径 String[] ars = new String[] { inputPath, outputPath }; String[] otherArgs = new GenericOptionsParser(conf, ars) .getRemainingArgs(); if (otherArgs.length != 2) { System.err.println("Usage: sotTest <in> <out>"); System.exit(2); } Job job = new Job(conf, "TestSmsMR"); job.setJarByClass(TestSmsMR.class); job.setMapperClass(TSmsMap.class); job.setReducerClass(TSmsReduce.class); // job.setNumReduceTasks(4); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); // FileInputFormat.addInputPath(job, new Path(otherArgs[0])); FileOutputFormat.setOutputPath(job, new Path(otherArgs[1])); if (job.waitForCompletion(true)) { System.out.println("The preprocess mapreduce has finished!"); } } //main函数测试都好着，为什么就不能并行呢 public static void main(String[] args) throws Exception { Long startTime = System.currentTimeMillis(); String srcPath = "campusSms"; String dstPath = "campusSmsLabelOut"; preExectue(srcPath,dstPath); Long runTime = System.currentTimeMillis() - startTime; System.out.println("run time: " + runTime); } } 还是觉得问题出在这个函数上： public static void preExectue(String inputPath, String outputPath) 运行前提是： 环境已搭建好，而且测试主机分机都能正常通信，且主机从机都起来了。希望解答时，能多考虑些编程方面的问题。 该程序运行起来，就是在主机上跑，MapReduce机制到分机并没有得到任务，运行的数据250M，按照hadoop默认块64M来算，也应该分为4块，3台分机应得到执行任务的，可是程序就是不能并行。 请有经验的hadoop学习实践者给予指导。 在集群上运行Pi值计算，都能看到并行。就是自己写的程序，不行！......

运行mapredurce出现Method threw 'java.lang.IllegalStateException' exception. Cannot evaluate org.apache.hadoop.mapreduce.Job.toString()

执行下述代码后在，创建job后会有上述异常，但是可以执行到最后，但是job没有提交上去执行，在历史里也看不到有执行记录求帮助新手o(╥﹏╥)o。 package MapReducer; import com.sun.org.apache.bcel.internal.generic.RETURN; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import java.io.File; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.StringTokenizer; /** * @Describe MapReducer第一个读取文档并计数 * @Author zhanglei * @Date 2019/11/18 22:53 **/ public class WordCountApp extends Configured implements Tool { public int run(String[] strings) throws Exception { String input_path="hdfs://192.168.91.130:8020/data/wc.txt"; String output_path="hdfs://192.168.91.130:8020/data/outputwc"; Configuration configuration = getConf(); final FileSystem fileSystem = FileSystem.get(new URI(input_path),configuration); if(fileSystem.exists(new Path(output_path))){ fileSystem.delete(new Path(output_path),true); } Job job = Job.getInstance(configuration,"WordCountApp"); job.setJarByClass(WordCountApp.class); job.setMapperClass(WordCountMapper.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); job.setReducerClass(WordCountReducer.class); job.setInputFormatClass(TextInputFormat.class); Path inpath = new Path(input_path); FileInputFormat.addInputPath(job,inpath); job.setOutputFormatClass(TextOutputFormat.class); Path outpath = new Path(output_path); FileOutputFormat.setOutputPath(job,outpath); return job.waitForCompletion(true) ? 0:1; } //继承 public static class WordCountMapper extends Mapper<Object,Text,Text,IntWritable>{ private final static IntWritable one= new IntWritable(1); private Text word = new Text(); public void map(Object key,Text value,Context context) throws IOException, InterruptedException { Text t = value; StringTokenizer itr = new StringTokenizer(value.toString()); while(itr.hasMoreTokens()){ word.set(itr.nextToken()); context.write(word,one); } } } public static class WordCountReducer extends Reducer<Object,Text,Text,IntWritable>{ private final static IntWritable res= new IntWritable(1); public void reduce(Text key,Iterable<IntWritable> values,Context context) throws IOException, InterruptedException { int sum = 0; for(IntWritable val:values){ sum+=val.get(); } res.set(sum); context.write(key,res); } } public static void main(String[] args) throws Exception { int exitCode = ToolRunner.run(new WordCountApp(), args); System.exit(exitCode); } }

本地eclipse运行 hadoop-2.6 mapreduce 报错，求助

报错信息是： 2016-02-26 11:24:07,722 INFO [main] Configuration.deprecation (Configuration.java:warnOnceIfDeprecated(1174)) - session.id is deprecated. Instead, use dfs.metrics.session-id 2016-02-26 11:24:07,727 INFO [main] jvm.JvmMetrics (JvmMetrics.java:init(76)) - Initializing JVM Metrics with processName=JobTracker, sessionId= 2016-02-26 11:24:08,081 WARN [main] mapreduce.JobResourceUploader (JobResourceUploader.java:uploadFiles(171)) - No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2016-02-26 11:24:08,091 INFO [main] mapreduce.JobSubmitter (JobSubmitter.java:submitJobInternal(252)) - Cleaning up the staging area file:/tmp/hadoop-fire/mapred/staging/fire1322517587/.staging/job_local1322517587_0001 2016-02-26 11:24:08,095 WARN [main] security.UserGroupInformation (UserGroupInformation.java:doAs(1674)) - PriviledgedActionException as:fire (auth:SIMPLE) cause:org.apache.hadoop.mapreduce.lib.input.InvalidInputException: Input path does not exist: file:/user/fire/dedup_in Exception in thread "main" org.apache.hadoop.mapreduce.lib.input.InvalidInputException: Input path does not exist: file:/user/fire/dedup_in at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.singleThreadedListStatus(FileInputFormat.java:323) at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.listStatus(FileInputFormat.java:265) at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.getSplits(FileInputFormat.java:387) at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:304) at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:321) at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:199) at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1307) at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1304) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) at org.apache.hadoop.mapreduce.Job.submit(Job.java:1304) at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1325) at com.hebut.mr.Dedup.main(Dedup.java:135)

在eclipse运行hadoop mapreduce例子报错

在终端运行hadoop带的例子正常，hadoop节点正常，错误如下 17/09/05 20:20:16 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 17/09/05 20:20:16 INFO Configuration.deprecation: session.id is deprecated. Instead, use dfs.metrics.session-id 17/09/05 20:20:16 INFO jvm.JvmMetrics: Initializing JVM Metrics with processName=JobTracker, sessionId= Exception in thread "main" java.net.ConnectException: Call From master/192.168.1.110 to localhost:9000 failed on connection exception: java.net.ConnectException: 拒绝连接; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:792) at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:732) at org.apache.hadoop.ipc.Client.call(Client.java:1479) at org.apache.hadoop.ipc.Client.call(Client.java:1412) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:229) at com.sun.proxy.$Proxy9.getFileInfo(Unknown Source) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:191) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) at com.sun.proxy.$Proxy9.getFileInfo(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:707) at org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:1785) at org.apache.hadoop.hdfs.DistributedFileSystem$17.doCall(DistributedFileSystem.java:1068) at org.apache.hadoop.hdfs.DistributedFileSystem$17.doCall(DistributedFileSystem.java:1064) at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1064) at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1426) at org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.checkOutputSpecs(FileOutputFormat.java:145) at org.apache.hadoop.mapreduce.JobSubmitter.checkSpecs(JobSubmitter.java:266) at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:139) at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1290) at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1287) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698) at org.apache.hadoop.mapreduce.Job.submit(Job.java:1287) at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1308) at mapreduce.Temperature.main(Temperature.java:202) Caused by: java.net.ConnectException: 拒绝连接 at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717) at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:531) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:495) at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:614) at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:712) at org.apache.hadoop.ipc.Client$Connection.access$2900(Client.java:375) at org.apache.hadoop.ipc.Client.getConnection(Client.java:1528) at org.apache.hadoop.ipc.Client.call(Client.java:1451) ... 28 more

用eclipse连接虚拟机hadoop集群执行MapReduce程序，但是报以下错误，请问如何解决？

# 说明：eclipse中关于hadoop的各项advanced parameter参数均已按配置文件进行配置。但在执行过程中还是报如下错误，请问如何解决。 # 执行日志： 2018-09-22 22:59:11,429 INFO [org.apache.commons.beanutils.FluentPropertyBeanIntrospector] - Error when creating PropertyDescriptor for public final void org.apache.commons.configuration2.AbstractConfiguration.setProperty(java.lang.String,java.lang.Object)! Ignoring this property. 2018-09-22 22:59:11,443 WARN [org.apache.hadoop.metrics2.impl.MetricsConfig] - Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2018-09-22 22:59:11,496 INFO [org.apache.hadoop.metrics2.impl.MetricsSystemImpl] - Scheduled Metric snapshot period at 10 second(s). 2018-09-22 22:59:11,496 INFO [org.apache.hadoop.metrics2.impl.MetricsSystemImpl] - JobTracker metrics system started 2018-09-22 22:59:20,863 WARN [org.apache.hadoop.mapreduce.JobResourceUploader] - Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2018-09-22 22:59:20,879 WARN [org.apache.hadoop.mapreduce.JobResourceUploader] - No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2018-09-22 22:59:20,928 INFO [org.apache.hadoop.mapreduce.lib.input.FileInputFormat] - Total input files to process : 1 2018-09-22 22:59:20,984 INFO [org.apache.hadoop.mapreduce.JobSubmitter] - number of splits:1 2018-09-22 22:59:21,072 INFO [org.apache.hadoop.mapreduce.JobSubmitter] - Submitting tokens for job: job_local1513265977_0001 2018-09-22 22:59:21,074 INFO [org.apache.hadoop.mapreduce.JobSubmitter] - Executing with tokens: [] 2018-09-22 22:59:21,950 INFO [org.apache.hadoop.mapred.LocalDistributedCacheManager] - Creating symlink: \tmp\hadoop-启政先生\mapred\local\1537628361150\movies.csv <- G:\java_workspace\MapReduce_DEMO/movies.csv 2018-09-22 22:59:21,995 WARN [org.apache.hadoop.fs.FileUtil] - Command 'E:\hadoop-3.0.0\bin\winutils.exe symlink G:\java_workspace\MapReduce_DEMO\movies.csv \tmp\hadoop-启政先生\mapred\local\1537628361150\movies.csv' failed 1 with: CreateSymbolicLink error (1314): ??????????? 2018-09-22 22:59:21,995 WARN [org.apache.hadoop.mapred.LocalDistributedCacheManager] - Failed to create symlink: \tmp\hadoop-启政先生\mapred\local\1537628361150\movies.csv <- G:\java_workspace\MapReduce_DEMO/movies.csv 2018-09-22 22:59:21,996 INFO [org.apache.hadoop.mapred.LocalDistributedCacheManager] - Localized hdfs://192.168.5.110:9000/temp/input/movies.csv as file:/tmp/hadoop-启政先生/mapred/local/1537628361150/movies.csv 2018-09-22 22:59:22,046 INFO [org.apache.hadoop.mapreduce.Job] - The url to track the job: http://localhost:8080/ 2018-09-22 22:59:22,047 INFO [org.apache.hadoop.mapreduce.Job] - Running job: job_local1513265977_0001 2018-09-22 22:59:22,047 INFO [org.apache.hadoop.mapred.LocalJobRunner] - OutputCommitter set in config null 2018-09-22 22:59:22,051 INFO [org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter] - File Output Committer Algorithm version is 2 2018-09-22 22:59:22,051 INFO [org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter] - FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false 2018-09-22 22:59:22,052 INFO [org.apache.hadoop.mapred.LocalJobRunner] - OutputCommitter is org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter 2018-09-22 22:59:22,100 INFO [org.apache.hadoop.mapred.LocalJobRunner] - Waiting for map tasks 2018-09-22 22:59:22,101 INFO [org.apache.hadoop.mapred.LocalJobRunner] - Starting task: attempt_local1513265977_0001_m_000000_0 2018-09-22 22:59:22,120 INFO [org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter] - File Output Committer Algorithm version is 2 2018-09-22 22:59:22,120 INFO [org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter] - FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false 2018-09-22 22:59:22,128 INFO [org.apache.hadoop.yarn.util.ProcfsBasedProcessTree] - ProcfsBasedProcessTree currently is supported only on Linux. 2018-09-22 22:59:22,169 INFO [org.apache.hadoop.mapred.Task] - Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@7ef907ef 2018-09-22 22:59:22,172 INFO [org.apache.hadoop.mapred.MapTask] - Processing split: hdfs://192.168.5.110:9000/temp/input/ratings.csv:0+2438233 ----------cachePath=/temp/input/movies.csv---------- 2018-09-22 22:59:22,226 INFO [org.apache.hadoop.mapred.LocalJobRunner] - map task executor complete. 2018-09-22 22:59:22,233 WARN [org.apache.hadoop.mapred.LocalJobRunner] - job_local1513265977_0001 java.lang.Exception: java.io.FileNotFoundException: \temp\input\movies.csv (系统找不到指定的路径。) at org.apache.hadoop.mapred.LocalJobRunner$Job.runTasks(LocalJobRunner.java:492) at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:552) Caused by: java.io.FileNotFoundException: \temp\input\movies.csv (系统找不到指定的路径。) at java.io.FileInputStream.open0(Native Method) at java.io.FileInputStream.open(Unknown Source) at java.io.FileInputStream.<init>(Unknown Source) at java.io.FileInputStream.<init>(Unknown Source) at java.io.FileReader.<init>(Unknown Source) at MovieJoinExercise1.MovieJoin$MovieJoinMapper.setup(MovieJoin.java:79) at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:143) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:794) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:342) at org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run(LocalJobRunner.java:271) at java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.util.concurrent.FutureTask.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) 2018-09-22 22:59:23,051 INFO [org.apache.hadoop.mapreduce.Job] - Job job_local1513265977_0001 running in uber mode : false 2018-09-22 22:59:23,052 INFO [org.apache.hadoop.mapreduce.Job] - map 0% reduce 0% 2018-09-22 22:59:23,053 INFO [org.apache.hadoop.mapreduce.Job] - Job job_local1513265977_0001 failed with state FAILED due to: NA 2018-09-22 22:59:23,058 INFO [org.apache.hadoop.mapreduce.Job] - Counters: 0

hadoop部署完成后，运行Pi实例检查集群是否成功时，出现错误

部署完Hadoop,运行Pi实例检查群集是否成功时，遇到下面的问题。请问哪里错误？ [zkpk@master hadoop-2.5.1]$ cd [zkpk@master ~]$ cd ~/hadoop-2.5.1/share/hadoop/mapreduce/ [zkpk@master mapreduce]$ hadoop jar ~/hadoop-2.5.1/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.5.1.jar pi 10 10 Number of Maps = 10 Samples per Map = 10 17/10/23 21:11:08 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable Wrote input for Map #0 Wrote input for Map #1 Wrote input for Map #2 Wrote input for Map #3 Wrote input for Map #4 Wrote input for Map #5 Wrote input for Map #6 Wrote input for Map #7 Wrote input for Map #8 Wrote input for Map #9 Starting Job java.io.IOException: Cannot initialize Cluster. Please check your configuration for mapreduce.framework.name and the correspond server addresses. at org.apache.hadoop.mapreduce.Cluster.initialize(Cluster.java:120) at org.apache.hadoop.mapreduce.Cluster.<init>(Cluster.java:82) at org.apache.hadoop.mapreduce.Cluster.<init>(Cluster.java:75) at org.apache.hadoop.mapreduce.Job$9.run(Job.java:1255) at org.apache.hadoop.mapreduce.Job$9.run(Job.java:1251) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1614) at org.apache.hadoop.mapreduce.Job.connect(Job.java:1250) at org.apache.hadoop.mapreduce.Job.submit(Job.java:1279) at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1303) at org.apache.hadoop.examples.QuasiMonteCarlo.estimatePi(QuasiMonteCarlo.java:306) at org.apache.hadoop.examples.QuasiMonteCarlo.run(QuasiMonteCarlo.java:354) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70) at org.apache.hadoop.examples.QuasiMonteCarlo.main(QuasiMonteCarlo.java:363) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.util.ProgramDriver$ProgramDescription.invoke(ProgramDriver.java:72) at org.apache.hadoop.util.ProgramDriver.run(ProgramDriver.java:145) at org.apache.hadoop.examples.ExampleDriver.main(ExampleDriver.java:74) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.util.RunJar.main(RunJar.java:212)

Hadoop mapreduce传值问题

最近mapreduce编写遇到了问题。在step4中，reduce可以同时收到从map中传来的A和B两组数据。但是在step5中的reudce却无法同时收到A、B两组数据，出现了有A没B，有B没A的现象，即A和B无法在同一次循环中出现。 step5，我几乎是从step4复制过来的，很奇怪他们的执行步骤为什么不一样。 step4 ``` import java.io.IOException; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.regex.Pattern; import org.apache.commons.net.telnet.EchoOptionHandler; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.FileSplit; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogWriter; //同现矩阵和用户偏好矩阵相乘 public class Step4 { public static boolean run(Configuration con, Map<String, String>map) { try { FileSystem fs = FileSystem.get(con); Job job = Job.getInstance(); job.setJobName("step4"); job.setJarByClass(App.class); job.setMapperClass(Step4_Mapper.class); job.setReducerClass(Step4_Reducer.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); FileInputFormat.setInputPaths(job, new Path[] { new Path(map.get("Step4Input1")), new Path(map.get("Step4Input2")) }); Path outpath = new Path(map.get("Step4Output")); if(fs.exists(outpath)){ fs.delete(outpath,true); } FileOutputFormat.setOutputPath(job, outpath); boolean f = job.waitForCompletion(true); return f; }catch(Exception e) { e.printStackTrace(); } return false; } static class Step4_Mapper extends Mapper<LongWritable, Text, Text, Text>{ private String flag; //每次map时都会先判断一次 @Override protected void setup(Context context )throws IOException,InterruptedException{ FileSplit split = (FileSplit) context.getInputSplit(); flag = split.getPath().getParent().getName(); System.out.print(flag+ "*************************"); } @Override protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException{ String[] tokens = Pattern.compile("[\t,]").split(value.toString()); //物品共现矩阵 if(flag.equals("step3")) { // i2:i3 1 // i2:i2 2 String[] v1 = tokens[0].split(":"); String itemID1 = v1[0]; String itemID2 = v1[1]; String num = tokens[1]; Text k = new Text(itemID1); Text v = new Text("A:"+itemID2+","+num); //A:i2,1 context.write(k,v); }else if(flag.equals("step2")) {//用户评价矩阵 // u2 i1:2,i3:4 String userID = tokens[0]; for(int i=1;i<tokens.length;i++) { String[] vector = tokens[i].split(":"); String itemID = vector[0]; //物品ID String pref = vector[1];//评分 Text k = new Text(itemID); Text v = new Text("B:"+userID+","+pref); context.write(k, v); } } } } static class Step4_Reducer extends Reducer<Text, Text, Text, Text>{ @Override protected void reduce(Text key, Iterable<Text>values, Context context) throws IOException,InterruptedException{ //A为同现矩阵，B为用户偏好矩阵 //某一个物品k，针对它和其他所有物品的同现次数v，都在mapA集合中 // Text k = new Text(itemID1); //Text v = new Text("A:"+itemID2+","+num); //A:i2,1 // context.write(k,v); //和该物品（key中的itemID）同现的其他物品的同现集合 //其他物品ID为map的key，同现数字为值 Map<String, Integer> mapA = new HashMap<String,Integer>(); //该物品（key中的itemID），所有用户的推荐权重分数 Map<String, Integer>mapB = new HashMap<String,Integer>(); for(Text line:values) { String val = line.toString(); if(val.startsWith("A:")) { String[] kv = Pattern.compile("[\t,]").split(val.substring(2)); try { mapA.put(kv[0], Integer.parseInt(kv[1])); }catch(Exception e) { e.printStackTrace(); } }else if(val.startsWith("B:")) { String[] kv = Pattern.compile("[\t,]").split(val.substring(2)); try { mapB.put(kv[0], Integer.parseInt(kv[1])); }catch(Exception e) { e.printStackTrace(); } } } double result = 0; Iterator<String>iter = mapA.keySet().iterator(); while(iter.hasNext()) { String mapk = iter.next(); //itemID int num =mapA.get(mapk).intValue(); // 获取同现值 Iterator<String>iterb = mapB.keySet().iterator(); while(iterb.hasNext()) { String mapkb = iterb.next(); int pref = mapB.get(mapkb).intValue(); result = num*pref; Text k = new Text(mapkb); Text v = new Text(mapk+ "," + result); context.write(k, v); } } } } } ``` step5 ``` import java.io.IOException; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.regex.Pattern; import org.apache.commons.net.telnet.EchoOptionHandler; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.Mapper.Context; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.FileSplit; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogWriter; //获得结果矩阵 public class Step5 { public static boolean run(Configuration con, Map<String, String>map) { try { FileSystem fs = FileSystem.get(con); Job job = Job.getInstance(); job.setJobName("step5"); job.setJarByClass(App.class); job.setMapperClass(Step5_Mapper.class); job.setReducerClass(Step5_Reducer.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); FileInputFormat.setInputPaths(job, new Path[] { new Path(map.get("Step5Input1")), new Path(map.get("Step5Input2")) }); Path outpath = new Path(map.get("Step5Output")); if(fs.exists(outpath)){ fs.delete(outpath,true); } FileOutputFormat.setOutputPath(job, outpath); boolean f = job.waitForCompletion(true); return f; }catch(Exception e) { e.printStackTrace(); } return false; } static class Step5_Mapper extends Mapper<LongWritable, Text, Text, Text>{ private String flag; //每次map时都会先判断一次 @Override protected void setup(Context context )throws IOException,InterruptedException{ FileSplit split = (FileSplit) context.getInputSplit(); flag = split.getPath().getParent().getName(); System.out.print(flag+ "*************************"); } @Override protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException{ String[] tokens = Pattern.compile("[\t,]").split(value.toString()); if(flag.equals("step4")) { // i2:i3 1 // i2:i2 2 Text k = new Text(tokens[0]); Text v = new Text("A:"+tokens[1]+","+tokens[2]); context.write(k, v); }else if(flag.equals("step2")) {//用户评价矩阵 // u2 i1:2,i3:4 String userID = tokens[0]; for(int i=1;i<tokens.length;i++) { String[] vector = tokens[i].split(":"); String itemID = vector[0]; //物品ID String pref = vector[1];//评分 Text k = new Text(itemID); Text v = new Text("B:"+userID+","+pref); context.write(k, v); } } } } //本reduce 负责累加结果 static class Step5_Reducer extends Reducer<Text, Text, Text, Text>{ protected void reduce(Text key, Iterable<Text>values, Context context) throws IOException,InterruptedException{ //其他物品ID为map的key，同现数字为值 Map<String, Double> mapA = new HashMap<String,Double>(); //该物品（key中的itemID），所有用户的推荐权重分数 Map<String, Integer>mapB = new HashMap<String,Integer>(); for(Text line : values) { String val = line.toString(); if(val.startsWith("A:")) { String[] kv = Pattern.compile("[\t,]").split(val.substring(2)); String tokens = kv[1]; String itemID = kv[0];//物品id Double score = Double.parseDouble(tokens); //相乘结果 //相加计算 if(mapA.containsKey(itemID)) { mapA.put(itemID, mapA.get(itemID)+score); }else { mapA.put(itemID, score); } }else if(val.startsWith("B:")) { String[] kv = Pattern.compile("[\t,]").split(val.substring(2)); try { mapB.put(kv[0], Integer.parseInt(kv[1])); }catch(Exception e) { e.printStackTrace(); } } } Iterator<String> iter = mapA.keySet().iterator(); while(iter.hasNext()) { String itemID = iter.next(); double score = mapA.get(itemID); Text v = new Text(itemID+","+score); Iterator<String>iterb = mapB.keySet().iterator(); while(iterb.hasNext()) { String mapkb = iterb.next(); Text k = new Text(mapkb); if(k.equals(key)) { continue; }else { context.write(key, v); } } } } } } ``` step4和step5配置  step4，在for循环中同时出现A和B  step5中，A和B无法出现在同一次循环  直接跳出了for循环进入下面的while循环，此时没有mapB，while无法正常进行  进行了多次step5后，输出完所有mapA之后，在下一次step5才进入mapB，此时轮到mapA是空的，而只有mapB 

cdh hadoop mapreduce 运行时的问题：（有时候会出现，有时候不出现，急求大神帮助）

15/10/08 08:49:13 INFO mapreduce.Job: Job job_1419225162729_18465 running in uber mode : false 15/10/08 08:49:13 INFO mapreduce.Job: map 0% reduce 0% 15/10/08 08:49:13 INFO mapreduce.Job: Job job_1419225162729_18465 failed with state FAILED due to: Application application_1419225162729_18465 failed 1 times due to AM Container for appattempt_1419225162729_18465_000001 exited with exitCode: -1000 due to: java.io.IOException: Not able to initialize app-log directories in any of the configured local directories for app application_1419225162729_18465 at org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.createAppLogDirs(DefaultContainerExecutor.java:459) at org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.startLocalizer(DefaultContainerExecutor.java:91) at org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService$LocalizerRunner.run(ResourceLocalizationService.java:861) .Failing this attempt.. Failing the application. 15/10/08 08:49:13 INFO mapreduce.Job: Counters: 0 Moved: 'hdfs://oiddhnode02:8020/user/nmger/worktemp/2015100408' to trash at: hdfs://oiddhnode02:8020/user/nmger/.Trash/Current

eclipse运行hadoop mapreduce程序如下错误

2017-09-06 15:48:42,677 INFO [LocalJobRunner Map Task Executor #0] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(591)) - 2017-09-06 15:48:42,686 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:flush(1460)) - Starting flush of map output 2017-09-06 15:48:42,686 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:flush(1482)) - Spilling map output 2017-09-06 15:48:42,686 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:flush(1483)) - bufstart = 0; bufend = 108; bufvoid = 104857600 2017-09-06 15:48:42,686 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:flush(1485)) - kvstart = 26214396(104857584); kvend = 26214352(104857408); length = 45/6553600 2017-09-06 15:48:42,733 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:sortAndSpill(1667)) - Finished spill 0 2017-09-06 15:48:42,743 INFO [LocalJobRunner Map Task Executor #0] mapred.Task (Task.java:done(1038)) - Task:attempt_local1469942249_0001_m_000000_0 is done. And is in the process of committing 2017-09-06 15:48:42,751 INFO [Thread-19] mapred.LocalJobRunner (LocalJobRunner.java:runTasks(456)) - map task executor complete. 2017-09-06 15:48:42,783 WARN [Thread-19] mapred.LocalJobRunner (LocalJobRunner.java:run(560)) - job_local1469942249_0001 java.lang.Exception: java.lang.NoSuchMethodError: org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree.getRssMemorySize()J at org.apache.hadoop.mapred.LocalJobRunner$Job.runTasks(LocalJobRunner.java:462) at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:522) Caused by: java.lang.NoSuchMethodError: org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree.getRssMemorySize()J at org.apache.hadoop.mapred.Task.updateResourceCounters(Task.java:872) at org.apache.hadoop.mapred.Task.updateCounters(Task.java:1021) at org.apache.hadoop.mapred.Task.done(Task.java:1040) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:345) at org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run(LocalJobRunner.java:243) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 2017-09-06 15:48:43,333 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1360)) - Job job_local1469942249_0001 running in uber mode : false 2017-09-06 15:48:43,335 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1367)) - map 0% reduce 0% 2017-09-06 15:48:43,337 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1380)) - Job job_local1469942249_0001 failed with state FAILED due to: NA 2017-09-06 15:48:43,352 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1385)) - Counters: 10 Map-Reduce Framework Map input records=12 Map output records=12 Map output bytes=108 Map output materialized bytes=0 Input split bytes=104 Combine input records=0 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 File Input Format Counters Bytes Read=132 Finished

hadoop安装完并正常运行，输入以下命令进行测试，发现如下异常，求大神解答！

ubuntu@master:~$ hadoop-2.5.2/bin/hadoop jar hadoop-2.5.2/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.5.2.jar wordcount /hadoop-2.5.2/input/* /hadoop/output 15/10/09 16:13:28 INFO client.RMProxy: Connecting to ResourceManager at /115.156.236.181:8032 15/10/09 16:13:28 INFO mapreduce.JobSubmitter: Cleaning up the staging area /tmp/hadoop-yarn/staging/ubuntu/.staging/job_1444351299360_0007 org.apache.hadoop.mapreduce.lib.input.InvalidInputException: Input Pattern hdfs://115.156.236.181:9000/hadoop-2.5.2/input/* matches 0 files at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.singleThreadedListStatus(FileInputFormat.java:321) at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.listStatus(FileInputFormat.java:264) at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.getSplits(FileInputFormat.java:385) at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:493) at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:510) at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:394) at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1285) at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1282) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1614) at org.apache.hadoop.mapreduce.Job.submit(Job.java:1282) at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1303) at org.apache.hadoop.examples.WordCount.main(WordCount.java:87) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.util.ProgramDriver$ProgramDescription.invoke(ProgramDriver.java:72) at org.apache.hadoop.util.ProgramDriver.run(ProgramDriver.java:145) at org.apache.hadoop.examples.ExampleDriver.main(ExampleDriver.java:74) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.util.RunJar.main(RunJar.java:212)

org.apache.hadoop.mapred.LocalJobRunner这个类在那个包里？

我在用sqoop1的javaapi操作，但是一执行命令就会报这个错，hadoop集群并不在运行程序的机器上，我是缺少这个类么，我翻了一般依赖里面确实没有 ``` Exception in thread "main" java.lang.NoSuchMethodError: org.apache.hadoop.mapred.LocalJobRunner.<init>(Lorg/apache/hadoop/conf/Configuration;)V at org.apache.hadoop.mapred.LocalClientProtocolProvider.create(LocalClientProtocolProvider.java:42) at org.apache.hadoop.mapreduce.Cluster.initialize(Cluster.java:95) at org.apache.hadoop.mapreduce.Cluster.<init>(Cluster.java:82) at org.apache.hadoop.mapreduce.Cluster.<init>(Cluster.java:75) at org.apache.hadoop.mapreduce.Job$9.run(Job.java:1260) at org.apache.hadoop.mapreduce.Job$9.run(Job.java:1256) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1866) at org.apache.hadoop.mapreduce.Job.connect(Job.java:1255) at org.apache.hadoop.mapreduce.Job.submit(Job.java:1284) at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1308) at org.apache.sqoop.mapreduce.ExportJobBase.doSubmitJob(ExportJobBase.java:322) at org.apache.sqoop.mapreduce.ExportJobBase.runJob(ExportJobBase.java:299) at org.apache.sqoop.mapreduce.ExportJobBase.runExport(ExportJobBase.java:440) at org.apache.sqoop.manager.SqlManager.exportTable(SqlManager.java:931) at org.apache.sqoop.tool.ExportTool.exportTable(ExportTool.java:80) at org.apache.sqoop.tool.ExportTool.run(ExportTool.java:99) at org.apache.sqoop.Sqoop.run(Sqoop.java:147) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:183) at com.mshuoke.datagw.impl.sqoop.SqoopTest.main(SqoopTest.java:52) 09:55:47.069 [Thread-4] DEBUG org.apache.hadoop.util.ShutdownHookManager - ShutdownHookManger complete shutdown. ```

Eclipse里如何debug跟踪MapReduce程序到hadoop源码里？

我本地一台机子起了 ``` 4504 ResourceManager 4066 DataNode 4761 NodeManager 5068 JobHistoryServer 4357 SecondaryNameNode 3833 NameNode 5127 Jps ``` 在hadoop-env.sh里设置了HADOOP_OPTS="$HADOOP_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,address=9000" 然后导入编译好的hadoop全部源码并各处打了很多断点， 然后bin/hadoop jar path/to/hadoop-mapreduce-examples-2.2.0.jar wordcount /wordcount /output 然后开始在Eclipse里远程调试，可是在最后只有这个断点生效了。。。 ``` Thread [main] (Suspended (breakpoint at line 342 in JobSubmitter)) JobSubmitter.submitJobInternal(Job, Cluster) line: 342 ... RunJar.main(String[]) line: 212 ```

关于eclipse中运行mapreduce不是在hadoop集群环境运行而是在本地运行的问题

1.我用eclipse远程连接linux上的hadoop集群，跑Mapreduce程序都可以顺利完成，结果在集群里也可以看得到。 但是，跑程序的时候，我去集群上Jps没有我正在跑的程序 而且，我到job的web界面下，也没有我的MapReduce任务记录。。。 是不是eclipse其实在本地跑的，没有在集群中跑，我无法想明白，还请指教

hadoop单词统计报错Job job_1581768459583_0001 failed

3个节点hadoop01、hadoop02、hadoop03 hadoop01是主节点 hadoop01、hadoop02、hadoop03是从节点，目前集群已搭建好，jps查看三个节点运行都很正常，而且UI也能正常显示，但是使用hadoop自带的hadoop-mapreduce-examples-2.7.4.jar的wordcount进行单词统计时报错如下，请高人指点，看不懂呀： ```[root@hadoop01 mapreduce]# hadoop jar hadoop-mapreduce-examples-2.7.4.jar wordcount /wordcount/input /wordcount/output 20/02/15 20:14:25 INFO client.RMProxy: Connecting to ResourceManager at hadoop01/192.168.233.132:8032 20/02/15 20:14:27 INFO input.FileInputFormat: Total input paths to process : 1 20/02/15 20:14:27 INFO mapreduce.JobSubmitter: number of splits:1 20/02/15 20:14:28 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1581768459583_0001 20/02/15 20:14:28 INFO impl.YarnClientImpl: Submitted application application_1581768459583_0001 20/02/15 20:14:28 INFO mapreduce.Job: The url to track the job: http://hadoop01:8088/proxy/application_1581768459583_0001/ 20/02/15 20:14:28 INFO mapreduce.Job: Running job: job_1581768459583_0001 20/02/15 20:15:38 INFO mapreduce.Job: Job job_1581768459583_0001 running in uber mode : false 20/02/15 20:15:38 INFO mapreduce.Job: map 0% reduce 0% 20/02/15 20:15:38 INFO mapreduce.Job: Job job_1581768459583_0001 failed with state FAILED due to: Application application_1581768459583_0001 failed 2 times due to Error launching appattempt_1581768459583_0001_000002. Got exception: java.io.IOException: Failed on local exception: java.io.IOException: java.io.IOException: Connection reset by peer; Host Details : local host is: "hadoop01.com/79.124.78.101"; destination host is: "79.124.78.101":43276; at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:776) at org.apache.hadoop.ipc.Client.call(Client.java:1480) at org.apache.hadoop.ipc.Client.call(Client.java:1413) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:229) at com.sun.proxy.$Proxy83.startContainers(Unknown Source) at org.apache.hadoop.yarn.api.impl.pb.client.ContainerManagementProtocolPBClientImpl.startContainers(ContainerManagementProtocolPBClientImpl.java:96) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:191) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) at com.sun.proxy.$Proxy84.startContainers(Unknown Source) at org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher.launch(AMLauncher.java:119) at org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher.run(AMLauncher.java:250) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.io.IOException: java.io.IOException: Connection reset by peer at org.apache.hadoop.ipc.Client$Connection$1.run(Client.java:688) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1746) at org.apache.hadoop.ipc.Client$Connection.handleSaslConnectionFailure(Client.java:651) at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:738) at org.apache.hadoop.ipc.Client$Connection.access$2900(Client.java:376) at org.apache.hadoop.ipc.Client.getConnection(Client.java:1529) at org.apache.hadoop.ipc.Client.call(Client.java:1452) ... 16 more Caused by: java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at java.io.FilterInputStream.read(FilterInputStream.java:133) at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) at java.io.BufferedInputStream.read(BufferedInputStream.java:265) at java.io.DataInputStream.readInt(DataInputStream.java:387) at org.apache.hadoop.security.SaslRpcClient.saslConnect(SaslRpcClient.java:367) at org.apache.hadoop.ipc.Client$Connection.setupSaslConnection(Client.java:561) at org.apache.hadoop.ipc.Client$Connection.access$1900(Client.java:376) at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:730) at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:726) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1746) at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:726) ... 19 more . Failing the application. 20/02/15 20:15:38 INFO mapreduce.Job: Counters: 0 ```

Hadoop Distcp报错 队列问题

``` sudo -uxiaosi hadoop distcp hdfs:///user/xiaosi/tmp/data_group/histories/day=20161116 hdfs:///user/xiaosi/data_group/histories ``` 报错： ``` 17/01/17 19:18:46 ERROR security.UserGroupInformation: PriviledgedActionException as:xiaosi (auth:SIMPLE) cause:java.io.IOException: Failed to run job : User xiaosi cannot submit applications to queue root.default 17/01/17 19:18:46 ERROR tools.DistCp: Exception encountered java.io.IOException: Failed to run job : User xiaosi cannot submit applications to queue root.default at org.apache.hadoop.mapred.YARNRunner.submitJob(YARNRunner.java:299) at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:430) at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1268) at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1265) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:396) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1491) at org.apache.hadoop.mapreduce.Job.submit(Job.java:1265) at org.apache.hadoop.tools.DistCp.execute(DistCp.java:153) at org.apache.hadoop.tools.DistCp.run(DistCp.java:118) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70) at org.apache.hadoop.tools.DistCp.main(DistCp.java:375) ```

hdfs无法进行词频统计

# hadoop集群，hdfs无法进行词频统计 ## 执行语句： hadoop jar hadoop-mapreduce-examples-2.7.4.jar wordcount \ > /wordcount/input /wordcount/output ``` ``` ## ResourceManager已经启动 [root@hadoop01 mapreduce]# jps 2977 NodeManager 2597 DataNode 3557 Jps 2873 ResourceManager 2494 NameNode ``` ``` ## 防火墙已关闭 [root@hadoop01 mapreduce]# service iptables stop [root@hadoop01 mapreduce]# ``` ``` 错误提示： INFO client.RMProxy: Connecting to ResourceManager at hadoop01/192.168.131.131:8032 org.apache.hadoop.mapred.FileAlreadyExistsException: Output directory hdfs://hadoop01:9000/wordcount/output already exists at org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.checkOutputSpecs(FileOutputFormat.java:146) at org.apache.hadoop.mapreduce.JobSubmitter.checkSpecs(JobSubmitter.java:266) at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:139) at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1290) at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1287) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1746) at org.apache.hadoop.mapreduce.Job.submit(Job.java:1287) at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1308) at org.apache.hadoop.examples.WordCount.main(WordCount.java:87) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.util.ProgramDriver$ProgramDescription.invoke(ProgramDriver.java:71) at org.apache.hadoop.util.ProgramDriver.run(ProgramDriver.java:144) at org.apache.hadoop.examples.ExampleDriver.main(ExampleDriver.java:74) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.util.RunJar.run(RunJar.java:221) at org.apache.hadoop.util.RunJar.main(RunJar.java:136) ``` ```

window系统下开发hadoop2.2出现报错

Exception in thread "main" java.io.IOException: Cannot run program "E:\hadoop-2.4.0\bin\winutils.exe": CreateProcess error=216, ӳÏñÎļþ %1 ÓÐЧ£¬µ«²»ÊÊÓÃÓڴ˼ÆË at java.lang.ProcessBuilder.start(Unknown Source) at org.apache.hadoop.util.Shell.runCommand(Shell.java:404) at org.apache.hadoop.util.Shell.run(Shell.java:379) at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:589) at org.apache.hadoop.util.Shell.execCommand(Shell.java:678) at org.apache.hadoop.util.Shell.execCommand(Shell.java:661) at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:639) at org.apache.hadoop.fs.RawLocalFileSystem.mkdirs(RawLocalFileSystem.java:435) at org.apache.hadoop.fs.FilterFileSystem.mkdirs(FilterFileSystem.java:277) at org.apache.hadoop.mapreduce.JobSubmissionFiles.getStagingDir(JobSubmissionFiles.java:125) at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:344) at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1268) at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1265) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Unknown Source) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1491) at org.apache.hadoop.mapreduce.Job.submit(Job.java:1265) at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1286) at WordCount.main(WordCount.java:84) Caused by: java.io.IOException: CreateProcess error=216, ӳÏñÎļþ %1 ÓÐЧ£¬µ«²»ÊÊÓÃÓڴ˼ÆË at java.lang.ProcessImpl.create(Native Method) at java.lang.ProcessImpl.<init>(Unknown Source) at java.lang.ProcessImpl.start(Unknown Source) ... 19 more

Hadoop2.x ，一直报无法初始化对象，这个是什么原因啊

15/07/25 03:54:19 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 15/07/25 03:54:31 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032 15/07/25 03:54:32 WARN mapreduce.JobSubmitter: Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 15/07/25 03:54:32 INFO mapreduce.JobSubmitter: Cleaning up the staging area /tmp/hadoop-yarn/staging/root/.staging/job_1437805442648_0002 Exception in thread "main" java.lang.RuntimeException: java.lang.InstantiationException at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:131) at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:594) at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:614) at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:492) at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1296) at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1293) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1628) at org.apache.hadoop.mapreduce.Job.submit(Job.java:1293) at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1314) at com.baizhi.myhadoop.TestCombineFileInputFormat.main(TestCombineFileInputFormat.java:66) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.util.RunJar.run(RunJar.java:221) at org.apache.hadoop.util.RunJar.main(RunJar.java:136) Caused by: java.lang.InstantiationException at sun.reflect.InstantiationExceptionConstructorAccessorImpl.newInstance(InstantiationExceptionConstructorAccessorImpl.java:48) at java.lang.reflect.Constructor.newInstance(Constructor.java:526) at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:129) ... 17 more

Hadoop 搭建好，运行 PI 实例检查集群，发现到 Running job一直没反应

``` [hyc@master ~]$ hadoop jar ~/hadoop-2.5.1/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.5.1.jar pi 10 10 Number of Maps = 10 Samples per Map = 10 Wrote input for Map #0 Wrote input for Map #1 Wrote input for Map #2 Wrote input for Map #3 Wrote input for Map #4 Wrote input for Map #5 Wrote input for Map #6 Wrote input for Map #7 Wrote input for Map #8 Wrote input for Map #9 Starting Job 19/11/19 15:17:56 INFO client.RMProxy: Connecting to ResourceManager at master/192.168.16.4:18040 19/11/19 15:17:59 INFO input.FileInputFormat: Total input paths to process : 10 19/11/19 15:17:59 INFO mapreduce.JobSubmitter: number of splits:10 19/11/19 15:18:00 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1574142670145_0002 19/11/19 15:18:01 INFO impl.YarnClientImpl: Submitted application application_1574142670145_0002 19/11/19 15:18:01 INFO mapreduce.Job: The url to track the job: http://master:18088/proxy/application_1574142670145_0002/ 19/11/19 15:18:01 INFO mapreduce.Job: Running job: job_1574142670145_0002 ```

大学四年自学走来，这些私藏的实用工具/学习网站我贡献出来了

大学四年，看课本是不可能一直看课本的了，对于学习，特别是自学，善于搜索网上的一些资源来辅助，还是非常有必要的，下面我就把这几年私藏的各种资源，网站贡献出来给你们。主要有：电子书搜索、实用工具、在线视频学习网站、非视频学习网站、软件下载、面试/求职必备网站。 注意：文中提到的所有资源，文末我都给你整理好了，你们只管拿去，如果觉得不错，转发、分享就是最大的支持了。 一、电子书搜索 对于大部分程序员...

【JSON解析】浅谈JSONObject的使用

简介 在程序开发过程中，在参数传递，函数返回值等方面，越来越多的使用JSON。JSON(JavaScript Object Notation)是一种轻量级的数据交换格式，同时也易于机器解析和生成、易于理解、阅读和撰写，而且Json采用完全独立于语言的文本格式，这使得Json成为理想的数据交换语言。 JSON建构于两种结构： “名称/值”对的集合(A Collection of name/va...

程序员请照顾好自己，周末病魔差点一套带走我。

程序员在一个周末的时间，得了重病，差点当场去世，还好及时挽救回来了。

卸载 x 雷某度！GitHub 标星 1.5w+，从此我只用这款全能高速下载工具！

作者 | Rocky0429 来源 | Python空间 大家好，我是 Rocky0429，一个喜欢在网上收集各种资源的蒟蒻… 网上资源眼花缭乱，下载的方式也同样千奇百怪，比如 BT 下载，磁力链接，网盘资源等等等等，下个资源可真不容易，不一样的方式要用不同的下载软件，因此某比较有名的 x 雷和某度网盘成了我经常使用的工具。 作为一个没有钱的穷鬼，某度网盘几十 kb 的下载速度让我...

只因接了一个电话，程序员被骗 30 万！

今天想给大家说一个刚刚发生在我身边的一起真实的诈骗经历，我的朋友因此被骗走30万。注：为了保护当事人隐私，部分情节进行了修改。1平安夜突来的电话开始以为就像普通的诈骗一样，想办法让你把钱...

我一个37岁的程序员朋友

周末了，人一旦没有点事情干，心里就瞎想，而且跟几个老男人坐在一起，更容易瞎想，我自己现在也是 30 岁了，也是无时无刻在担心自己的职业生涯，担心丢掉工作没有收入，担心身体机能下降，担心突...

python自动下载图片

近日闲来无事，总有一种无形的力量萦绕在朕身边，让朕精神涣散，昏昏欲睡。 可是，像朕这么有职业操守的社畜怎么能在上班期间睡瞌睡呢，我不禁陷入了沉思。。。。 突然旁边的IOS同事问：‘嘿，兄弟，我发现一个网站的图片很有意思啊，能不能帮我保存下来提升我的开发灵感？’ 作为一个坚强的社畜怎么能说自己不行呢，当时朕就不假思索的答应：‘oh， It’s simple. Wait for me for a ...

一名大专同学的四个问题

【前言】 　　收到一封来信，赶上各种事情拖了几日，利用今天要放下工作的时机，做个回复。 　　2020年到了，就以这一封信，作为开年标志吧。 【正文】 　　您好，我是一名现在有很多困惑的大二学生。有一些问题想要向您请教。 　　先说一下我的基本情况，高考失利，不想复读，来到广州一所大专读计算机应用技术专业。学校是偏艺术类的，计算机专业没有实验室更不用说工作室了。而且学校的学风也不好。但我很想在计算机领...

复习一周，京东+百度一面，不小心都拿了Offer

京东和百度一面都问了啥，面试官百般刁难，可惜我全会。

Java 14 都快来了，为什么还有这么多人固守Java 8？

从Java 9开始，Java版本的发布就让人眼花缭乱了。每隔6个月，都会冒出一个新版本出来，Java 10 , Java 11, Java 12, Java 13， 到2020年3月份，...

达摩院十大科技趋势发布：2020 非同小可！

【CSDN编者按】1月2日，阿里巴巴发布《达摩院2020十大科技趋势》，十大科技趋势分别是：人工智能从感知智能向认知智能演进；计算存储一体化突破AI算力瓶颈；工业互联网的超融合；机器间大规模协作成为可能；模块化降低芯片设计门槛；规模化生产级区块链应用将走入大众；量子计算进入攻坚期；新材料推动半导体器件革新；保护数据隐私的AI技术将加速落地；云成为IT技术创新的中心 。 新的画卷，正在徐徐展开。...

轻松搭建基于 SpringBoot + Vue 的 Web 商城应用

首先介绍下在本文出现的几个比较重要的概念： 函数计算（Function Compute）: 函数计算是一个事件驱动的服务，通过函数计算，用户无需管理服务器等运行情况，只需编写代码并上传。函数计算准备计算资源，并以弹性伸缩的方式运行用户代码，而用户只需根据实际代码运行所消耗的资源进行付费。Fun: Fun 是一个用于支持 Serverless 应用部署的工具，能帮助您便捷地管理函数计算、API ...

讲真，这两个IDE插件，可以让你写出质量杠杠的代码

周末躺在床上看《拯救大兵瑞恩》 周末在闲逛的时候，发现了两个优秀的 IDE 插件，据说可以提高代码的质量，我就安装了一下，试了试以后发现，确实很不错，就推荐给大家。 01、Alibaba Java 代码规范插件 《阿里巴巴 Java 开发手册》，相信大家都不会感到陌生，其 IDEA 插件的下载次数据说达到了 80 万次，我今天又贡献了一次。嘿嘿。 该项目的插件地址： https://github....

Python+OpenCV实时图像处理

目录 1、导入库文件 2、设计GUI 3、调用摄像头 4、实时图像处理 4.1、阈值二值化 4.2、边缘检测 4.3、轮廓检测 4.4、高斯滤波 4.5、色彩转换 4.6、调节对比度 5、退出系统 初学OpenCV图像处理的小伙伴肯定对什么高斯函数、滤波处理、阈值二值化等特性非常头疼，这里给各位分享一个小项目，可通过摄像头实时动态查看各类图像处理的特点，也可对各位调参、测试...

2020年一线城市程序员工资大调查

人才需求 一线城市共发布岗位38115个，招聘120827人。 其中 beijing 22805 guangzhou 25081 shanghai 39614 shenzhen 33327 工资分布 2020年中国一线城市程序员的平均工资为16285元，工资中位数为14583元，其中95%的人的工资位于5000到20000元之间。 和往年数据比较： yea...

为什么猝死的都是程序员，基本上不见产品经理猝死呢？

相信大家时不时听到程序员猝死的消息，但是基本上听不到产品经理猝死的消息，这是为什么呢？ 我们先百度搜一下：程序员猝死，出现将近700多万条搜索结果： 搜索一下：产品经理猝死，只有400万条的搜索结果，从搜索结果数量上来看，程序员猝死的搜索结果就比产品经理猝死的搜索结果高了一倍，而且从下图可以看到，首页里面的五条搜索结果，其实只有两条才是符合条件。 所以程序员猝死的概率真的比产品经理大，并不是错...

害怕面试被问HashMap？这一篇就搞定了！

声明：本文以jdk1.8为主！ 搞定HashMap 作为一个Java从业者，面试的时候肯定会被问到过HashMap，因为对于HashMap来说，可以说是Java集合中的精髓了，如果你觉得自己对它掌握的还不够好，我想今天这篇文章会非常适合你，至少，看了今天这篇文章，以后不怕面试被问HashMap了 其实在我学习HashMap的过程中，我个人觉得HashMap还是挺复杂的，如果真的想把它搞得明明白...

毕业5年，我问遍了身边的大佬，总结了他们的学习方法

我问了身边10个大佬，总结了他们的学习方法，原来成功都是有迹可循的。

python爬取百部电影数据，我分析出了一个残酷的真相

2019年就这么匆匆过去了，就在前几天国家电影局发布了2019年中国电影市场数据，数据显示去年总票房为642.66亿元，同比增长5.4%；国产电影总票房411.75亿元，同比增长8.65%，市场占比 64.07%；城市院线观影人次17.27亿，同比增长0.64%。 看上去似乎是一片大好对不对？不过作为一名严谨求实的数据分析师，我从官方数据中看出了一点端倪：国产票房增幅都已经高达8.65%了，为什...

推荐10个堪称神器的学习网站

每天都会收到很多读者的私信，问我：“二哥，有什么推荐的学习网站吗？最近很浮躁，手头的一些网站都看烦了，想看看二哥这里有什么新鲜货。” 今天一早做了个恶梦，梦到被老板辞退了。虽然说在我们公司，只有我辞退老板的份，没有老板辞退我这一说，但是还是被吓得 4 点多都起来了。（主要是因为我掌握着公司所有的核心源码，哈哈哈） 既然 4 点多起来，就得好好利用起来。于是我就挑选了 10 个堪称神器的学习网站，推...

这些软件太强了，Windows必装！尤其程序员！

Windows可谓是大多数人的生产力工具，集娱乐办公于一体，虽然在程序员这个群体中都说苹果是信仰，但是大部分不都是从Windows过来的，而且现在依然有很多的程序员用Windows。 所以，今天我就把我私藏的Windows必装的软件分享给大家，如果有一个你没有用过甚至没有听过，那你就赚了????，这可都是提升你幸福感的高效率生产力工具哦！ 走起！???? NO、1 ScreenToGif 屏幕，摄像头和白板...

阿里面试，面试官没想到一个ArrayList，我都能跟他扯半小时

我是真的没想到，面试官会这样问我ArrayList。

曾经优秀的人，怎么就突然不优秀了。

职场上有很多辛酸事，很多合伙人出局的故事，很多技术骨干被裁员的故事。说来模板都类似，曾经是名校毕业，曾经是优秀员工，曾经被领导表扬，曾经业绩突出，然而突然有一天，因为种种原因，被裁员了，...

大学四年因为知道了这32个网站，我成了别人眼中的大神！

依稀记得，毕业那天，我们导员发给我毕业证的时候对我说“你可是咱们系的风云人物啊”，哎呀，别提当时多开心啦????，嗯，我们导员是所有导员中最帅的一个，真的???? 不过，导员说的是实话，很多人都叫我大神的，为啥，因为我知道这32个网站啊，你说强不强????，这次是绝对的干货，看好啦，走起来！ PS：每个网站都是学计算机混互联网必须知道的，真的牛杯，我就不过多介绍了，大家自行探索，觉得没用的，尽管留言吐槽吧???? 社...

良心推荐，我珍藏的一些Chrome插件

上次搬家的时候，发了一个朋友圈，附带的照片中不小心暴露了自己的 Chrome 浏览器插件之多，于是就有小伙伴评论说分享一下我觉得还不错的浏览器插件。 我下面就把我日常工作和学习中经常用到的一些 Chrome 浏览器插件分享给大家，随便一个都能提高你的“生活品质”和工作效率。 Markdown Here Markdown Here 可以让你更愉快的写邮件，由于支持 Markdown 直接转电子邮...

【程序人生】程序员接私活常用平台汇总

00. 目录 文章目录00. 目录01. 前言02. 程序员客栈03. 码市04. 猪八戒网05. 开源众包06. 智城外包网07. 实现网08. 猿急送09. 人人开发10. 开发邦11. 电鸭社区12. 快码13. 英选14. Upwork15. Freelancer16. Dribbble17. Remoteok18. Toptal19. AngelList20. Topcoder21. ...

看完这篇HTTP，跟面试官扯皮就没问题了

我是一名程序员，我的主要编程语言是 Java，我更是一名 Web 开发人员，所以我必须要了解 HTTP，所以本篇文章就来带你从 HTTP 入门到进阶，看完让你有一种恍然大悟、醍醐灌顶的感觉。 最初在有网络之前，我们的电脑都是单机的，单机系统是孤立的，我还记得 05 年前那会儿家里有个电脑，想打电脑游戏还得两个人在一个电脑上玩儿，及其不方便。我就想为什么家里人不让上网，我的同学 xxx 家里有网，每...

史上最全的IDEA快捷键总结

现在Idea成了主流开发工具，这篇博客对其使用的快捷键做了总结，希望对大家的开发工作有所帮助。

阿里程序员写了一个新手都写不出的低级bug，被骂惨了。

这种新手都不会范的错，居然被一个工作好几年的小伙子写出来，差点被当场开除了。

谁是华为扫地僧？

是的，华为也有扫地僧！2020年2月11-12日，“养在深闺人不知”的华为2012实验室扫地僧们，将在华为开发者大会2020（Cloud）上，和大家见面。到时，你可以和扫地僧们，吃一个洋...