sinat_31516211
GRAMMAC
采纳率100%
2017-04-28 09:34 阅读 1.4k
已采纳

Hadoop2.7.3本地调试reduce失败

50

2017-04-28 17:13:23,238 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1360)) - Job job_local1046505697_0023 running in uber mode : false
2017-04-28 17:13:23,238 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1367)) - map 100% reduce 0%
2017-04-28 17:13:23,239 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1380)) - Job job_local1046505697_0023 failed with state FAILED due to: NA
2017-04-28 17:13:23,242 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1385)) - Counters: 22
之前运行成功了,今天再次运行报了这个错误,求指导。

  • 点赞
  • 写回答
  • 关注问题
  • 收藏
  • 复制链接分享

3条回答 默认 最新

  • 已采纳
    devmiao devmiao 2017-04-28 10:34
  • devmiao devmiao 2017-04-28 10:34
  • sinat_31516211 GRAMMAC 2017-05-01 02:39

    Caused by: java.lang.ArrayIndexOutOfBoundsException: 2
    at mykmeans.KMapper.map(KMapper.java:53)
    at mykmeans.KMapper.map(KMapper.java:1)
    package mykmeans;
    import java.io.ByteArrayOutputStream;

    import java.io.IOException;

    import java.net.URI;

    import java.util.StringTokenizer;

    import org.apache.hadoop.conf.Configuration;

    import org.apache.hadoop.fs.FSDataInputStream;

    import org.apache.hadoop.fs.FileSystem;

    import org.apache.hadoop.fs.Path;

    import org.apache.hadoop.io.IOUtils;

    import org.apache.hadoop.io.LongWritable;

    import org.apache.hadoop.io.Text;

    import org.apache.hadoop.mapreduce.Mapper;

    public class KMapper extends Mapper{
    private String[] center;

    //读取3.txt中更新的中心点坐标,并将坐标存入center数组中

    protected void setup(Context context) throws IOException,InterruptedException //read centerlist, and save to center[]

    {

    String centerlist = "hdfs://Master:9000/hadoop/kmeans/3.txt"; //center文件

    Configuration conf1 = new Configuration();

    conf1.set("hadoop.job.ugi", "hadoop-user, hadoop-user");

    FileSystem fs = FileSystem.get(URI.create(centerlist), conf1);

    FSDataInputStream in = null;

    ByteArrayOutputStream out = new ByteArrayOutputStream();

    try{

    in = fs.open( new Path(centerlist) );

    IOUtils.copyBytes(in, out, 100, false);

    center = out.toString().split(" ");

    }finally{

    IOUtils.closeStream(in);

    }

    }

    //从hadoop接收的数据在2.txt中保存

    public void map(LongWritable key,Text value,Context context) throws IOException,InterruptedException

    {

    StringTokenizer itr = new StringTokenizer(value.toString());

    //从2.txt读入数据,以空格为分割符,一个一个处理

    while(itr.hasMoreTokens())//用于判断所要分析的字符串中,是否还有语言符号,如果有则返回true,反之返回false

    {

    //计算第一个坐标跟第一个中心的距离min

    String outValue = new String(itr.nextToken());//逐个获取以空格为分割符的字符串(2,3) (10,30) (34,40) (1,1)

    String[] list = outValue.replace("(", "").replace(")", "").split(",");

    String[] c = center[0].replace("(", "").replace(")", "").split(",");
    System.out.println("list:" + list.length);
    System.out.println("c:" + c.length);
    float min = 0;

    int pos = 0;

    for(int i = 0; i < list.length; i++)

    {

    System.out.println(i + "list:" + list[i]);

    System.out.println(i + "c:" + c[i]);

    min += (float) Math.pow((Float.parseFloat(list[i]) - Float.parseFloat(c[i])),2);//求欧式距离,为加根号

    }

                for(int i = 0; i < center.length; i++)  
                {  
                    String[] centerStrings = center[i].replace("(", "").replace(")", "").split(",");  
                    float distance = 0;  
                    for(int j = 0; j < list.length; j++)  
                        distance += (float) Math.pow((Float.parseFloat(list[j]) - Float.parseFloat(centerStrings[j])), 2);  
                    if(min > distance)  
                    {  
                        min = distance;  
                        pos = i;  
                    }  
                }  
                context.write(new Text(center[pos]), new Text(outValue));//输出:中心点,对应的坐标  
                System.out.println("The Center" + center[pos] + "Cooedinate" + outValue);  
                System.out.println("Mapper Output:" + center[pos] + " " + outValue);  
            }  
        }  
    

    }

    点赞 评论 复制链接分享

相关推荐