MapReduce的输出错误问题

Question

为什么最后的输出并没有输出我理想的班级学科平均分呢？请求指导！

这里是原始数据

这里是输出结果

package experiment.big101;


import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
import java.util.Arrays;


public class SMapper extends Mapper<LongWritable, Text, Text, Text> {

    public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

        String line = value.toString();

        String[] str = line.split(",");
        int studentID = Integer.valueOf(str[0]);                //学号


        String[] score = Arrays.copyOfRange(str, 2, 5);            //数学、英语、语文成绩

        String scorestr = String.join(",",score);

        context.write(new Text(String.valueOf(studentID)), new Text(String.valueOf(scorestr)));

    }

}


package experiment.big101;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;


public class SReducer extends Reducer<Text, Text, Text, Text> {
    public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
        int subject1Total = 0;
        int subject2Total = 0;
        int subject3Total = 0;
        int studentCount = 0;
        StringBuilder studentAverages = new StringBuilder();
        for (Text value : values) {
            String[] scores = value.toString().split(",");
            int s1 = Integer.parseInt(scores[0].trim());
            int s2 = Integer.parseInt(scores[1].trim());
            int s3 = Integer.parseInt(scores[2].trim());
            int studentAvg = (s1 + s2 + s3) / 3;
            if (studentAverages.length() > 0) {
                studentAverages.append(",");
            }
            studentAverages.append(studentAvg);
            subject1Total += s1;
            subject2Total += s2;
            subject3Total += s3;
            studentCount++;
        }
        int classAvg1 = subject1Total / studentCount;
        int classAvg2 = subject2Total / studentCount;
        int classAvg3 = subject3Total / studentCount;
        String classAverages = classAvg1 + "," + classAvg2 + "," + classAvg3;
        String output = classAverages + " | " + studentAverages.toString();
        context.write(key, new Text(output));
    }
}


package experiment.big101;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class SMain {

    public static void main(String[] args) throws Exception {

        args = new String[] { "hdfs://master:9000/root/experiment/datas", "hdfs://master:9000/root/experiment/output" };

        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);

        job.setJarByClass(SMain.class);
        job.setMapperClass(SMapper.class);
        job.setReducerClass(SReducer.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);

        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        job.waitForCompletion(true);

    }

}

Answer 1

阿里嘎多学长整理AIGC生成，因移动端显示问题导致当前答案未能完全显示，请使用PC端查看更加详细的解答过程

MapReduce输出错误问题

你遇到了MapReduce输出错误的问题，最后的输出结果与你期望的班级学科平均分不符。下面是可能的解决方案：

1. 检查MapReduce程序的逻辑：请检查你的MapReduce程序是否正确地计算了班级学科平均分。如果你的MapReduce程序使用了错误的逻辑或算法，可能会导致输出结果错误。
2. 检查数据处理过程：请检查数据处理过程是否正确地处理了原始数据。如果数据处理过程中出现了错误，可能会导致输出结果错误。
3. 检查MapReduce程序的配置：请检查MapReduce程序的配置是否正确。如果MapReduce程序的配置错误，可能会导致输出结果错误。
4. 检查输出结果：请检查输出结果是否正确。如果输出结果正确，但是与你期望的班级学科平均分不符，可能是因为数据处理过程或MapReduce程序的逻辑错误。

以下是一个简单的MapReduce程序示例，用于计算班级学科平均分：

public class AverageScore {
    public static class MapClass extends Mapper<LongWritable, Text, Text, Float> {
        @Override
        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String[] fields = value.toString().split(",");
            String className = fields[0];
            float score = Float.parseFloat(fields[1]);
            context.write(new Text(className), new Float(score));
        }
    }

    public static class ReduceClass extends Reducer<Text, Float, Text, Float> {
        @Override
        public void reduce(Text key, Iterable<Float> values, Context context) throws IOException, InterruptedException {
            float sum = 0;
            int count = 0;
            for (Float value : values) {
                sum += value;
                count++;
            }
            float average = sum / count;
            context.write(key, new Float(average));
        }
    }

    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf, "AverageScore");
        job.setJarByClass(AverageScore.class);
        job.setMapperClass(MapClass.class);
        job.setReducerClass(ReduceClass.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Float.class);
        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }
}

这个示例程序使用MapReduce计算班级学科平均分，输入数据格式为className,score，输出结果为className,averageScore。