问题代码(wordcount)

问答 鲁西黄牛 ⋅ 于 2017-02-15 13:15:33 ⋅ 2884 阅读

package com.hadoop.mrdemo;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class WordCountJob extends Configured implements Tool {

public static class WordCountMapper extends Mapper<LongWritable, Text, Text, LongWritable> {

    private Text word = new Text();
    private LongWritable one = new LongWritable(1);
    @Override
    protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, LongWritable>.Context context)
            throws IOException, InterruptedException {
        String[] words = StringUtils.split(value.toString(),' ');
        for(String w : words){
            word.set(w);
            context.write(word, one);
        }
    }

}

public static class WordCountReducer extends Reducer<Text,LongWritable,Text,LongWritable>{

    private LongWritable result = new LongWritable();

    @Override
    protected void reduce(Text key, Iterable<LongWritable> values,
            Reducer<Text, LongWritable, Text, LongWritable>.Context context) throws IOException, InterruptedException {
        long sum = 0L;
        for(LongWritable v : values){
            sum +=v.get();
        }
        result.set(sum);
        context.write(key, result);
    }

}

@Override
public int run(String[] args) throws Exception  {
    // TODO Auto-generated method stub
    if(args.length <2){
        System.out.println("<input dir> <output dir>");
        return 2;
    }
    Job job =Job.getInstance(getConf(), "WordCountJob");
    job.setJarByClass(WordCountJob.class);
    job.setMapperClass(WordCountMapper.class);
    job.setReducerClass(WordCountReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    return job.waitForCompletion(true) ? 0:1 ;
}

public static void main(String[] args) throws Exception  {
    // TODO Auto-generated method stub
    int res = ToolRunner.run(new WordCountJob(), args);
    System.exit(res);
}

}

成为第一个点赞的人吧 :bowtie:
回复数量: 0
    暂无评论~~
    • 请注意单词拼写,以及中英文排版,参考此页
    • 支持 Markdown 格式, **粗体**、~~删除线~~、`单行代码`, 更多语法请见这里 Markdown 语法
    • 支持表情,可用Emoji的自动补全, 在输入的时候只需要 ":" 就可以自动提示了 :metal: :point_right: 表情列表 :star: :sparkles:
    • 上传图片, 支持拖拽和剪切板黏贴上传, 格式限制 - jpg, png, gif,教程
    • 发布框支持本地存储功能,会在内容变更时保存,「提交」按钮点击时清空
    Ctrl+Enter