标签：des class blog code java ext
MapReduce TopK统计加排序中介绍的TopK在mapreduce的实现。
本案例省略的上面案例中的Sort步骤，改用TreeMap来实现获取前K个词
package TopK1;
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
/**
* 统计词频
* @author zx
* zhangxian1991@qq.com
*/
public class WordCount {
/**
* 读取单词
* @author zx
*
*/
public static class Map extends Mapper<Object,Text,Text,IntWritable>{
IntWritable count = new IntWritable(1);
@Override
protected void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
StringTokenizer st = new StringTokenizer(value.toString());
while(st.hasMoreTokens()){
String word = st.nextToken().replaceAll("\"", "").replace("'", "").replace(".", "");
context.write(new Text(word), count);
}
}
}
/**
* 统计词频
* @author zx
*
*/
public static class Reduce extends Reducer<Text,IntWritable,Text,IntWritable>{
@SuppressWarnings("unused")
@Override
protected void reduce(Text key, Iterable<IntWritable> values,Context context)
throws IOException, InterruptedException {
int count = 0;
for (IntWritable intWritable : values) {
count ++;
}
context.write(key,new IntWritable(count));
}
}
@SuppressWarnings("deprecation")
public static boolean run(String in,String out) throws IOException, ClassNotFoundException, InterruptedException{
FileUtil.deleteFile(out);
Configuration conf = new Configuration();
Job job = new Job(conf,"WordCount1");
job.setJarByClass(WordCount.class);
job.setMapperClass(Map.class);
job.setReducerClass(Reduce.class);
// 设置Map输出类型
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
// 设置Reduce输出类型
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
// 设置输入和输出目录
FileInputFormat.addInputPath(job, new Path(in));
FileOutputFormat.setOutputPath(job, new Path(out));
return job.waitForCompletion(true);
}
}
package TopK1;
import java.io.IOException;
import java.util.Comparator;
import java.util.Iterator;
import java.util.TreeMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
/**
*
* @author zx
*zhangxian1991@qq.com
*/
public class TopK {
public static class TopKMap extends Mapper<Object, Text, IntWritable, IntWritable>{
TreeMap<Integer,String> tm = new TreeMap<Integer,String>(new Comparator<Integer>() {
/**
* treeMap中的元素逆序排列
* @param o1
* @param o2
* @return
*/
@Override
public int compare(Integer o1, Integer o2) {
return o2.compareTo(o1);
}
});
int k = 0;
@Override
protected void cleanup(Context context) throws IOException,
InterruptedException {
Configuration conf = context.getConfiguration();
Path topKPath = new Path(conf.get("topKOut"));
FileSystem fs = topKPath.getFileSystem(conf);
FSDataOutputStream fsDOS = fs.create(topKPath);
Iterator<Integer> it = tm.keySet().iterator();
while(it.hasNext()){
Integer key = it.next();
String value = tm.get(key).toString();
String line = value + "\t" + key + "\r";
fsDOS.write(line.getBytes(), 0, line.length());
}
fsDOS.flush();
fsDOS.close();
}
@Override
protected void setup(Context context) throws IOException,
InterruptedException {
k = Integer.parseInt(context.getConfiguration().get("K"));
}
@Override
protected void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
String[] parts = value.toString().split("\t");
tm.put(Integer.parseInt(parts[1]),parts[0]);
if(tm.size() > k){
tm.remove(tm.lastKey());
}
}
}
@SuppressWarnings("deprecation")
public static void main(String args[]) throws ClassNotFoundException, IOException, InterruptedException{
if(args.length < 4){
throw new IllegalArgumentException("要有4个参数:1，要统计的文本文件名。2，统计后的结果路径。3，topK的结果目录，4，K");
}
FileUtil.deleteFile(args[2]);
//要统计字数的文本文件名
String in = args[0];
//统计字数后的结果
String wordCount = args[1];
in = FileUtil.loadFile(wordCount, "TopK1", in);
//如果统计字数的job完成后就开始求topK
if(WordCount.run(in, wordCount)){
int k = Integer.parseInt(args[3]);
Configuration conf = new Configuration();
FileUtil.deleteFile(args[2]);
conf.set("topKOut", args[2]);
conf.set("K", k+"");
Job job = new Job(conf,"TopK1");
job.setJarByClass(TopK.class);
job.setMapperClass(TopKMap.class);
job.setOutputKeyClass(IntWritable.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job, new Path(wordCount));
job.setOutputFormatClass(NullOutputFormat.class);
System.exit(job.waitForCompletion(true)?0:1);
}
}
}
package TopK1;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
/**
*
* @author zx
*
*/
public class FileUtil {
/**
* 上传数据文件到hdfs
* @param inputPath
* @param fileName
* @return
* @throws IOException
*/
public static String loadFile(String inputPath,String folder,String fileName) throws IOException{
//获取数据文件的全路径
if(null != folder && !"".equals(folder)){
folder = folder + "/";
}
String srcPathDir = FileUtil.class.getProtectionDomain().getCodeSource().getLocation()
.getFile() + folder + fileName;
Path srcpath = new Path("file:///" + srcPathDir);
Path dstPath = new Path(getJobRootPath(inputPath) + fileName);
Configuration conf = new Configuration();
FileSystem fs = dstPath.getFileSystem(conf);
fs.delete(dstPath, true);
fs.copyFromLocalFile(srcpath, dstPath);
fs.close();
return getJobRootPath(inputPath) + fileName;
}
/**
* 如果路径的最后不包哈“/”就加一个“/”
* @param path
* @return
*/
public static String getJobRootPath(String path){
if(path.lastIndexOf("/") == path.length()-1){
path = path.substring(0, path.lastIndexOf("/"));
}
return path.substring(0, path.lastIndexOf("/")+1);
}
public static void deleteFile(String ...filePath) throws IOException{
Configuration conf = new Configuration();
for (int i = 0; i < filePath.length; i++) {
Path path = new Path(filePath[i]);
FileSystem fs = path.getFileSystem(conf);
fs.delete(path,true);
}
}
}
MapReduceTopK TreeMap,布布扣,bubuko.com
标签：des class blog code java ext
原文地址：http://blog.csdn.net/ilovezhangxian/article/details/33319281
