标签:herf val class 阶段 span spl dex void output
数据:
A:B,C,D,F,E,O B:A,C,E,K C:F,A,D,I D:A,E,F,L E:B,C,D,M,L F:A,B,C,D,E,O,M G:A,C,D,E,F H:A,C,D,E,O I:A,O J:B,O K:A,C,D L:D,E,F M:E,F,G O:A,H,I,J,K
求哪些人两两之间是互粉好友,形如:A的好友有B,B的好友有A 。 那么A和B就是互粉好友。
思路:
对每一行数据进行组合输出 (person-person,1),
然后再Reducer阶段进行统计,等于2的就是互粉好友对;
问题:将B-A转换成A-B这种形式;
方案:比较两个字符之间的大小,小的在前;
import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; public class EachOtherFriend { public static class MyMapper extends Mapper<LongWritable, Text, Text, IntWritable>{ @Override protected void map(LongWritable key, Text value,Context context) throws IOException, InterruptedException { String[] lines = value.toString().split(":"); char person = lines[0].charAt(0); for (String str : lines[1].split(",")) { char friend = str.charAt(0); String per_per = ""; if(person > friend){ per_per += friend+"-"+person; }else{ per_per += person+"-"+friend; } context.write(new Text(per_per), new IntWritable(1)); } } } public static class MyReducer extends Reducer<Text, IntWritable, Text, NullWritable>{ @Override protected void reduce(Text key, Iterable<IntWritable> values,Context context) throws IOException, InterruptedException { int num = 0; for (IntWritable it : values) { num++; } if(num > 1){ context.write(key, null); } } } public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Job job = Job.getInstance(conf); job.setJarByClass(EachOtherFriend.class); job.setMapperClass(MyMapper.class); job.setReducerClass(MyReducer.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(NullWritable.class); FileInputFormat.setInputPaths(job, new Path("G:/files/input")); FileOutputFormat.setOutputPath(job, new Path("G:/files/output")); boolean isDone = job.waitForCompletion(true); System.exit(isDone ? 0:1); } }
标签:herf val class 阶段 span spl dex void output
原文地址:https://www.cnblogs.com/break-lian/p/9780133.html