码迷,mamicode.com
首页 > 其他好文 > 详细

数据清洗与数据处理

时间:2019-11-13 22:12:52      阅读:117      评论:0      收藏:0      [点我收藏+]

标签:contex   cer   style   alt   asi   set   super   writable   根据   

技术图片

 

 

package mapreduce;                                                                                                       
import java.io.IOException;                                                                                              
import org.apache.hadoop.conf.Configuration;                                                                             
import org.apache.hadoop.fs.Path;                                                                                        
import org.apache.hadoop.io.IntWritable;                                                                                 
import org.apache.hadoop.io.Text;                                                                                        
import org.apache.hadoop.mapreduce.Job;                                                                                  
import org.apache.hadoop.mapreduce.Mapper;                                                                               
import org.apache.hadoop.mapreduce.Reducer;                                                                              
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;                                                            
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;                                                            
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;                                                          
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;                                                          
public class OneSort {                                                                                                   
    public static class Map extends Mapper<Object , Text , IntWritable,Text >{                                           
    private static Text goods=new Text();                                                                                
    private static IntWritable num=new IntWritable();                                                                    
    public void map(Object key,Text value,Context context) throws IOException, InterruptedException{                     
    String line=value.toString();                                                                                        
    String arr[]=line.split("\t");                                                                                       
    num.set(Integer.parseInt(arr[1]));                                                                                   
    goods.set(arr[0]);                                                                                                   
    context.write(num,goods);                                                                                            
    }                                                                                                                    
    }                                                                                                                    
    public static class Reduce extends Reducer< IntWritable, Text, IntWritable, Text>{                                   
    private static IntWritable result= new IntWritable();                                                                
    public void reduce(IntWritable key,Iterable<Text> values,Context context) throws IOException, InterruptedException{  
        for(Text val:values){                                                                                            
        context.write(key,val);                                                                                          
        }                                                                                                                
        }
    }
    public static class IntWritableDecreasingComparator extends    IntWritable.Comparator 
    {        
        public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) 
        {            
            return -super.compare(b1, s1, l1, b2, s2, l2);        
            }    
        }

                                                                                                                        
        public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException{         
        Configuration conf=new Configuration();                                                                          
        Job job =new Job(conf,"OneSort");                                                                                
        job.setJarByClass(OneSort.class);                                                                                
        job.setMapperClass(Map.class);                                                                                   
        job.setReducerClass(Reduce.class);                                                                               
        job.setOutputKeyClass(IntWritable.class);                                                                        
        job.setOutputValueClass(Text.class);                                                                             
        job.setInputFormatClass(TextInputFormat.class);                                                                  
        job.setOutputFormatClass(TextOutputFormat.class);                                                                
        Path in=new Path("hdfs://192.168.43.114:9000/mymapreduce3/in/one");                                          
        Path out=new Path("hdfs://192.168.43.114:9000/mymapreduce3/out");                                                     
        FileInputFormat.addInputPath(job,in);                                                                            
        FileOutputFormat.setOutputPath(job,out);   
        job.setSortComparatorClass(IntWritableDecreasingComparator.class);
        System.exit(job.waitForCompletion(true) ? 0 : 1);                                                                
                                                                                                                         
        }                                                                                                                
        }                                                                                                                
                                                                                                                         
利用mapreduce的wordcount程序进行id的计算,相同id合并并计数,之后 将输出的文件根据次数降序,
因为mapreduce的排序是是默认升序排序,所以需要写排序类重写降序类,最后将输出结果存到hive与mysql中。

 

数据清洗与数据处理

标签:contex   cer   style   alt   asi   set   super   writable   根据   

原文地址:https://www.cnblogs.com/Evak/p/11853356.html

(0)
(0)
   
举报
评论 一句话评论(0
登录后才能评论!
© 2014 mamicode.com 版权所有  联系我们:gaon5@hotmail.com
迷上了代码!