基于Hadoop2.6.0 + ICTCLAS2015的并行化中文分词

时间：2015-10-28 17:03:43 阅读：299 评论：0 收藏：0 [点我收藏+]

标签：

1、在Linux下安装ICTCLAS2015

•1) 环境

　　Eclipse、Linux、ICTCLAS2015、jna-platform-4.1.0.jar（ JNA类库）

•2) 安装ICTCLAS2015

　　在Linux下的 Eclipse中新建MapReduce Project，假设工程名为RF；

　　下载并解压ICTCLAS2015,将ICTCLAS中lib目录下libNLPIR.so文件（对应Linux 32位）拷入到RF下的src文件夹下(注意该路径下，还应该有log4j文件)；

　　将ICTCLAS2015目录下的Data文件夹整个复制到项目RF中，放在根目录下；

　　导入JNA类库 jna-platform-4.1.0.jar

　　项目结构图，如下：

2、使用JNA调用C++接口

CLibrary类:

package nlpir2015.rf.cs.swjtu;
import com.sun.jna.Library;


//import com.sun.jna.Library;
public interface CLibrary extends Library{
//初始化
    public int NLPIR_Init(String sDataPath, int encoding, String sLicenceCode);
    //对字符串进行分词
    public String NLPIR_ParagraphProcess(String sSrc, int bPOSTagged);
    //对TXT文件内容进行分词
    public double NLPIR_FileProcess(String sSourceFilename,String sResultFilename, int bPOStagged);
    //从字符串中提取关键词 
    public String NLPIR_GetKeyWords(String sLine, int nMaxKeyLimit,boolean bWeightOut);
    //从TXT文件中提取关键词 
    public String NLPIR_GetFileKeyWords(String sLine, int nMaxKeyLimit,boolean bWeightOut);
    //添加单条用户词典
    public int NLPIR_AddUserWord(String sWord);
    //删除单条用户词典
    public int NLPIR_DelUsrWord(String sWord);
    //从TXT文件中导入用户词典
    public int NLPIR_ImportUserDict(String sFilename);
    //将用户词典保存至硬盘
    public int NLPIR_SaveTheUsrDic();
    //从字符串中获取新词
    public String NLPIR_GetNewWords(String sLine, int nMaxKeyLimit, boolean bWeightOut);
    //从TXT文件中获取新词
    public String NLPIR_GetFileNewWords(String sTextFile,int nMaxKeyLimit, boolean bWeightOut);
    //获取一个字符串的指纹值
    public long NLPIR_FingerPrint(String sLine);
    //设置要使用的POS map
    public int NLPIR_SetPOSmap(int nPOSmap);
    //获取报错日志
    public String NLPIR_GetLastErrorMsg();
    //退出
    public void NLPIR_Exit();    
}

3、分词类feici

package nlpir2015.rf.cs.swjtu;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import com.sun.jna.Native;

public class fenci {
 //Windows下的加载方式。如果需要支持Linux，需要修改这一行为libNLPIR.so的路径。
	String Path=System.getProperty("user.dir").toString();
    CLibrary Instance = (CLibrary)Native.loadLibrary("NLPIR", CLibrary.class);
    private boolean initFlag = false;
    
    public boolean init(){
        String argu = null;
        // String system_charset = "GBK";//GBK----0
        int charset_type = 1;
        
        int init_flag = Instance.NLPIR_Init(argu, charset_type, "0");
        String nativeBytes = null;

        if (0 == init_flag) {
            nativeBytes = Instance.NLPIR_GetLastErrorMsg();
            System.err.println("初始化失败！fail reason is "+nativeBytes);
            return false;
        }
        initFlag = true;
        return true;
    }
    
    public boolean unInit(){
        try {
            Instance.NLPIR_Exit();
        } catch (Exception e) {
            System.out.println(e);
            return false;
        }
        initFlag = false;
        return true;
    }
    
    public String parseSen(String str){
        String nativeBytes = null;
        try {
            nativeBytes = Instance.NLPIR_ParagraphProcess(str, 0);
        } catch (Exception ex) {
            // TODO Auto-generated catch block
            ex.printStackTrace();
        }
        return nativeBytes;
    }
    
    public CLibrary getInstance() {
        return Instance;
    }

    public boolean isInitFlag() {
        return initFlag;
    }
    
}

　　4、编写MapReduce函数

package segwords.rf.cs.swjtu;

import java.io.IOException;

import nlpir2015.rf.cs.swjtu.fenci;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;

public class SegWords {
	public static class SegWordsMap extends  Mapper<Object, Text, Text, Text> {  
		fenci tt = new fenci(); //
	
	protected void setup(Context context) throws IOException, InterruptedException {  
		 
	        tt.init();
		
      }  
	
	public void map(Object key, Text value,Context context) throws IOException {  
	
	 String line = value.toString();  
	 line = tt.parseSen(line.replaceAll("[\\pP‘’“”]", ""));
	 
	 	try {
	 		context.write(new Text(""),new Text(line));
	 		System.out.println(line);
	 	} catch (InterruptedException e) {
		// TODO Auto-generated catch block
	 		e.printStackTrace();
		}
	 }
	
	protected void cleanup(Context context){
		 tt.unInit();
	}
	}  
	public static class SegWordsReduce extends  Reducer<Text, Text, Text, Text> {  
	public void reduce(Text key, Text value, Context context) throws IOException, NumberFormatException, InterruptedException {  

			context.write(key,  value);  // 
		}  
	}  
}

package segwords.rf.cs.swjtu;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import segwords.rf.cs.swjtu.SegWords.SegWordsMap;
import segwords.rf.cs.swjtu.SegWords.SegWordsReduce;

public class tcMain {

	/**
	 * @param args
	 * @throws IOException 
	 * @throws InterruptedException 
	 * @throws ClassNotFoundException 
	 */
	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		// TODO Auto-generated method stub
		//JOB 1  TF
		Path segIn = new Path("hdfs://localhost:9000/input/segIn");
		Path segOut = new Path("hdfs://localhost:9000/output/segOut");
		 
		Configuration conf1=new Configuration();
		Job job1=new Job(conf1,"segWords");
		
		FileSystem fs1 = FileSystem.get(conf1); 
		if(fs1.exists(segOut)){
			fs1.delete(segOut);
		}
		fs1.close();
		
		job1.setJarByClass(SegWords.class);
		
		job1.setMapperClass( SegWordsMap.class);  
		job1.setReducerClass( SegWordsReduce.class);  
		
		job1.setOutputKeyClass(Text.class);  
		job1.setOutputValueClass(Text.class);  
		  
		FileInputFormat.addInputPath(job1, segIn);
		FileOutputFormat.setOutputPath(job1, segOut);  
		
		//System.exit(job1.waitForCompletion(true)?0:1);
		job1.waitForCompletion(true);

	}

}

　　5、并行分词结果

　　1）输入文档（2个文档模拟2个split）

　　技术分享

　　2）结果

15/10/28 01:07:12 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
15/10/28 01:07:16 INFO Configuration.deprecation: session.id is deprecated. Instead, use dfs.metrics.session-id
15/10/28 01:07:16 INFO jvm.JvmMetrics: Initializing JVM Metrics with processName=JobTracker, sessionId=
15/10/28 01:07:16 WARN mapreduce.JobSubmitter: Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this.
15/10/28 01:07:18 INFO input.FileInputFormat: Total input paths to process : 2
15/10/28 01:07:18 INFO mapreduce.JobSubmitter: number of splits:2
15/10/28 01:07:18 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_local598241144_0001
15/10/28 01:07:19 INFO mapreduce.Job: The url to track the job: http://localhost:8080/
15/10/28 01:07:19 INFO mapreduce.Job: Running job: job_local598241144_0001
15/10/28 01:07:19 INFO mapred.LocalJobRunner: OutputCommitter set in config null
15/10/28 01:07:19 INFO mapred.LocalJobRunner: OutputCommitter is org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
15/10/28 01:07:19 INFO mapred.LocalJobRunner: Waiting for map tasks
15/10/28 01:07:19 INFO mapred.LocalJobRunner: Starting task: attempt_local598241144_0001_m_000000_0
15/10/28 01:07:19 INFO mapred.Task: Using ResourceCalculatorProcessTree : [ ]
15/10/28 01:07:20 INFO mapreduce.Job: Job job_local598241144_0001 running in uber mode : false
15/10/28 01:07:20 INFO mapreduce.Job: map 0% reduce 0%
15/10/28 01:07:20 INFO mapred.MapTask: Processing split: hdfs://localhost:9000/input/segIn/segtest01:0+2359
15/10/28 01:07:23 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584)
15/10/28 01:07:23 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 100
15/10/28 01:07:23 INFO mapred.MapTask: soft limit at 83886080
15/10/28 01:07:23 INFO mapred.MapTask: bufstart = 0; bufvoid = 104857600
15/10/28 01:07:23 INFO mapred.MapTask: kvstart = 26214396; length = 6553600
15/10/28 01:07:23 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
15/10/28 01:07:40 INFO mapred.LocalJobRunner:
联合早报网综合讯据中新网报道中国各地 2015年前三季度居民收入数据近日陆续出炉截至 10月 26日中国至少已有 25 省份公布数据仅上海北京全体居民人均可支配收入超过 3万元大关而城乡差距有继续扩大的趋势
据中国国家统计局数据前三季度全国居民人均可支配收入 16367 元按常住地分城镇居民人均可支配收入 23512 元扣除价格因素实际增长 68 农村居民人均可支配收入 8297 元扣除价格因素实际增长 81
据报道截至 10月 26日除了吉林辽宁黑龙江内蒙古山西西藏外其余 25 省份均已公布了当地的前三季度居民人均可支配收入数据
在全体居民人均可支配收入方面位居前两位的为上海和北京其人均收入均突破 3万元其中上海前三季度居民人均可支配收入为 37568 元北京前三季度居民人均可支配收入为 36047 元这两地也是目前全国仅有的居民人均可支配收入超过 3万元大关的地区
如果单单从在城镇居民收入方面在已公布数据的 25 个地区中上海城镇居民人均可支配收入最高达到了 39686 元北京排第二为 39142 元浙江位列第三为 33464 元这三地也是目前全国仅有的城镇居民人均可支配收入突破 3万元大关的地区而城镇居民人均可支配收入暂排在末位是甘肃省仅为 176385 元收入最高和最低地区的差异超 2万元
据统计上海北京浙江江苏广东天津福建山东这 8 个地区的城镇居民人均可支配收入超过了全国平均水平而在农村居民收入方面上海浙江北京天津江苏山东辽宁福建海南河北这 10 个地区的农村居民人均可支配收入超过了全国平均水平
在已公布数据的 25 个地区中上海农村居民人均可支配收入最高达到了 19237 元浙江排第二为 17004 元北京位列第三为 16450 元但从城乡居民人均收入倍差来看与全国城乡居民人均收入倍差 283 相比新疆陕西广西云南贵州宁夏青海甘肃这 8 地区的城乡居民人均收入倍差超过了全国水平
联合早报网实习编辑关丽明
15/10/28 01:07:41 INFO mapred.MapTask: Starting flush of map output
15/10/28 01:07:41 INFO mapred.MapTask: Spilling map output
15/10/28 01:07:41 INFO mapred.MapTask: bufstart = 0; bufend = 2552; bufvoid = 104857600
15/10/28 01:07:41 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 26214368(104857472); length = 29/6553600
15/10/28 01:07:41 INFO mapred.MapTask: Finished spill 0
15/10/28 01:07:41 INFO mapred.Task: Task:attempt_local598241144_0001_m_000000_0 is done. And is in the process of committing
15/10/28 01:07:42 INFO mapred.LocalJobRunner: map
15/10/28 01:07:42 INFO mapred.Task: Task ‘attempt_local598241144_0001_m_000000_0‘ done.
15/10/28 01:07:42 INFO mapred.LocalJobRunner: Finishing task: attempt_local598241144_0001_m_000000_0
15/10/28 01:07:42 INFO mapred.LocalJobRunner: Starting task: attempt_local598241144_0001_m_000001_0
15/10/28 01:07:42 INFO mapred.Task: Using ResourceCalculatorProcessTree : [ ]
15/10/28 01:07:42 INFO mapreduce.Job: map 100% reduce 0%
15/10/28 01:07:42 INFO mapred.MapTask: Processing split: hdfs://localhost:9000/input/segIn/segtest00:0+2314
15/10/28 01:07:42 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584)
15/10/28 01:07:42 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 100
15/10/28 01:07:42 INFO mapred.MapTask: soft limit at 83886080
15/10/28 01:07:42 INFO mapred.MapTask: bufstart = 0; bufvoid = 104857600
15/10/28 01:07:42 INFO mapred.MapTask: kvstart = 26214396; length = 6553600
15/10/28 01:07:42 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
联合早报网讯中国外交部证实美国军舰今天进入南沙群岛渚碧礁 12 海里航行中国外交部发言人陆慷说今天美国拉森号军舰 USS Lassen 未经中国政府允许非法进入中国南沙群岛有关岛礁邻近海域中国有关部门对美国舰艇实施了监视跟踪和警告
陆慷表示美方军舰有关行为威胁中国主权和安全利益危及岛礁人员及设施安全损害地区和平稳定中方对此表示强烈不满和坚决反对
陆慷说正如中方多次强调的中国对南沙群岛及其附近海域拥有无可争辩的主权中国在南海的主权和相关权利是在长期的历史过程中形成的为历代中国政府所坚持中方在自己的领土上开展建设是主权范围内的事不针对不影响任何国家不会对各国依国际法在南海享有的航行和飞越自由造成任何影响
他强调中方强烈敦促美方认真对待中方严正交涉立即纠正错误不得采取任何威胁中方主权和安全利益的危险挑衅行为恪守在领土主权争议问题上不持立场的承诺以免进一步损害中美关系和地区和平稳定
据早前报道中国外交部长王毅今天在北京出席中日韩研讨会期间针对有境外媒体问及美军将派遣军舰进入中国南海岛礁 12 海里一事表示关于此事中方正在核实当中如果属实奉劝美方三思而后行不要轻举妄动不要无事生非
据报道美国国防部周一 26日宣布将于 24 小时内派遣导弹驱逐舰到中国在南海人工岛礁 12 海里水域的范围内航行
路透社报道美军将派出拉森号导弹驱逐舰前往渚碧礁及美济礁一带水域航行并会同时派出 P8A 侦察机到该海域巡逻美国国防部发言人表示美军的行动不会是一次性的而会定期进行这次行动并非针对中国
美国国防部一名官员之后证实美国海军派出的导弹驱逐舰拉森号周二早上驶到中国在南中国海兴建的多个人工岛礁 12 海浬范围并逗留数小时美军之后表示行动已经完成拉森号离开无发生任何事故
联合早报网编辑沈茂华
15/10/28 01:07:51 INFO mapred.LocalJobRunner:
15/10/28 01:07:51 INFO mapred.MapTask: Starting flush of map output
15/10/28 01:07:51 INFO mapred.MapTask: Spilling map output
15/10/28 01:07:51 INFO mapred.MapTask: bufstart = 0; bufend = 2534; bufvoid = 104857600
15/10/28 01:07:51 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 26214364(104857456); length = 33/6553600
15/10/28 01:07:51 INFO mapred.MapTask: Finished spill 0
15/10/28 01:07:51 INFO mapred.Task: Task:attempt_local598241144_0001_m_000001_0 is done. And is in the process of committing
15/10/28 01:07:51 INFO mapred.LocalJobRunner: map
15/10/28 01:07:51 INFO mapred.Task: Task ‘attempt_local598241144_0001_m_000001_0‘ done.
15/10/28 01:07:51 INFO mapred.LocalJobRunner: Finishing task: attempt_local598241144_0001_m_000001_0
15/10/28 01:07:51 INFO mapred.LocalJobRunner: map task executor complete.
15/10/28 01:07:52 INFO mapred.LocalJobRunner: Waiting for reduce tasks
15/10/28 01:07:52 INFO mapred.LocalJobRunner: Starting task: attempt_local598241144_0001_r_000000_0
15/10/28 01:07:52 INFO mapred.Task: Using ResourceCalculatorProcessTree : [ ]
15/10/28 01:07:52 INFO mapred.ReduceTask: Using ShuffleConsumerPlugin: org.apache.hadoop.mapreduce.task.reduce.Shuffle@1f0dbfd
15/10/28 01:07:53 INFO reduce.MergeManagerImpl: MergerManager: memoryLimit=178821520, maxSingleShuffleLimit=44705380, mergeThreshold=118022208, ioSortFactor=10, memToMemMergeOutputsThreshold=10
15/10/28 01:07:53 INFO reduce.EventFetcher: attempt_local598241144_0001_r_000000_0 Thread started: EventFetcher for fetching Map Completion Events
15/10/28 01:07:54 INFO reduce.LocalFetcher: localfetcher#1 about to shuffle output of map attempt_local598241144_0001_m_000001_0 decomp: 2568 len: 2572 to MEMORY
15/10/28 01:07:54 INFO reduce.InMemoryMapOutput: Read 2568 bytes from map-output for attempt_local598241144_0001_m_000001_0
15/10/28 01:07:54 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 2568, inMemoryMapOutputs.size() -> 1, commitMemory -> 0, usedMemory ->2568
15/10/28 01:07:54 INFO reduce.LocalFetcher: localfetcher#1 about to shuffle output of map attempt_local598241144_0001_m_000000_0 decomp: 2583 len: 2587 to MEMORY
15/10/28 01:07:54 INFO reduce.InMemoryMapOutput: Read 2583 bytes from map-output for attempt_local598241144_0001_m_000000_0
15/10/28 01:07:54 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 2583, inMemoryMapOutputs.size() -> 2, commitMemory -> 2568, usedMemory ->5151
15/10/28 01:07:54 INFO reduce.EventFetcher: EventFetcher is interrupted.. Returning
15/10/28 01:07:54 INFO mapred.LocalJobRunner: 2 / 2 copied.
15/10/28 01:07:54 INFO reduce.MergeManagerImpl: finalMerge called with 2 in-memory map-outputs and 0 on-disk map-outputs
15/10/28 01:07:54 INFO mapred.Merger: Merging 2 sorted segments
15/10/28 01:07:54 INFO mapred.Merger: Down to the last merge-pass, with 2 segments left of total size: 5145 bytes
15/10/28 01:07:54 INFO reduce.MergeManagerImpl: Merged 2 segments, 5151 bytes to disk to satisfy reduce memory limit
15/10/28 01:07:54 INFO reduce.MergeManagerImpl: Merging 1 files, 5153 bytes from disk
15/10/28 01:07:54 INFO reduce.MergeManagerImpl: Merging 0 segments, 0 bytes from memory into reduce
15/10/28 01:07:54 INFO mapred.Merger: Merging 1 sorted segments
15/10/28 01:07:54 INFO mapred.Merger: Down to the last merge-pass, with 1 segments left of total size: 5146 bytes
15/10/28 01:07:55 INFO mapred.LocalJobRunner: 2 / 2 copied.
15/10/28 01:07:57 INFO Configuration.deprecation: mapred.skip.on is deprecated. Instead, use mapreduce.job.skiprecords
15/10/28 01:07:58 INFO mapred.LocalJobRunner: reduce > reduce
15/10/28 01:07:59 INFO mapreduce.Job: map 100% reduce 100%
15/10/28 01:08:01 INFO mapred.LocalJobRunner: reduce > reduce
15/10/28 01:08:05 INFO mapred.Task: Task:attempt_local598241144_0001_r_000000_0 is done. And is in the process of committing
15/10/28 01:08:05 INFO mapred.LocalJobRunner: reduce > reduce
15/10/28 01:08:05 INFO mapred.Task: Task attempt_local598241144_0001_r_000000_0 is allowed to commit now
15/10/28 01:08:05 INFO output.FileOutputCommitter: Saved output of task ‘attempt_local598241144_0001_r_000000_0‘ to hdfs://localhost:9000/output/segOut/_temporary/0/task_local598241144_0001_r_000000
15/10/28 01:08:05 INFO mapred.LocalJobRunner: reduce > reduce
15/10/28 01:08:05 INFO mapred.Task: Task ‘attempt_local598241144_0001_r_000000_0‘ done.
15/10/28 01:08:05 INFO mapred.LocalJobRunner: Finishing task: attempt_local598241144_0001_r_000000_0
15/10/28 01:08:05 INFO mapred.LocalJobRunner: reduce task executor complete.
15/10/28 01:08:07 INFO mapreduce.Job: Job job_local598241144_0001 completed successfully
15/10/28 01:08:07 INFO mapreduce.Job: Counters: 38
File System Counters
FILE: Number of bytes read=33097684
FILE: Number of bytes written=34155860
FILE: Number of read operations=0
FILE: Number of large read operations=0
FILE: Number of write operations=0
HDFS: Number of bytes read=11705
HDFS: Number of bytes written=5059
HDFS: Number of read operations=28
HDFS: Number of large read operations=0
HDFS: Number of write operations=8
Map-Reduce Framework
Map input records=17
Map output records=17
Map output bytes=5086
Map output materialized bytes=5159
Input split bytes=216
Combine input records=0
Combine output records=0
Reduce input groups=1
Reduce shuffle bytes=5159
Reduce input records=17
Reduce output records=17
Spilled Records=34
Shuffled Maps =2
Failed Shuffles=0
Merged Map outputs=2
GC time elapsed (ms)=179
CPU time spent (ms)=0
Physical memory (bytes) snapshot=0
Virtual memory (bytes) snapshot=0
Total committed heap usage (bytes)=457125888
Shuffle Errors
BAD_ID=0
CONNECTION=0
IO_ERROR=0
WRONG_LENGTH=0
WRONG_MAP=0
WRONG_REDUCE=0
File Input Format Counters
Bytes Read=4673
File Output Format Counters
Bytes Written=5059

http://blog.csdn.net/luoyhang003/article/details/44586731

http://jingyan.baidu.com/article/9158e0004251b4a2541228e5.html

基于Hadoop2.6.0 + ICTCLAS2015的并行化中文分词

标签：

原文地址：http://www.cnblogs.com/merru/p/4917665.html

踩

(0)

评论一句话评论（0）

分享档案

更多>

2021年07月29日 (22)
2021年07月28日 (40)
2021年07月27日 (32)
2021年07月26日 (79)
2021年07月23日 (29)
2021年07月22日 (30)
2021年07月21日 (42)
2021年07月20日 (16)
2021年07月19日 (90)
2021年07月16日 (35)

周排行