import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.RandomAccessFile; import java.util.StringTokenizer; /* * 文本格式:已分词的中文文本,空格分割。有若干行,每行为一个段落。 * 功能:遍历文档,逐个返回词语。 * 两种模式: * 1 到文档末尾后,结束 * 2 到文档末尾后,从头再读。 / public class WordReader { static final int normalMode = 0; static final int againMode = 1; int currentMode = 0; //BufferedReader br=null; RandomAccessFile raf= null; StringTokenizer tokenizer = null; String nextWord=null; int currentLine = 0; int allCounts = 0; public WordReader(String fileName) throws IOException { File file=new File(fileName); //br=new BufferedReader(new InputStreamReader(new FileInputStream(file),"utf-8")); raf = new RandomAccessFile(file,"r") ; } private boolean hasNextWord() throws IOException { if( tokenizer!=null && tokenizer.hasMoreTokens()) { nextWord = tokenizer.nextToken(); return true; } else { String line=raf.readLine(); if(line == null) { if(currentMode == normalMode) return false; else //从头再来 { raf.seek(0); return hasNextWord();//递归 } } tokenizer = null; line = new String(line.getBytes("iso8859-1"),"utf-8"); tokenizer= new StringTokenizer(line," "); return hasNextWord();//递归 } } private String getNextWord() throws IOException { if(nextWord != null) { String word = nextWord; nextWord = null; allCounts ++; return word; } else if(hasNextWord()) { return getNextWord(); } else return null; } public static void main(String[] args) throws IOException { // TODO Auto-generated method stub WordReader wordReader = new WordReader("/home/linger/sources/ParaModel/electronic_seg.txt"); wordReader.currentMode = WordReader.againMode; //while(wordReader.hasNextWord())//共10329309个词 for(int i=0;i<10329319;i++)//文本从头读 { System.out.println(wordReader.getNextWord()); } System.out.println(wordReader.allCounts); } }
用randomaccessfile类很容易操作文件指针。
但是遇到中文乱码问题,参考了这里http://blog.chinaunix.net/uid-15490606-id-211958.html,解决了。
line = new String(line.getBytes("iso8859-1"),"utf-8");
对编码不是很精通,有时见看看这个http://blog.sina.com.cn/s/blog_673c81990100t1lc.html。
原文地址:http://blog.csdn.net/lingerlanlan/article/details/37932803