java实现文件的分割与合并

标签：

无非就是io流-------------------------------------------------------

实例类：

1.抽象类

import java.io.File;
import java.io.IOException;


public abstract class PartitionFile {

/** 
* 单个文件设置的字节数 
*/  
public static long MAX_BYTE = 1024*1024*1000;


/** 
* 获取可以分割的文件数 
*  
* @param filePath 
* @param max_byte 
* @return 
*/  
public int getPartitionFileNum(long fileByte, String filePath) {  
if (MAX_BYTE < fileByte) {  
if (fileByte % MAX_BYTE == 0) {  
return (int) (fileByte / MAX_BYTE);
} else {  
return (int) (fileByte / MAX_BYTE) + 1;
}
}  
return 1;
}

/** 
* 获取文件长度 
*  
* @param file 
* @return 
* @throws IOException 
*/  
public abstract long getFileLength(File file) throws IOException;


/** 
* 分割Byte文件 
*  
* @param file 
* @throws IOException 
* @throws IOException 
*/  
public abstract String[] partitionFile(File outPath,File srcFile, int partitionFileNum) throws IOException;


/** 
* 合并文件 
* @param files 
* @param newFile 
* @throws IOException 
*/
public abstract void uniteFile(String[] files, String newFile) throws IOException;

}

2.实现类

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;


public class SplitBigTextFile extends PartitionFile{

public static void main(String[] args) throws Exception {

/*大文件分割*/
String srcFile = "H:\\proData\\datahktw3\\hk\\b_account_audit.csv";//要分割的文件
String outPath = "E:\\111111111111111111111111111\\";//分割后文件输出位置
File file = new File(srcFile);
File oPath = new File(outPath);
SplitBigTextFile partitionTextFile = new SplitBigTextFile();
int count = partitionTextFile.getPartitionFileNum(file.length(), srcFile);
String str[] = partitionTextFile.partitionFile(oPath,file, count);
System.out.println(str.length);
}

@SuppressWarnings("finally")
@Override
public long getFileLength(File file) throws IOException {

FileReader fr = null;
BufferedReader br = null;
long fileSize = 0;
try {
fr = new FileReader(file);
br = new BufferedReader(fr);
String line = br.readLine();
while (line != null) {
fileSize += line.length();
line = br.readLine();
}
} catch (FileNotFoundException ex) {
ex.printStackTrace();
} catch (IOException ex) {
ex.printStackTrace();
} finally {
if (br != null){
br.close();
}
if (fr != null)
fr.close();
return fileSize;
}
}

@Override
public String[] partitionFile(File outPath,File srcFile, int partitionFileNum) throws IOException {

if (partitionFileNum <= 0){
return null;
}
FileReader fr = null;
BufferedReader br = null;
long readNum = 0;
String[] partitions = new String[partitionFileNum];
try {
fr = new FileReader(srcFile);
br = new BufferedReader(new InputStreamReader(new FileInputStream(srcFile), "gb2312"));
int i = 1;
while (partitionFileNum > i) {
String name = "";
String fileType = "";
if (srcFile.getName().indexOf(".") != -1) {
name = srcFile.getName().substring(0,srcFile.getName().indexOf("."));
fileType = srcFile.getName().substring(srcFile.getName().lastIndexOf(".")+1,srcFile.getName().length());
} else {
name = srcFile.getName();
}

//windows文件分割??
if(outPath.getAbsolutePath().contains("\\")){
partitions[i] = outPath.getParent() + "\\" + name +"\\"+name+"_" + i+".txt";

partitions[i] = partitions[i].toUpperCase();
}else{
//linux文件分割??
partitions[i] = outPath.getParent() + "/" + name +"/"+name+"_" + i+"."+fileType;
}

System.out.println(partitions[i]);

File wfile = new File(partitions[i]);

if (!wfile.exists()) {
wfile.getParentFile().mkdirs();
wfile.createNewFile();
}


FileWriter fw = new FileWriter(wfile,false);
BufferedWriter bw = new BufferedWriter(fw);


String line = br.readLine();
int flush=0;
String []tem = null;
StringBuffer sb = null;

while (line != null) {

sb = new StringBuffer();

if (line.trim().length() == 0) {
line = br.readLine();
continue;
}

tem = line.split("-");
for(int n=0;n<tem.length;n++){
sb.append(tem[n]+",");
}

readNum += sb.toString().length();

if (i + 1 == partitionFileNum) {
bw.write(line);
bw.newLine();


} else {
if (readNum >= MAX_BYTE) {
bw.write(sb.toString());
bw.newLine();
break;
} else {
if(sb.length()>0){
bw.write(sb.toString());
bw.newLine();
}
}


}
line = br.readLine();
if(flush%1000==0){
bw.flush();
}
}

bw.flush();
fw.flush();
bw.close();
fw.close();


if(sb!=null && sb.length()==0){
wfile.delete();
}



readNum = 0;
i++;
}
} finally {
try {
if (br != null){
br.close();
}
if (fr != null){
fr.close();
}
} catch (IOException e) {
e.printStackTrace();
} finally {
br = null;
fr = null;
}
}
return partitions;
}

@Override
public void uniteFile(String[] files, String newFile) throws IOException {
File wFile = new File(newFile);
FileWriter writer = null;
BufferedWriter bufferedWriter = null;
try {
writer = new FileWriter(wFile,false);
bufferedWriter = new BufferedWriter(writer);
for (int i = 0; i < files.length; i++) {
File rFile = new File(files[i]);
FileReader reader = new FileReader(rFile);
BufferedReader bufferedReader = new BufferedReader(reader);
String line = bufferedReader.readLine();
while (line != null) {
if (line.trim().length() == 0) {
line = bufferedReader.readLine();
continue;
}
bufferedWriter.write(line);
bufferedWriter.newLine();
line = bufferedReader.readLine();
}
bufferedWriter.flush();
writer.flush();
}
} finally {
if (bufferedWriter != null){
bufferedWriter.close();
bufferedWriter = null;
}
if (writer != null){
writer.close();
}
writer = null;
}
}
}

java实现文件的分割与合并

标签：

原文地址：http://blog.csdn.net/lbf5210/article/details/51331262