标签：读取csv文件
1.TestFilingTextLoadFareClassProcessorForIntegration
public class TestFilingTextLoadFareClassProcessorForIntegration {
protected static MockDBDataGenerationUtil dbDataGenerationUtil;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
dbDataGenerationUtil = new MockDBDataGenerationUtil(
"/com/atpco/textload/processor/TestFilingTextLoadFareClassProcessorForIntegration.csv");
}
@Test
public void test() throws Exception {
dbDataGenerationUtil.exceuteStatement("DELETE_RULE_SEGMENT");
System.out.println();
dbDataGenerationUtil.exceuteStatement("FTNT_GOVT_CRRN");
}
}
2.MockDBDataGenerationUtil
public class MockDBDataGenerationUtil {
private Map<String, ArrayList<String>> statementList;
/**
* Variable used to find statement ID
*/
private boolean readFileFromLocalDrive = false;
public MockDBDataGenerationUtil(String csvFileName) throws Exception {
super();
setStatementList(csvFileName);
}
private void setStatementList(String csvFileName) throws Exception {
if (this.statementList == null) {
statementList = new HashMap<String, ArrayList<String>>();
} else {
statementList.clear();
}
createSQLStatements(csvFileName);
}
private void createSQLStatements(String csvFileName) throws Exception {
BufferedReader bufRdr = null;
if (this.readFileFromLocalDrive) {
bufRdr = getBufferedReaderFormLocal(csvFileName);
} else {
bufRdr = getBufferedReader(csvFileName);
}
String line = null;
boolean bNewRecordInStatement = false;
boolean bVariableNameInNewRecord = false;
ArrayList<String> currStatement = null;
String currStatementsID = "";
String currTableNames = "";
int size = 0;
ArrayList<String> currVariableNames = new ArrayList<String>();
ArrayList<String> currValues = new ArrayList<String>();
while ((line = bufRdr.readLine()) != null) {
// keep a space for empty value
String value = " ";
// System.out.println(line);
// pass empty line
// Excel create empty line like
// ,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
String tmp0 = line.replace(‘,‘, ‘ ‘);
if (tmp0.trim().length() > 0) {
StringTokenizer st = new StringTokenizer(line, ",");
int col = 0;
while (st.hasMoreTokens()) {
value = st.nextToken();
if (value.trim().equalsIgnoreCase("STATEMENT ID")) {
bNewRecordInStatement = false;
// set statement ID
value = st.nextToken();
currStatement = setStatement(value);
currStatementsID = value;
size = 0;
currVariableNames.clear();
break;
} else if (value.trim().equalsIgnoreCase("TABLE NAME")) {
// set table name
value = st.nextToken();
// next row will be variable names in a record
bVariableNameInNewRecord = true;
currTableNames = value;
break;
} else if (bVariableNameInNewRecord) {
if (col == 0) {
size = countVariables(line);
}
col++;
if (value == null || value.trim().length() == 0) {
if (col == size) {
// last column has space(s)
size--;
bVariableNameInNewRecord = false;
} else {
// Found empty name
System.out.println("Found empty name - stop.");
}
break;
}
value = value.toUpperCase();
currVariableNames.add(value);
if (col == size) {
bVariableNameInNewRecord = false;
}
} else {
// next row will be record data
bNewRecordInStatement = true;
currValues.clear();
break;
}
}
// create a new sql command
if (bNewRecordInStatement) {
// the line removed space for empty column that will
// cause incorrect reading data problem
String tmp = checkLine(line);
st = new StringTokenizer(tmp, ",");
// set record
for (int i = 0; i < size; i++) {
value = " ";
if (st.hasMoreTokens()) {
value = st.nextToken();
}
if (bVariableNameInNewRecord) {
currVariableNames.add(value);
if (i == size - 1) {
// next row will be record data
bVariableNameInNewRecord = false;
}
} else {
bNewRecordInStatement = true;
currValues.add(value);
}
}
// create one in current st
String aSqlStr = createOneSqlInStatement(currStatementsID,
currTableNames, currVariableNames, currValues);
if (aSqlStr != null && aSqlStr.length() > 0) {
currStatement.add(aSqlStr);
}
}
}
}
// close the file
bufRdr.close();
}
/**
* This method calculates the RUNDATE based on the Current Date + No. Days
* passed in argument. The format of this input string is
* "RUNDATE+<number of days>" where run date is current date.
*
* @param value
* @return DB2 Date format Sring (yyyy-mm-dd)
*/
public static String calculateRunDate(String value, String inputFormat) {
String retValue = "";
int numberOfDays = 0;
try {
if (value.indexOf("RUNDATE") != -1) {
String tmpValue = value.substring(value.indexOf("RUNDATE") + 7,
value.length()).trim();
if (!"".equals(tmpValue)) {
if (tmpValue.indexOf("+") != -1) {
numberOfDays = new Integer(tmpValue.substring(
tmpValue.indexOf("+") + 1, tmpValue.length())
.trim());
} else if (tmpValue.indexOf("-") != -1) {
numberOfDays = -new Integer(tmpValue.substring(
tmpValue.indexOf("-") + 1, tmpValue.length())
.trim());
}
}
}
// Get the current date
// DateTime currDate = new DateTime();
if (numberOfDays != 0) {
// Add No. of days to the current date.
// currDate.addDays(numberOfDays);
}
// retValue = currDate.format(inputFormat);
} catch (Exception ex) {
}
return retValue.toString();
}
private static String calculateRunDateForDB(String value) {
return "‘" + calculateRunDate(value, "yyyy-MM-dd") + "‘";
}
/**
* create SQL statements by using csv file
*
* @param fileName
* @return
*/
private String createAddSqlStatement(String tbNm, List<String> names,
List<String> values) {
String tmpNames = "";
String tmpValues = "";
int size = names.size();
for (int i = 0; i < size; i++) {
String value = values.get(i);
String name = names.get(i);
// ignore empty value
if (value.trim().length() > 0) {
value = value.toUpperCase();
if (value.indexOf("RUNDATE") != -1) {
value = calculateRunDateForDB(value);
}
if (tmpNames.length() > 0) {
tmpNames += "," + name;
tmpValues += "," + value;
} else {
tmpNames += name;
tmpValues += value;
}
}
}
String sqlStr = null;
if (tmpNames.length() > 0 && tmpNames.length() > 0) {
// do not create command without data
sqlStr = "INSERT INTO " + tbNm + "(" + tmpNames + ")VALUES("
+ tmpValues + ");";
}
return sqlStr;
}
private String createOneSqlInStatement(String stID, String currTableNames,
List<String> currVariableNames, List<String> currValues) {
String aSql = "";
// if (stID.indexOf("CREATE") >= 0) {
aSql = createAddSqlStatement(currTableNames, currVariableNames,
currValues);
// }
return aSql;
}
/**
* check line read from csv file the line removed space for empty column
* that will cause incorrect wrong reading data problem
*
* @param line
* @return String
*/
// the line removed space for empty column that will cause incorrect
// wrong reading data problem
private String checkLine(String line) {
int pos = line.indexOf(‘,‘);
int posNext = pos;
while (pos >= 0) {
posNext = line.indexOf(‘,‘, pos + 1);
if (posNext - pos == 1) {
// reset line "643, ‘ATP1CCQ‘,,08/01/2007"
// back to "643, ‘ATP1CCQ‘, ,08/01/2007"
line = line.replace(",,", ", ,");
pos = line.indexOf(‘,‘, posNext + 1);
} else {
pos = posNext;
}
}
return line;
}
/**
* count number of variables
*
* @param line
* @return int
*/
private int countVariables(String line) {
int count = 0;
int pos = line.indexOf(‘,‘);
while (pos >= 0) {
count++;
pos = line.indexOf(‘,‘, pos + 1);
}
if (count >= 1) {
// remove empty column(s) at end - e.g. line
// ="FEET173_SG,CREATE_ID,CREATE_TS,DISUSE_DT,,,,,,,,,,,,,"
while (line.charAt(line.length() - 1) == ‘,‘) {
count--;
line = line.substring(0, line.length() - 1);
}
// count last non-empty column at end - e.g. line
// ="FEET173_SG,CREATE_ID,CREATE_TS,DISUSE_DT"
if (line.charAt(line.length() - 1) != ‘,‘) {
count++;
}
}
return count;
}
/**
* count number of variables
*
* @param line
* @return int
*/
private ArrayList<String> setStatement(String stID) {
// new statement
ArrayList<String> statement = new ArrayList<String>();
this.statementList.put(stID, statement);
return statement;
}
private BufferedReader getBufferedReader(String csvFileName) {
BufferedReader bufRdr = null;
InputStream input = null;
try {
input = this.getClass().getResourceAsStream(csvFileName);
} catch (Exception e) {
throw new RuntimeErrorException(null, "");
}
if (input == null) {
throw new RuntimeErrorException(null, "");
}
bufRdr = new BufferedReader(new InputStreamReader(input));
return bufRdr;
}
private BufferedReader getBufferedReaderFormLocal(String csvFileName) {
BufferedReader bufRdr = null;
File file = new File(csvFileName);
try {
if (!file.exists()) {
throw new Exception("csv file is not found under" + csvFileName);
}
bufRdr = new BufferedReader(new FileReader(file));
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return bufRdr;
}
// //////////////////////////////////////////////////
/**
* Executes a group of Insert sql(s) for a statementID - used by JUnit test
*
* @param statementID
* @param jdbcTemplate
*/
public void exceuteStatement(String statementID) {
cleanTestData(statementID);
ArrayList<String> aStatement = this.statementList.get(statementID);
for (Iterator<String> it = aStatement.iterator(); it.hasNext();) {
String insertSQL = it.next();
System.out.println(insertSQL);
// jdbcTemplate.execute(insertSQL);
}
}
/**
* clean test data to avoid duplication
*
* @param JdbcTemplate
* @param String
* @return
*/
public void cleanTestData(String statementID) {
ArrayList<String> aStatement = this.statementList.get(statementID);
if (aStatement == null)
return;
for (Iterator<String> it = aStatement.iterator(); it.hasNext();) {
String insertSQL = it.next();
cleanATestRecode(insertSQL);
}
}
/**
* clean test data to avoid duplication duplication may stop test process.
* For an insert SQL, create related delete SQL with same data except date
* and time. Then run the delete before insert process. Example: insert SQL
* INSERT INTO
* PARGRP.CXR_FLT_REST_986(CXR_FLT_986_TB_NO,DSUS_DT,D_OCC_CNT,CREATE_ID,
* CREATE_TS,LAST_MNT_ACT,LAST_MNT_ID,LAST_MNT_TS) VALUES(888885,CURRENT
* DATE,1,‘ATP1JXD‘,CURRENT TIMESTAMP,‘I‘,‘ATP1JXD‘,CURRENT TIMESTAMP);
* related delete SQL delete from PARGRP.CXR_FLT_REST_986 where
* CXR_FLT_986_TB_NO=888885 and D_OCC_CNT=1 and CREATE_ID=‘ATP1JXD‘ and
* LAST_MNT_ACT=‘I‘ and LAST_MNT_ID=‘ATP1JXD‘
*
* @param JdbcTemplate
* @param String
* @return
*/
private void cleanATestRecode(String sql_i) {
boolean deleteDataFlag = false;
String sqlStatement = "delete from ";
int colNameBeginPos = sql_i.indexOf("INSERT INTO ");
if (colNameBeginPos >= 0) {
int colNameEndPos = sql_i.indexOf("(");
// 12 is size of "INSERT INTO "
if (colNameEndPos >= 12) {
String tableName = sql_i.substring(colNameBeginPos + 12,
colNameEndPos);
if (tableName != null && tableName.length() > 0) {
sqlStatement += tableName + " where ";
String valueSeparater = "VALUES(";
String separater1 = "";
int colValueEndPos = 0;
int colValueBeginPos = sql_i.indexOf(valueSeparater) + 1;
while (colNameEndPos >= 0) {
// get column name
colNameBeginPos = colNameEndPos + 1;
colNameEndPos = sql_i.indexOf(",", colNameBeginPos);
// logger.debug(" colNameBeginPos="+colNameBeginPos+",colNameEndPos="+colNameEndPos);
if (colNameEndPos < 0
|| colNameEndPos <= colNameBeginPos) {
break;
}
String name = sql_i.substring(colNameBeginPos,
colNameEndPos);
if (name != null && name.length() > 0) {
// get column value
colValueEndPos = sql_i.indexOf(",",
colValueBeginPos);
if ("VALUES(".equalsIgnoreCase(valueSeparater)) {
colValueBeginPos += 6;
}
if (colValueEndPos < 0
|| colValueEndPos <= colValueBeginPos) {
// System.out.println("!!! colValueBeginPos="+colValueBeginPos+",colValueEndPos="+colValueEndPos);
break;
}
// 7 is size of "VALUES("
String value = sql_i.substring(colValueBeginPos,
colValueEndPos);
if (value != null && value.length() > 0) {
// bypass date created by RUNDATE
boolean bDate = isDate(value);
if (!bDate
&& // isDate(value) &&
// bypass CURRENT date and time
// value
!value.equalsIgnoreCase("CURRENT TIMESTAMP")
&& !value
.equalsIgnoreCase("CURRENT DATE")) {
sqlStatement += separater1 + name;
sqlStatement += "=" + value;
deleteDataFlag = true;
}
colValueBeginPos = colValueEndPos + 1;
separater1 = " and ";
valueSeparater = ",";
} else {
break;
}
}
}
}
}
}
if (deleteDataFlag) {
System.out.println("run sqlStatement=" + sqlStatement);
// jdbcTemplate.execute(sqlStatement);
}
}
/**
* Check a date value with format as ‘2010-12-29‘ or ‘2010-12-29 2:47:32 PM‘
*
* @param String
* @return boolean
*/
private boolean isDate(String value) {
boolean bDate = false;
String tmp = value.trim().replaceAll("‘", "");
if (tmp.trim().length() == 10) {
tmp = tmp.replaceAll("-", "");
if (isNumericString(tmp)) {
bDate = true;
}
}
return bDate;
}
public static boolean isNumericString(String paramString) {
try {
Integer.parseInt(paramString);
return true;
} catch (Exception localException) {
}
return false;
}
}
3.TestFilingTextLoadFareClassProcessorForIntegration.csv
STATEMENT ID,DELETE_RULE_SEGMENT,,,,
TABLE NAME,GFSGRP.RULE_CAT_SEG_CRRN,,,,
IN_EFF_YR,CXR_CD,RULE_TAR_CD,RULE_CD,CAT_NO,,,
2012,‘MH‘,‘IPRG‘,‘MHMU‘,10,,,
2019,‘‘,, ‘MHMU‘ ,8,,
,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
STATEMENT ID,FTNT_GOVT_CRRN,,,,,,,
TABLE NAME,GFSGRP.FTNT_CAT_GOVT_FILG_CRRN,,,,
IN_EFF_YR,CXR_CD,FTNT_TAR_CD,FTNT_CD,,
2014, ‘AA‘, ‘IPRB‘, ‘GH‘
标签：读取csv文件
原文地址：http://5618698.blog.51cto.com/5608698/1630718
