标签：tom 记录 beans sch convert 系统繁忙 @param art sem
最近一次迭代，参与了公司数据应用平台的开发，其中负责的一块功能早早的就完成了代码的编写工作，即将进入测试阶段，因为有时间思考和总结代码编写中遇到的难题，便想着将代码做一次重构：其中优化的一个功能就是关于数据平台敏感字段的收集
功能描述：数据平台敏感字段的收集：
提供 service 方法，查询是否需要扫描表做触发式收集，指定具体实例与库的表，随机取 N 行（1~max(id) 之间）；
a.对每一行的每一个字段的值（取非 null 非空的值）做正则匹配
b. 对每一行的每一个字段的值看是否包含了敏感字段的 key
c. 对每一行的每一个字段名做匹配；如果匹配，再判断该字段为敏感字段还是疑似敏感字段，添加到 secret_column 中
开始的版本：
/**
* 敏感字段的收集
*
* @param instance
* @param schema
*/
public void collectSecretColumn(String instance, String schema, String table) {
//查询该表是否扫描过
CollectedTable collectedTable = collectedTableService.getCollectedTable(instance, schema, table);
if (collectedTable != null) {
return;
}
//随机获取n行记录
JdbcResult query = getPartQueryResult(instance, schema, table);
if (query == null || (query != null && StringUtils.isNotBlank(query.getQueryErrorMsg()))) {
throw new CjjServerException(500, "系统繁忙,请稍后再试");
}
//key为column value为值的集合
Map<String, List<String>> groupMap = convertListToMap(query.getResult());
Set<Map.Entry<String, List<String>>> entries = groupMap.entrySet();
List<SecretContainedKey> secretContainedKeys = secretContainedKeyMapper.listSecretContainedKeys();
List<SecretValueRegex> secretValueRegexs = secretValueRegexService.getSecretValueRegexes();
for (Map.Entry<String, List<String>> entry : entries) {
//获取column
String column = entry.getKey();
List<String> values = entry.getValue();
//判断该字段是否已经存在在敏感字段表中
boolean secretColumnExist = isSecretColumnExist(instance, schema, table, column);
if (secretColumnExist) {
continue;
}
//c:对字段名做匹配
boolean isValueContaninsKey = secretContainedKeyService.columnKeyIsContainsKey(instance, schema, table, secretContainedKeys, column);
if (isValueContaninsKey) {
continue;
}
//b:字段的值是否包含敏感字段的key
boolean isContainsKey = secretContainedKeyService.columnValueIsContainsKey(instance, schema, table, secretContainedKeys, column, values);
if (isContainsKey) {
continue;
}
//a:通过正则匹配字段值
secretValueRegexService.regexMatch(instance, schema, table, column, values, secretValueRegexs);
}
CollectedTable collected = CollectedTable
.builder()
.instanceName(instance)
.schemaName(schema)
.tableName(table)
.build();
collectedTableMapper.save(collected);
}
可以看出逻辑都散落在for循环中
通过责任链模式：后代码：
/**
* 敏感字段的收集
*
* @param instance
* @param schema
*/
public void collectSecretColumn(String instance, String schema, String table) {
//查询该表是否扫描过
CollectedTable collectedTable = collectedTableService.getCollectedTable(instance, schema, table);
if (collectedTable != null) {
return;
}
//随机获取n行记录
JdbcResult query = getPartQueryResult(instance, schema, table);
if (query == null || (query != null && StringUtils.isNotBlank(query.getQueryErrorMsg()))) {
throw new CjjServerException(500, "系统繁忙,请稍后再试");
}
//key为column value为值的集合
Map<String, List<String>> groupMap = convertListToMap(query.getResult());
Set<Map.Entry<String, List<String>>> entries = groupMap.entrySet();
secretValueRegexHandler.setSuccessor(secretValueContainedKeyHandler);
secretValueContainedKeyHandler.setSuccessor(secretColumnContainedKeyHandler);
for (Map.Entry<String, List<String>> entry : entries) {
//获取column
String column = entry.getKey();
List<String> values = entry.getValue();
//判断该字段是否已经存在在敏感字段表中
boolean secretColumnExist = isSecretColumnExist(instance, schema, table, column);
if (secretColumnExist) {
continue;
}
secretValueRegexHandler.handleCollect(instance, schema, table, column, values);
}
CollectedTable collected = CollectedTable
.builder()
.instanceName(instance)
.schemaName(schema)
.tableName(table)
.build();
collectedTableMapper.save(collected);
}
可以看出这边的代码量减少了，看起来结构更清晰了
为了方便理解：我会列出部分代码供大家参考
package cn.caijiajia.firekylin.service.secret;
import java.util.List;
/**
* 责任链设计模式
*
* @author chenlang
* date 2018/7/13
*/
public abstract class CollectSecretColumnHandler {
protected CollectSecretColumnHandler successor;
public abstract void handleCollect(String instance, String schema, String table, String column, List<String> values);
/**
* 获取责任对象
*/
public CollectSecretColumnHandler getSuccessor() {
return successor;
}
/**
* 设置后继的责任对象
*/
public void setSuccessor(CollectSecretColumnHandler successor) {
this.successor = successor;
}
}
package cn.caijiajia.firekylin.service.secret;
import cn.caijiajia.firekylin.domain.SecretContainedKey;
import cn.caijiajia.firekylin.mapper.SecretContainedKeyMapper;
import cn.caijiajia.firekylin.service.SecretColumnService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
/**
* @author chenlang
* date 2018/7/13
*/
@Component
public class SecretColumnContainedKeyHandler extends CollectSecretColumnHandler {
@Autowired
private SecretColumnService secretColumnService;
@Autowired
private SecretContainedKeyMapper secretContainedKeyMapper;
@Override
public void handleCollect(String instance, String schema, String table, String column, List<String> values) {
List<SecretContainedKey> secretContainedKeys = secretContainedKeyMapper.listSecretContainedKeys();
boolean columnKeyIsContainsKey = columnKeyIsContainsKey(instance, schema, table, secretContainedKeys, column);
if (!columnKeyIsContainsKey) {
}
}
public boolean columnKeyIsContainsKey(String instance, String schema, String table, List<SecretContainedKey> secretContainedKeys, String column) {
SecretContainedKey secretContainedKeyByColumn = getSecretContainedKeyByColumn(column, secretContainedKeys);
if (secretContainedKeyByColumn != null) {
secretColumnService.saveSecretColumn(instance, schema, table, column, secretContainedKeyByColumn.getSecretType(), secretContainedKeyByColumn.getColumnType());
return true;
}
return false;
}
/**
* 字段名是否包含敏感的key
*
* @param column
* @param secretContainedKeys
* @return
*/
public SecretContainedKey getSecretContainedKeyByColumn(String column, List<SecretContainedKey> secretContainedKeys) {
Map<String, SecretContainedKey> keysMap = secretContainedKeys.stream().collect(Collectors.toMap(SecretContainedKey::getContainedKey, a -> a));
Set<Map.Entry<String, SecretContainedKey>> entries = keysMap.entrySet();
for (Map.Entry<String, SecretContainedKey> entry : entries) {
String key = entry.getKey();
boolean contains = column.toLowerCase().contains(key);
if (contains) {
return keysMap.get(key);
}
}
return null;
}
}
package cn.caijiajia.firekylin.service.secret;
import cn.caijiajia.firekylin.domain.SecretContainedKey;
import cn.caijiajia.firekylin.mapper.SecretContainedKeyMapper;
import cn.caijiajia.firekylin.service.SecretColumnService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* @author chenlang
* date 2018/7/13
*/
@Component
public class SecretValueContainedKeyHandler extends CollectSecretColumnHandler {
@Autowired
private SecretColumnService secretColumnService;
@Autowired
private SecretContainedKeyMapper secretContainedKeyMapper;
@Override
public void handleCollect(String instance, String schema, String table, String column, List<String> values) {
List<SecretContainedKey> secretContainedKeys = secretContainedKeyMapper.listSecretContainedKeys();
boolean columnValueIsContainsKey = columnValueIsContainsKey(instance, schema, table, secretContainedKeys, column, values);
if (!columnValueIsContainsKey) {
getSuccessor().handleCollect(instance, schema, table, column, values);
}
}
public boolean columnValueIsContainsKey(String instance, String schema, String table, List<SecretContainedKey> secretContainedKeys, String column, List<String> values) {
for (SecretContainedKey secretContainedKey : secretContainedKeys) {
boolean isSecretColumnContainsKey = isSecretColumnContainsKey(values, secretContainedKey);
if (isSecretColumnContainsKey) {
secretColumnService.saveSecretColumn(instance, schema, table, column, secretContainedKey.getSecretType(), secretContainedKey.getColumnType());
return true;
}
}
return false;
}
/**
* 字段值是否包含敏感字段的key
*
* @param columnValues
* @param secretContainedKey
* @return
*/
public boolean isSecretColumnContainsKey(List<String> columnValues, SecretContainedKey secretContainedKey) {
for (String columnValue : columnValues) {
if (columnValue.toLowerCase().contains(secretContainedKey.getContainedKey())) {
return true;
}
}
return false;
}
}
package cn.caijiajia.firekylin.service.secret;
import cn.caijiajia.firekylin.constant.SecretType;
import cn.caijiajia.firekylin.domain.SecretValueRegex;
import cn.caijiajia.firekylin.service.SecretColumnService;
import cn.caijiajia.firekylin.service.SecretValueRegexService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.regex.Pattern;
/**
* 正则匹配收集敏感字段
*
* @author chenlang
* date 2018/7/13
*/
@Component
public class SecretValueRegexHandler extends CollectSecretColumnHandler {
@Autowired
private SecretColumnService secretColumnService;
@Autowired
private SecretValueRegexService secretValueRegexService;
@Override
public void handleCollect(String instance, String schema, String table, String column, List<String> values) {
List<SecretValueRegex> secretValueRegexs = secretValueRegexService.getSecretValueRegexes();
boolean regexMatch = regexMatch(instance, schema, table, column, values, secretValueRegexs);
if (!regexMatch) {
if (getSuccessor() != null) {
getSuccessor().handleCollect(instance, schema, table, column, values);
}
}
}
public boolean regexMatch(String instance, String schema, String table, String column, List<String> values, List<SecretValueRegex> secretValueRegexs) {
for (SecretValueRegex secretValueRegex : secretValueRegexs) {
boolean secretByRegex = isSecretByRegex(values, secretValueRegex.getPattern());
if (secretByRegex) {
secretColumnService.saveSecretColumn(instance, schema, table, column, SecretType.SECRECT, secretValueRegex.getCode());
return true;
}
}
return false;
}
/**
* 字段值是否匹配正则表达式
*
* @param columnValues
* @return
*/
public boolean isSecretByRegex(List<String> columnValues, Pattern compile) {
if (CollectionUtils.isEmpty(columnValues)) {
return false;
}
for (String columnValue : columnValues) {
boolean isSecret = compile.matcher(columnValue).matches();
if (!isSecret) {
return false;
}
}
return true;
}
}
现在每种情况对应一种handler，同时继承自
CollectSecretColumnHandler
标签：tom 记录 beans sch convert 系统繁忙 @param art sem
原文地址：https://www.cnblogs.com/clovejava/p/9311142.html
