标签：
最近要调优SQL语句，于是运维倒了一份SQL日志给我，我需要知道那一些是查询慢，更新多，总之就是哪些语句执行多。
1、需要将MySQL日志解析：例如：
2、需要实现成：
思路：
1、SQL语句分类：SELECT、UPDATE、INSERT、DELETE(增、删，改，查)
2、针对更新语句、插入语句单独处理
3、正则匹配、字符串Split
4、Dictionary<string, int>键值组合
5、异步读写
开始代码：
1、正则表达式：
/// <summary>
/// 关键的运算付
/// </summary>
string[] keys = { "<", ">", "=", "!=" };
/// <summary>
/// 匹配Lik和in
/// </summary>
Regex patLike = new Regex(@"(\b)(like|in)(\b)", RegexOptions.IgnoreCase);
//时间
Regex patTime = new Regex(@"(\d{2}|\d{4})(?:\-)?([0]{1}\d{1}|[1]{1}[0-2]{1})(?:\-)?([0-2]{1}\d{1}|[3]{1}[0-1]{1})(?:\s)?([0-1]{1}\d{1}|[2]{1}[0-3]{1})(?::)?([0-5]{1}\d{1})(?::)?([0-5]{1}\d{1})|([\d+]{4}\/[\d+]{1,2}\/[\d+]{1,2} [\d+]{2}\:[\d+]{2}\:[\d+]{2})", RegexOptions.IgnoreCase);
//带SQL运算符
Regex patSql = new Regex(@"(=|<|>|<=|>=|!=|<>|like|in\s+\(|in\()(\s+\S+)|(=|<|>|<=|>=|!=|<>|like|in\s+\(|in\()(\S+)", RegexOptions.IgnoreCase);
/// <summary>
/// 获取单词
/// </summary>
Regex patArry = new Regex(@"([a-z_]+)");
/// <summary>
/// 字符集合
/// </summary>
MatchCollection matchsMade;
2、读取文件：
1 //resultFile 文件全路径 2 StreamReader sr = new StreamReader(resultFile, Encoding.UTF8); 3 int i = 0; 4 String line; 5 string strSQL = string.Empty; 6 int IsNumberCount = 0;// 7 int lineNumber = 0;//本行行号 8 string Number = string.Empty; 9 10 string[] KeysArry = { "init", "connect", "quit", "statistics", "begin", "show", "commit" }; 11 12 bool isKeys = false; 13 while ((line = sr.ReadLine()) != null) 14 { 15 i++; 16 line = line.Trim();//去除首位空白 17 line = line.ToLower(); 18 isKeys = false; 19 foreach (string s in KeysArry) 20 { 21 if (line.IndexOf(s) > -1) 22 { 23 isKeys = true; 24 continue; 25 } 26 } 27 if (isKeys) 28 { 29 continue; 30 } 31 line = line.Replace("\t", ""); 32 33 if (line.IndexOf("query") > -1) 34 { 35 IsNumberCount += 1; 36 if (lineNumber == 1 && IsNumberCount == 2) 37 { 38 analyzeSQLString(strSQL); 39 IsNumberCount = 1; 40 strSQL = ""; 41 } 42 if (lineNumber != 1 && IsNumberCount == 1) 43 { 44 if (!string.IsNullOrEmpty(strSQL)) 45 { 46 analyzeSQLString(strSQL); 47 strSQL = ""; 48 } 49 } 50 lineNumber = 1; 51 Number = line.Split(‘ ‘).Length > 0 ? line.Split(‘ ‘)[0] : ""; 52 strSQL = line.ToString(); 53 strSQL = strSQL.Substring(Number.Length); 54 strSQL = strSQL.Replace("query", ""); 55 strSQL = strSQL.Trim(); 56 57 } 58 else 59 { 60 strSQL += " " + line.ToString(); 61 lineNumber++; 62 IsNumberCount = 0; 63 } 64 bgWorker.ReportProgress(i); 65 if (bgWorker.CancellationPending) 66 { 67 e.Cancel = true; 68 return; 69 } 70 Thread.Sleep(SleepMis); 71 72 } 73 if (strSQL != "") 74 { 75 analyzeSQLString(strSQL); 76 } 77 CountLine = i; 78 sr.Close(); 79 sr.Dispose();
3、SQL解析-----analyzeSQLString(string strSQL)
#region SQL解析 private void analyzeSQLString(string strSQL) { int t = 1; int idx = 0; if (strSQL.IndexOf("select") > -1) { t = 1; idx = strSQL.IndexOf("select"); } else if (strSQL.IndexOf("insert") > -1) { t = 2; idx = strSQL.IndexOf("insert"); } else if (strSQL.IndexOf("update") > -1) { t = 3; idx = strSQL.IndexOf("update"); } else if (strSQL.IndexOf("delete") > -1) { t = 4; idx = strSQL.IndexOf("delete"); } else { t = 5; } if (idx != 0) { strSQL = strSQL.Substring(idx); } switch (t) { case 1://查询 { //过滤参数 if (ckIgnore.Checked) { //过滤limit if (strSQL.LastIndexOf("limit") > 0) { strSQL = strSQL.Substring(0, strSQL.LastIndexOf("limit")); strSQL = strSQL.Trim(); } //过滤order by if (strSQL.Contains("order by")) { strSQL = strSQL.Substring(0, strSQL.LastIndexOf("order by")); strSQL = strSQL.Trim(); } strSQL = RegexString(strSQL); } if (Seldic.ContainsKey(strSQL)) { Seldic[strSQL] = Convert.ToInt32(Seldic[strSQL]) + 1; } else { Seldic.Add(strSQL, 1); } }; break; case 2://插入 { //过滤参数 if (ckIgnore.Checked) { if (strSQL.LastIndexOf("values") > 0) { strSQL = strSQL.Substring(0, strSQL.LastIndexOf("values")); } } if (Insertdic.ContainsKey(strSQL)) { Insertdic[strSQL] = Convert.ToInt32(Insertdic[strSQL]) + 1; } else { Insertdic.Add(strSQL, 1); } }; break; case 3://更新 { //过滤参数 if (ckIgnore.Checked) { matchsMade = patArry.Matches(strSQL); strSQL = ""; for (int i = 0, len = matchsMade.Count; i < len; i++) { if (i < 3) { strSQL += matchsMade[i].Value + " "; } else { if (matchsMade[i].Value != "where") { strSQL += matchsMade[i].Value + "=? "; } else { strSQL += matchsMade[i].Value + " "; } } } strSQL = strSQL.TrimEnd(); #region 老 //string strTable = string.Empty, strWhere = string.Empty; //int idxwhere = strSQL.LastIndexOf("where"); //if (idxwhere > 0) //{ // strWhere = RegexString(strSQL.Substring(idxwhere)); // strSQL = strSQL.Substring(0, idxwhere); // strSQL = strSQL.Trim(); //} //strTable = strSQL.Substring(0, strSQL.IndexOf("set")); //strSQL = strSQL.Substring(strSQL.IndexOf("set") + 3); //strSQL = RegRepac(strSQL).Trim(); ////解析 //foreach (string s in strSQL.Split(‘B‘)) //{ // if (s.Split(‘A‘).Length > 1) // { // strSQL = strSQL.Replace(s, s.Split(‘A‘)[0] + "=@p"); // } //} //strSQL = strSQL.Replace("B", ","); //strSQL = strTable + "set " + strSQL + (string.IsNullOrEmpty(strWhere) ? "" : " " + strWhere); #endregion 老 } if (Updic.ContainsKey(strSQL)) { Updic[strSQL] = Convert.ToInt32(Updic[strSQL]) + 1; } else { Updic.Add(strSQL, 1); } }; break; case 4://删除 { //过滤参数 if (ckIgnore.Checked) { strSQL = RegexString(strSQL); } if (Deletedic.ContainsKey(strSQL)) { Deletedic[strSQL] = Convert.ToInt32(Deletedic[strSQL]) + 1; } else { Deletedic.Add(strSQL, 1); } }; break; default://其它 { if (Othdic.ContainsKey(strSQL)) { Othdic[strSQL] = Convert.ToInt32(Othdic[strSQL]) + 1; } else { Othdic.Add(strSQL, 1); } }; break; } } #endregion
4、写入文件：
1 private void Write() 2 { 3 FileStream fs = new FileStream(System.Environment.CurrentDirectory + "\\sql.txt", FileMode.Create); 4 StreamWriter sw = new StreamWriter(fs, Encoding.UTF8); 5 //开始写入 6 Dictionary<string, int> sortdic = new Dictionary<string, int>(); 7 sw.Write("查询操作\r\n"); 8 sw.Write("-----------------------------------------------------------\r\n"); 9 WriteLog(Seldic, sortdic, sw, 0); 10 11 sw.Write("插入操作\r\n"); 12 sw.Write("-----------------------------------------------------------\r\n"); 13 WriteLog(Insertdic, sortdic, sw, 1); 14 15 sw.Write("更新操作\r\n"); 16 sw.Write("-----------------------------------------------------------\r\n"); 17 WriteLog(Updic, sortdic, sw, 2); 18 19 20 sw.Write("删除操作\r\n"); 21 sw.Write("-----------------------------------------------------------\r\n"); 22 WriteLog(Deletedic, sortdic, sw, 3); 23 24 sw.Write("其它操作\r\n"); 25 sw.Write("-----------------------------------------------------------\r\n"); 26 WriteLog(Othdic, sortdic, sw, 4); 27 28 //清空缓冲区 29 sw.Flush(); 30 //关闭流 31 sw.Close(); 32 fs.Close(); 33 } 34 35 36 private void WriteLog(Dictionary<String, int> data, Dictionary<String, int> sort, StreamWriter sw, int i) 37 { 38 if (data.Count > 0) 39 { 40 sort = data.OrderByDescending(o => o.Value).ToDictionary(o => o.Key, p => p.Value); 41 foreach (KeyValuePair<string, int> de in sort) 42 { 43 sw.Write(de.Value.ToString().PadRight(7, ‘ ‘) + " " + de.Key + "\r\n"); 44 } 45 bgWaritWorke.ReportProgress(i); 46 Thread.Sleep(SleepMis); 47 } 48 }
5、异步读写
6、最关键的正则解析
#region 正则过滤参数 /// <summary> /// 正则过滤参数 /// </summary> /// <param name="s">带解析字符串</param> /// <returns></returns> public string RegexString(string strSQL) { // "<", ">", "<=", ">=", "!=", "<>", "=" //(\b|\b\s+|\‘)(,) //(\b|\b\s+)(=) //(\=\w+\,)|(\=\s+\w+\,)|(\=\‘\w+\‘\,)|(\=\s\‘\w+\‘\,)|(\‘\,) strSQL = patTime.Replace(strSQL, "@p"); //时间 strSQL = patSql.Replace(strSQL, "@p"); //参数 #region 老 //int r = 0, len = 0; //temp = key = ""; ////带SQL运算符 //matchsMade = patSql.Matches(strSQL); //for (r = 0, len = matchsMade.Count; r < len; r++) //{ // //取出前缀关键字 // temp = matchsMade[r].Value; // key = ""; // if (patLike.IsMatch(temp)) // { // key = patLike.Matches(temp)[0].Value; // } // else // { // foreach (string t in keys) // { // if (temp.Replace(t, "").Length < temp.Length) // { // temp = temp.Replace(t, ""); // key += t; // } // } // } // //重新赋值 // temp = matchsMade[r].Value; // if (temp.IndexOf(key) > 0) // { // temp = temp.Substring(0, temp.IndexOf(key)); // } // strSQL = strSQL.Replace(matchsMade[r].Value, temp + " " + key + "@p"); // key = ""; //} #endregion return strSQL; } /// <summary> /// 替换 /// </summary> /// <param name="strSQL"></param> /// <param name="newValue"></param> /// <returns></returns> public string RegRepac(string strSQL) { //(\b|\b\s+|\‘)(,) //(\b|\b\s+)(=) Regex pat = new Regex(@"(\b|\b\s+|`)(=)"); strSQL = pat.Replace(strSQL, "A"); pat = new Regex(@"(\b|\b\s+|\‘)(,)"); strSQL = pat.Replace(strSQL, "B"); return strSQL; } #endregion
完整的实例：
=================================================================================================================================================
所遇问题：
1、解析速度偏慢（应该是正则没有写好）。
2、关于更新语句的处理效果太烂了。
3、很多SQL的运算付没有保留下来。
4、如果需要保留关键的运算付，就使用标记了 "老" 字的代码
列举一些列子：
select state as `status`, round(sum(duration),7) as `duration`, concat(round(sum(duration)/0.002432*100,3), ‘%‘) as `percentage` from information_schema.profiling
最后感言：
1、还有很多情况没有考虑到
2、不知道有没有谁可以推荐C#版的SQL语句解析的工具或者DLL
3、历时3天
这几天心情不好，唉。想回家休息个把月，哈哈
下载连接
http://pan.baidu.com/s/1mglImpU
标签：
原文地址：http://www.cnblogs.com/lifuzhen/p/4864721.html
