基于C#的机器学习--垃圾邮件过滤

 1 using Deedle;
 2 using EAGetMail;
 3 using System;
 4 using System.IO;
 5 using System.Linq;
 6 
 7 namespace 准备数据
 8 {
 9     internal class Program
10     {
11         private static void Main(string[] args)
12         {
13             // 获取所有原始的电子邮件格式的文件
14             // TODO: 更改指向数据目录的路径
15             string rawDataDirPath = @"D:\工作\代码库\AI\垃圾邮件过滤\raw-data";
16             string[] emailFiles = Directory.GetFiles(rawDataDirPath, "*.eml");
17 
18             // 从电子邮件文件中解析出主题和正文
19             var emailDF = ParseEmails(emailFiles);
20             // 获取每个电子邮件的标签(spam vs. ham)
21             var labelDF = Frame.ReadCsv(rawDataDirPath + "\\SPAMTrain.label", hasHeaders: false, separators: " ", schema: "int,string");
22             // 将这些标签添加到电子邮件数据框架中
23             emailDF.AddColumn("is_ham", labelDF.GetColumnAt<String>(0));
24             // 将解析后的电子邮件和标签保存为CSV文件
25             emailDF.SaveCsv("transformed.csv");
26 
27             Console.WriteLine("准备数据步骤完成!");
28             Console.ReadKey();
29         }
30 
31         private static Frame<int, string> ParseEmails(string[] files)
32         {
33             // 我们将解析每个电子邮件的主题和正文，并将每个记录存储到键值对中
34             var rows = files.AsEnumerable().Select((x, i) =>
35             {
36                 // 将每个电子邮件文件加载到邮件对象中
37                 Mail email = new Mail("TryIt");
38                 email.Load(x, false);
39 
40                 // 提取主题和正文
41                 string EATrialVersionRemark = "(Trial Version)"; // EAGetMail在试用版本中附加主题“(试用版本)”
42                 string emailSubject = email.Subject.EndsWith(EATrialVersionRemark) ?
43                     email.Subject.Substring(0, email.Subject.Length - EATrialVersionRemark.Length) : email.Subject;
44                 string textBody = email.TextBody;
45 
46                 // 使用电子邮件id (emailNum)、主题和正文创建键-值对
47                 return new { emailNum = i, subject = emailSubject, body = textBody };
48             });
49 
50             // 根据上面创建的行创建一个数据帧
51             return Frame.FromRecords(rows);
52         }
53     }
54 }

 1 private static Frame<int, string> CreateWordVec(Series<int, string> rows)
 2         {
 3             var wordsByRows = rows.GetAllValues().Select((x, i) =>
 4             {
 5                 var sb = new SeriesBuilder<string, int>();
 6 
 7                 ISet<string> words = new HashSet<string>(
 8                     Regex.Matches(
 9                         // 只字母字符
10                         x.Value, "[a-zA-Z]+(‘(s|d|t|ve|m))?"
11                     ).Cast<Match>().Select(
12                         // 然后，将每个单词转换为小写字母
13                         y => y.Value.ToLower()
14                     ).ToArray()
15                 );
16 
17                 // 对每行出现的单词进行1的编码
18                 foreach (string w in words)
19                 {
20                     sb.Add(w, 1);
21                 }
22 
23                 return KeyValue.Create(i, sb.Series);
24             });
25 
26             // 从我们刚刚创建的行创建一个数据框架 并将缺失的值编码为0
27             var wordVecDF = Frame.FromRows(wordsByRows).FillMissing(0);
28 
29             return wordVecDF;
30         }

 1 var hamTermFrequencies = subjectWordVecDF.Where(
 2                 x => x.Value.GetAs<int>("is_ham") == 1
 3             ).Sum().Sort().Reversed.Where(x => x.Key != "is_ham");
 4 
 5             var spamTermFrequencies = subjectWordVecDF.Where(
 6                 x => x.Value.GetAs<int>("is_ham") == 0
 7             ).Sum().Sort().Reversed;
 8 
 9             // 查看排名前十的垃圾邮件和非垃圾邮件
10             var topN = 10;
11 
12             var hamTermProportions = hamTermFrequencies / hamEmailCount;
13             var topHamTerms = hamTermProportions.Keys.Take(topN);
14             var topHamTermsProportions = hamTermProportions.Values.Take(topN);
15 
16             System.IO.File.WriteAllLines(
17                 dataDirPath + "\\ham-frequencies.csv",
18                 hamTermFrequencies.Keys.Zip(
19                     hamTermFrequencies.Values, (a, b) => string.Format("{0},{1}", a, b)
20                 )
21             );
22 
23             var spamTermProportions = spamTermFrequencies / spamEmailCount;
24             var topSpamTerms = spamTermProportions.Keys.Take(topN);
25             var topSpamTermsProportions = spamTermProportions.Values.Take(topN);
26 
27             System.IO.File.WriteAllLines(
28                 dataDirPath + "\\spam-frequencies.csv",
29                 spamTermFrequencies.Keys.Zip(
30                     spamTermFrequencies.Values, (a, b) => string.Format("{0},{1}", a, b)
31                 )
32             );

1 var barChart = DataBarBox.Show(
2                 new string[] { "Ham", "Spam" },
3                 new double[] {
4                     hamEmailCount,
5                     spamEmailCount
6                 }
7             );
8 barChart.SetTitle("Ham vs. Spam in Sample Set");

 1 var hamBarChart = DataBarBox.Show(
 2                 topHamTerms.ToArray(),
 3                 new double[][] {
 4                     topHamTermsProportions.ToArray(),
 5                     spamTermProportions.GetItems(topHamTerms).Values.ToArray()
 6                 }
 7             );
 8             hamBarChart.SetTitle("Top 10 Terms in Ham Emails (blue: HAM, red: SPAM)");
 9             System.Threading.Thread.Sleep(3000);
10             hamBarChart.Invoke(
11                 new Action(() =>
12                 {
13                     hamBarChart.Size = new System.Drawing.Size(5000, 1500);
14                 })
15             );
16 
17             var spamBarChart = DataBarBox.Show(
18                 topSpamTerms.ToArray(),
19                 new double[][] {
20                     hamTermProportions.GetItems(topSpamTerms).Values.ToArray(),
21                     topSpamTermsProportions.ToArray()
22                 }
23             );
24             spamBarChart.SetTitle("Top 10 Terms in Spam Emails (blue: HAM, red: SPAM)");

 1 private static Frame<int, string> ParseEmails(string[] files)
 2         {
 3             // 我们将解析每个电子邮件的主题和正文，并将每个记录存储到键值对中
 4             var rows = files.AsEnumerable().Select((x, i) =>
 5             {
 6                 // 将每个电子邮件文件加载到邮件对象中
 7                 Mail email = new Mail("TryIt");
 8                 email.Load(x, false);
 9 
10                 // 提取主题和正文
11                 string EATrialVersionRemark = "(Trial Version)"; // EAGetMail在试用版本中附加主题“(试用版本)”
12                 string emailSubject = email.Subject.EndsWith(EATrialVersionRemark) ?
13                     email.Subject.Substring(0, email.Subject.Length - EATrialVersionRemark.Length) : email.Subject;
14                 string textBody = email.TextBody;
15 
16                 // 使用电子邮件id (emailNum)、主题和正文创建键-值对
17                 return new { emailNum = i, subject = emailSubject, body = textBody };
18             });
19 
20             // 根据上面创建的行创建一个数据帧
21             return Frame.FromRecords(rows);
22         }

1 //读停词表 
2 ISet<string> stopWords = new HashSet<string>(File.ReadLines(<path-to-your-stopwords.txt>);
3 //从词频序列中过滤出停止词
4 var spamTermFrequenciesAfterStopWords = spamTermFrequencies.Where(
5                 x => !stopWords.Contains(x.Key)
6 );

 1 private static Frame<int, string> CreateWordVec(Series<int, string> rows)
 2         {
 3             var wordsByRows = rows.GetAllValues()
 4                 .Select((x, i) =>
 5                 {
 6                     var sb = new SeriesBuilder<string, int>();
 7                     ISet<string> words = new HashSet<string>(
 8                         //仅字母字符
 9                         Regex.Matches(x.Value, "[a-zA-Z]+(‘(s|d|t|ve|m))?")
10                         .Cast<Match>()
11                         //然后，将每个单词转换为小写字母
12                         .Select(y => y.Value.ToLower())
13                         .ToArray()
14                     );
15                     //对每行出现的单词进行1的编码
16                     foreach (string w in words)
17                     {
18                         sb.Add(w, 1);
19                     }
20                     return KeyValue.Create(i, sb.Series);
21                 });
22             //从我们刚刚创建的行中创建一个数据帧，并用0对缺失的值进行编码
23             var wordVecDF = Frame.FromRows(wordsByRows).FillMissing(0);
24             return wordVecDF;
25         }

 1 var cvLogisticRegressionClassifier = CrossValidation.Create<LogisticRegression, 
 2                 IterativeReweightedLeastSquares<LogisticRegression>, double[], int>(
 3                     // 折叠数量    
 4                     k: numFolds,
 5                     // 学习算法    
 6                     learner: (p) => new IterativeReweightedLeastSquares<LogisticRegression>()
 7                     {
 8                         MaxIterations = 100,
 9                         Regularization = 1e-6
10                     },
11                     // 使用0 - 1损失函数作为成本函数 
12                     loss: (actual, expected, p) => new ZeroOneLoss(expected).Loss(actual),
13                     // 合适的分类器    
14                     fit: (teacher, x, y, w) => teacher.Learn(x, y, w),
15                     // 输入    
16                     x: input,
17                     // 输出    
18                     y: output
19                 );
20 // 运行交叉验证
21 var result = cvLogisticRegressionClassifier.Learn(input, output);

 1 var cvNaiveBayesClassifier = CrossValidation.Create<NaiveBayes<BernoulliDistribution>,
 2                 NaiveBayesLearning<BernoulliDistribution>, double[], int>(
 3                     // 折叠的数量
 4                     k: numFolds,
 5                     // 二项分布的朴素贝叶斯分类器
 6                     learner: (p) => new NaiveBayesLearning<BernoulliDistribution>(),
 7                     // 使用0 - 1损失函数作为成本函数
 8                     loss: (actual, expected, p) => new ZeroOneLoss(expected).Loss(actual),
 9                     // 合适的分类器
10                     fit: (teacher, x, y, w) => teacher.Learn(x, y, w),
11                     // 输入
12                     x: input,
13                     // 输出
14                     y: output
15                 );
16 // 运行交叉验证
17 var result = cvNaiveBayesClassifier.Learn(input, output);

1 // 改变特征的数量以减少过度拟合
2     int minNumOccurences = 1;
3     string[] wordFeatures = indexedSpamTermFrequencyDF.Where(
4         x => x.Value.GetAs<int>("num_occurences") >= minNumOccurences
5     ).RowKeys.ToArray();
6 Console.WriteLine("Num特征选择: {0}", wordFeatures.Count());

 1 // 运行交叉验证
 2     var result = cvNaiveBayesClassifier.Learn(input, output);
 3 // 训练错误
 4     double trainingError = result.Training.Mean;
 5     //验证错误
 6 double validationError = result.Validation.Mean;
 7 混淆矩阵:真阳性与假阳性和真阴性与假阴性：
 8 // 混淆矩阵
 9 GeneralConfusionMatrix gcm = result.ToConfusionMatrix(input, output);
10 float truePositive = (float)gcm.Matrix[1, 1];
11      float trueNegative = (float)gcm.Matrix[0, 0];
12      float falsePositive = (float)gcm.Matrix[1, 0];
13 float falseNegative = (float)gcm.Matrix[0, 1];

1 // 计算的准确率, 精度, 召回
2 float accuracy = (truePositive + trueNegative) / numberOfSamples;
3 float precision = truePositive / (truePositive + falsePositive);
4 float recall = truePositive / (truePositive + falseNegative);