标签:join mode english 显示 extract pam dict form limit
三、 垃圾邮件分类 数据准备: 用csv读取邮件数据,分解出邮件类别及邮件内容。 对邮件内容进行预处理:去掉长度小于3的词,去掉没有语义的词等 尝试使用nltk库: pip install nltk nltk.download 不成功:就使用词频统计的处理方法 训练集和测试集数据划分 from sklearn.model_selection import train_test_split
from nltk.corpue import stopwords stops=stopwords(‘english‘) stops tokens=[token for tokens if token not in stops] ‘ ‘.join(tokens)
text
#pip install nltk #nltk.download from sklearn.model_selection import train_test_split import nltk from nltk.stem import WordNetLemmatizer #lemmatizer=WordNetLemmatizer() #lemmatizer.lemmatize(‘leaves‘)
#垃圾邮件分类 text=‘‘‘Yes i think so. I am in office but my lap is in room i think thats on for the last few days. I didnt shut that down‘‘‘ import nltk from nltk.stem import WordNetLemmatizer #lemmatizer=WordNetLemmatizer() #lemmatizer.lemmatize(‘leaves‘) #预处理 def preprocessing(text): #text=text.decode("utf-8") tokens=[word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)] stops=stopwords.words(‘english‘) tokens=[token for token in tokens if token not in stops] tokens=[token.lower() for token in tokens if len(token)>=3] lmtzr=WordNetLemmatizer() tokens=[lmtzr.lemmatize(token) for token in tokens] preprocessed_text=‘ ‘.join(tokens) return preprocessed_text text #读取数据集 import csv #用csv读取邮件数据,分解出邮件类别及邮件内容 file_path = r‘C:\Users\Administrator\Desktop\SMSSpamCollectionjsn.txt‘ sms = open(file_path,‘r‘,encoding = ‘utf-8‘) sms_data = [] sms_label = [] csv_reader = csv.reader(sms,delimiter=‘\t‘) for line in csv_reader: sms_label.append(line[0]) sms_data.append(line[1]) sms.close() #按0.7:0.3比例分为训练集和测试集 import numpy as np sms_data=np.array(sms_data) sms_label=np.array(sms_label) from sklearn.model_selection import train_test_split x_train,x_test,y_train,y_test = train_test_split(sms_data,sms_label,test_size=0.3,random_state=0,stratify=sms_label) #训练集,测试集 #将其向量化 from sklearn.feature_extraction.text import TfidfVectorizer vectorizer = TfidfVectorizer(min_df = 2,ngram_range=(1,2),stop_words=‘english‘,strip_accents=‘unicode‘,norm=‘l2‘) X_train = vectorizer.fit_transform(x_train) X_test = vectorizer.transform(x_test) #朴素贝叶斯分类群 from sklearn.naive_bayes import MultinomialNB clf=MultinomialNB().fit(X_train,y_train) y_nb_pred=clf.predict(X_test) #分类结果显示 from sklearn.metrics import confusion_matrix from sklearn.metrics import classification_report print(y_nb_pred.shape,y_nb_pred)#x_test预测结果 print(‘nb_confusion_matrik:‘) cm=confusion_matrix(y_test,y_nb_pred)#混淆矩阵 print(cm) print(‘nb_classification_report:‘) cr=classification_report(y_test,y_nb_pred)#主要分类指标的文本报告 print(cr) feature_names=vectorizer.get_feature_names()#出现过的单词列表 coefs=clf.coef_#先验概率 P(x_i|y),6034 feature_log_prob_ intercept=clf.intercept_#P(y),class_log_prior_:array,shape(n_classes,) coefs_with_fns=sorted(zip(coefs[0],feature_names))#对数概率P(x_i|y)与单词x_i映射 n=10 top=zip(coefs_with_fns[:n],coefs_with_fns[:-(n+1):-1]) for(coef_1,fn_1),(coef_2,fn_2) in top: print(‘\t%.4f\t%-15s\t\t%.4f\t%-15s‘%(coef_1,fn_1,coef_2,fn_2)) sms_label print(len(x_train),len(x_test)) print(X_train.shape,X_test.shape) x_train X_train a=X_train.toarray() a for i in range(1000): for j in range(5984): if a[i,j]!=0: print(i,j,a[i,j]) vectorizer.get_feature_names()[1610]
标签:join mode english 显示 extract pam dict form limit
原文地址:https://www.cnblogs.com/GZCC-11-28/p/10037447.html