标签:朴素 shape ESS text nis emma pen repr tran
#读取数据集 import csv file_path=r‘C:\Users\Administrator\Desktop\江南.txt‘ sms=open(file_path,‘r‘,encoding=‘utf-8‘) text=csv.reader(sms,delimiter=‘\t‘) text #预处理 def preprocessing(text): #text=text.decode("utf-8") tokens=[word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)] #进行分词 stops=stopwords.words(‘english‘) #去掉停用词 tokens=[token for token in tokens if token not in stops] tokens=[token.lower() for token in tokens if len(token)>=3] lmtzr=WordNetLemmatizer() #词性还原 tokens=[lmtzr.lemmatize(token) for token in tokens] preprocessed_text=‘ ‘.join(tokens) return preprocessed_text #将其向量化 from sklearn.feature_extraction.text import TfidfVectorizer vectorizer=TfidfVectorizer(min_df=2,ngram_range=(1,2),stop_words=‘english‘,strip_accents=‘unicode‘,norm=‘12‘) x_train=[u‘没有 你 的 地方 都是 他乡‘,u‘没有 你 的 旅行 都是 流浪‘] X_train=vectorizer.fit_transform(x_train) X_test=vectorizer.transform(x_test) #按比例分为训练集和测试集 import numpy as np sms_data=np.array(sms_data) sms_label=np.array(sms_label) from sklearn.model_selection import train_test_split x_train,x_test,y_train,y_test=train_test_split(sms_data,sms_label,test_size=0.3,random_state=0,stratify=sms_label) #朴素贝叶斯 from sklearn.naive_bayes import MultinomialNB clf=MultinomialNB().fit(x_train,y_train) #测试模型 from sklearn.metrics import confusion_matrix from sklearn.metrics import classification_report print(y_nb_pred.shape,y_nb_pred) print(‘nb_confusion_matrix:‘) cm=confusion_matrix(y_test.y_nb_pred) print(cm) print(‘nb_classification_report:’) cr=classification_report(y_test.y_nb_pred) print(cr)
标签:朴素 shape ESS text nis emma pen repr tran
原文地址:https://www.cnblogs.com/h000/p/10037415.html