自然语言处理（英语文章）———初略处理

#-*- coding:utf-8 -*-
from urllib2 import urlopen
import re
import string
import operator


#单词清洗
def isCommon(ngram):
    ngrams=ngram.split(‘ ‘)
    #清洗以下没有意义的单词
    commonWords=[‘the‘, ‘be‘, ‘and‘, ‘of‘, ‘a‘, ‘in‘, ‘to‘, ‘have‘, ‘it‘, ‘i‘, ‘for‘, ‘you‘, ‘he‘,
                 ‘with‘, ‘on‘, ‘do‘, ‘say‘, ‘this‘, ‘they‘, ‘is‘, ‘an‘, ‘at‘, ‘but‘, ‘we‘, ‘his‘,
                 ‘from‘, ‘that‘, ‘not‘, ‘by‘, ‘she‘, ‘or‘, ‘what‘, ‘go‘, ‘their‘, ‘can‘, ‘who‘,
                 ‘get‘, ‘if‘, ‘would‘, ‘her‘, ‘all‘, ‘my‘, ‘make‘, ‘about‘, ‘know‘, ‘will‘, ‘as‘,
                 ‘up‘, ‘one‘, ‘time‘, ‘has‘, ‘been‘, ‘there‘, ‘year‘, ‘so‘, ‘think‘, ‘when‘, ‘which‘,
                 ‘them‘, ‘some‘, ‘me‘, ‘people‘, ‘take‘, ‘out‘, ‘into‘, ‘just‘, ‘see‘, ‘him‘, ‘your‘,
                 ‘come‘, ‘could‘, ‘now‘, ‘than‘, ‘like‘, ‘other‘, ‘how‘, ‘then‘, ‘its‘, ‘our‘, ‘two‘,
                 ‘more‘, ‘these‘, ‘want‘, ‘way‘, ‘look‘, ‘first‘, ‘also‘, ‘new‘, ‘because‘, ‘day‘, ‘use‘,
                 ‘no‘, ‘man‘, ‘find‘, ‘here‘, ‘thing‘, ‘give‘, ‘many‘, ‘well‘]
    #判断2-gram中是否存在要清洗的单词
    for word in ngrams:
        if word.lower() in commonWords:
            return False
    return True


#数据清洗
def cleanInput(input):
    #装换多个\n和空格为单个空格
    input=re.sub(‘\n+‘,‘ ‘,input)
    input=re.sub(‘\[[0-9]*\]‘,‘‘,input)
    input=re.sub(‘ +‘,‘ ‘,input)
    input=bytes(input.decode(‘utf-8‘))
    input=input.decode(‘ascii‘,‘ignore‘)
    cleanInput=[]
    input=input.split(‘ ‘)
    for item in input:
        #string.punctuation 去除所有符号：!"#$%&‘()*+,-./:;<=>?@[\]^_`{|}~
        item=item.strip(string.punctuation)
        if len(item)>1 or (item.lower()==‘a‘ or item.lower()==‘i‘):
            cleanInput.append(item)
    return cleanInput

#input为输入的整个字符串，n表示以几个字符作为参照，即n-gram
def ngrams(input,n):
    input=cleanInput(input)
    #声明一个数组
    output={}
    for i in range(len(input)-n-1):
        ngramTemp=‘ ‘.join(input[i:i+n])
        if isCommon(ngramTemp):
            if ngramTemp not in output:
                output[ngramTemp]=0
            output[ngramTemp]+=1
    return  output

html=urlopen(‘http://pythonscraping.com/files/inaugurationSpeech.txt‘).read().decode(‘utf-8‘)
content=str(html)
ngrams=ngrams(content,2)

#key=operator.itemgetter(0) 表示以字典中的key(字符首字母)为前提排序
#key=operator.itemgetter(1) 表示以字典中的value(数字)为前提排序
#reverse=True 表示降序输出
sortedNGrams=sorted(ngrams.items(),key=operator.itemgetter(1),reverse=True)
#输出有意义的2-gram的单词，以及它们出现的数据
print sortedNGrams

#获取上面的的2-gram单词
keywords=[]
for i in range(0,len(sortedNGrams)):
    word=sortedNGrams[i]
    #除去概率小于2的词组
    if int(word[1])>2:
        keywords.append(word[0])

#定义一个集合存取文章的所有句子
sentences=set()
#定义一个main_sentences来存储结果
main_sentences=set()
i=content.split(‘.‘)
for j in i:
    sentences.add(j)

for keyword in keywords:
    for sentence in sentences:
        #获取第一个存在该词组的句子
        b=sentence.find(keyword)
        if b!=-1:
            #除去句子里的\n和多余空格
            sentence=re.sub(" +"," ",sentence)
            sentence=re.sub("\n+","",sentence)
            main_sentences.add(sentence)
            break

for i in main_sentences:
    print i