自然语言预处理

#按空格进行分词，同时针对推文一些特性，去除@用户名，保留表情等一些特殊符号
tokenizer = TweetTokenizer()
for counter,rev in enumerate(reviews):
    # 去除HTML网页格式
    temp = BeautifulSoup(rev)
    text = temp.get_text()
    # 去除空格
    text = re.sub(‘ +‘,‘ ‘,text)
    # strip leading and trailing white space
    text = text.strip()
    # tokenize 
    tokens = tokenizer.tokenize(text)
    cleaned_reviews.append(tokens)
    if counter % round(len(reviews)/10) == 0:
        print(counter, ‘/‘, len(reviews), ‘reviews cleaned‘)