NLPCC2013中文微博细粒度情感识别（二）

#回复@王羊羊的微博:是，要的就是安静的生存环境，而且，到北京五环25分#钟车程，938离我住处只有300米，饮食一条街更是北京饮食街的翻版，应有尽有。     
#//@王羊羊的微博:廊坊是个好地方，紧邻京津，有点闹中取静的意思。
#[抓狂]我竟然被那个考了车牌几个月但一直没有开过车的老妈笑话我笨！ 
#这家位于槟城George Town，Tunes Hotel斜对面的饼家，家庭式作业，她#的咸蛋酥，年糕椰蓉酥，叉烧酥都很不错。[good]
#做人原则：⑴能不骂人，就不骂！
#联系方式：13810938511 QQ：122095021

import jieba
import re
import gensim
import collections
import matplotlib.pyplot as plt
from wordcloud import WordCloud
from wordcloud import STOPWORDS
import numpy as np
from PIL import Image
from gensim.models import KeyedVectors
from sklearn.manifold import TSNE
import tqdm
import matplotlib
from matplotlib import font_manager
#matplotlib.use(‘qt4agg‘)

input_file = open("data_all.txt")
counter = 0
corpora = []
real_vocab = []

"""
第一部分，预处理和分词
"""
def chn_tokenize(sentence): # 使用jieba分词实现完整的中文数据集分词
    line_list = jieba.lcut(sentence, HMM=True)
    return line_list
    #print(line_list)


def del_reply_mark(sentence):
    output = re.sub(‘@.*?:‘, ‘‘, sentence).strip("回复").strip("//")
    output = re.sub(‘@.*? ‘, ‘‘, output)
    if output == "":
        output = "***"
    #print(output)
    return output


def rep_chn_punc(sentence):
    table = {ord(f): ord(t) for f, t in zip(
        u‘，。！？【】（）％＃＠＆１２３４５６７８９０①②③④⑤、·：[]():;‘,
        u‘,.!?....%#@&123456789012345,........‘)}
    output = sentence.translate(table).replace("...", "…").replace("《", "").replace("》", "").replace("℃", "度")        .replace("——", "").replace("..", "…").replace("「", "").replace("」", "").replace("....", "…")        .replace(".....", "…").replace(".", "").replace(",", "").replace("-", "").replace("T T", "TT")        .replace("T_T", "TT")
    return output


def clean_pun(sentence):
    strip_special_chars = re.compile("[^A-Za-z0-9]+")
    output = re.sub(strip_special_chars, "", sentence)
    return output


for i in tqdm.tqdm(input_file.readlines()):
    sen = rep_chn_punc(i)
    sen = del_reply_mark(sen).strip("\n").strip("\t").strip(".")
    counter += 1
    #print(sen)
    word_list = chn_tokenize(sen)
    try:
        while word_list.index(" "):
            del word_list[word_list.index(" ")]
    except ValueError:
        #print(word_list)
        corpora.append(word_list)
    for j in range(len(word_list)):
        if word_list[j] not in real_vocab:
            real_vocab.append(word_list[j])
print(counter)
"""
第二部分，
"""


def word_freq(text):
    corpora_in_line = []
    for line in range(len(text)):
        corpora_in_line.extend(text[line])
    frequency = collections.Counter(corpora_in_line)
    print(frequency)


def word_cloud(text, path):
    stopwords = set(STOPWORDS)
    weibo = np.array(Image.open(path))
    stopwords.add("一个")
    corpora_in_str = ""
    for line in range(len(text)):
        str_text = " ".join(text[line])
        #print(str_text)
        corpora_in_str += str_text
    my_wordcloud = WordCloud(font_path=‘SimHei.ttf‘, stopwords=STOPWORDS
                             , width=1500, height=750, background_color="black").generate(corpora_in_str)
    plt.imshow(my_wordcloud)
    plt.axis("off")
    plt.show()


def plot_with_labels(low_dim_embs, labels, filename=‘tsne.pdf‘):
    #from matplotlib import rcParams
    myfont = font_manager.FontProperties(fname=‘SimHei.ttf‘)
    #matplotlib.rcParams[‘font.family‘] = ‘sans-serif‘
    matplotlib.rcParams[‘font.family‘] = [‘SimHei‘]
    #matplotlib.rcParams[‘axes.unicode_minus‘] = False
    assert low_dim_embs.shape[0] >= len(labels), "More labels than embeddings"
    plt.figure(figsize=(18, 18))  # in inches
    print("flag2")
    for i, label in enumerate(labels):
        x, y = low_dim_embs[i, :]
        plt.scatter(x, y)
        plt.annotate(label,
                 xy=(x, y),
                 xytext=(5, 2),
                 textcoords=‘offset points‘,
                 ha=‘right‘,
                 va=‘bottom‘)
    plt.savefig(filename)

#wiki.zh.vec


def embedding(model_path, vocab):
    zh_model = KeyedVectors.load_word2vec_format(model_path)
    embedding = np.array([])
    show_vocab = []
    for m in range(len(vocab)):
        if vocab[m] in STOPWORDS:
            continue
        if len(show_vocab) > 499:
            break
        if vocab[m] in zh_model.vocab:
            show_vocab.append(vocab[m])
            embedding = np.append(embedding, zh_model[vocab[m]])
    print(len(show_vocab))
    embedding = embedding.reshape(len(show_vocab), 300)
    tsne = TSNE()
    print("flag0")
    low_dim_embedding = tsne.fit_transform(embedding)
    print("flag1")
    plot_with_labels(low_dim_embedding, show_vocab)


#word_freq(corpora)
#word_cloud(corpora, "weibologo.jpg")
embedding("wiki.zh.vec", real_vocab)