利用Tensorflow进行自然语言处理（NLP）系列之二高级Word2Vec

# Variables
 
# Embedding layer, contains the word embeddings
embeddings = tf.Variable(tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
 
# Softmax Weights and Biases
softmax_weights = tf.Variable(
    tf.truncated_normal([vocabulary_size, embedding_size],
                        stddev=0.5 / math.sqrt(embedding_size))
)
softmax_biases = tf.Variable(tf.random_uniform([vocabulary_size],0.0,0.01)

# 1. Compute negative sampels for a given batch of data
# Returns a [num_sampled] size Tensor
negative_samples, _, _ = tf.nn.log_uniform_candidate_sampler(train_labels, num_true=1, num_sampled=num_sampled, 
                                                           unique=True, range_max=vocabulary_size)
# 2. Look up embeddings for inputs, outputs and negative samples.
in_embed = tf.nn.embedding_lookup(in_embeddings, train_dataset)
out_embed = tf.nn.embedding_lookup(out_embeddings, tf.reshape(train_labels,[-1]))
negative_embed = tf.nn.embedding_lookup(out_embeddings, negative_samples) 
 
# 3. Manually defining negative sample loss
# As Tensorflow have a limited amount of flexibility in the built-in sampled_softmax_loss function,
# we have to manually define the loss fuction.
 
# 3.1. Computing the loss for the positive sample
# Exactly we compute log(sigma(v_o * v_i^T)) with this equation
loss = tf.reduce_mean(
  tf.log(
      tf.nn.sigmoid(
          tf.reduce_sum(
              tf.diag([1.0 for _ in range(batch_size)])*
              tf.matmul(out_embed,tf.transpose(in_embed)),
          axis=0)
      )
  )      
)
 
# 3.2. Computing loss for the negative samples
# We compute sum(log(sigma(-v_no * v_i^T))) with the following
# Note: The exact way this part is computed in TensorFlow library appears to be
# by taking only the weights corresponding to true samples and negative samples
# and then computing the softmax_cross_entropy_with_logits for that subset of weights.
# More infor at: https://github.com/tensorflow/tensorflow/blob/r1.8/tensorflow/python/ops/nn_impl.py
# Though the approach is different, the idea remains the same
loss += tf.reduce_mean(
  tf.reduce_sum(
      tf.log(tf.nn.sigmoid(-tf.matmul(negative_embed,tf.transpose(in_embed)))),
      axis=0
  )
)
 
# The above is the log likelihood. 
# We would like to transform this to the negative log likelihood
# to convert this to a loss. This provides us with
# L = - (log(sigma(v_o * v_i^T))+sum(log(sigma(-v_no * v_i^T))))
loss *= -1.0

Define Label colors for %d 20
Running K-Means for skip-gram
Running K-Means for CBOW
K-Means ran successfully

pylab.figure(figsize=(15,5))  # in inches
 
# Define the x axis
x = np.arange(len(skip_gram_loss))*2000
 
# Plotting standard CBOW loss, CBOW loss with unigram sampling and
# CBOW loss with unigram sampling + subsampling here in one plot
pylab.plot(x, cbow_loss, label="CBOW",linestyle=‘--‘,linewidth=2)
pylab.plot(x, cbow_loss_unigram, label="CBOW (Unigram)",linestyle=‘-.‘,linewidth=2,marker=‘^‘,markersize=5)
pylab.plot(x, cbow_loss_unigram_subsampled, label="CBOW (Unigram+Subsampling)",linewidth=2)
 
# Some text around the plots
pylab.title(‘Original CBOW vs Various Improvements Loss Decrease Over-Time‘,fontsize=24)
pylab.xlabel(‘Iterations‘,fontsize=22)
pylab.ylabel(‘Loss‘,fontsize=22)
pylab.legend(loc=1,fontsize=22)
 
# Use for saving the figure if needed
pylab.savefig(‘loss_cbow_vs_all_improvements.png‘)
pylab.show()

num_steps = 100001
glove_loss = []
 
average_loss = 0
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as session:
    
    tf.global_variables_initializer().run()
    print(‘Initialized‘)
    
    for step in range(num_steps):
        
        # generate a single batch (data,labels,co-occurance weights)
        batch_data, batch_labels, batch_weights = generate_batch(
            batch_size, skip_window) 
        
        # Computing the weights required by the loss function
        batch_weights = [] # weighting used in the loss function
        batch_xij = [] # weighted frequency of finding i near j
        
        # Compute the weights for each datapoint in the batch
        for inp,lbl in zip(batch_data,batch_labels.reshape(-1)):     
            point_weight = (cooc_mat[inp,lbl]/100.0)**0.75 if cooc_mat[inp,lbl]<100.0 else 1.0 
            batch_weights.append(point_weight)
            batch_xij.append(cooc_mat[inp,lbl])
        batch_weights = np.clip(batch_weights,-100,1)
        batch_xij = np.asarray(batch_xij)
        
        # Populate the feed_dict and run the optimizer (minimize loss)
        # and compute the loss. Specifically we provide
        # train_dataset/train_labels: training inputs and training labels
        # weights_x: measures the importance of a data point with respect to how much those two words co-occur
        # x_ij: co-occurence matrix value for the row and column denoted by the words in a datapoint
        feed_dict = {train_dataset : batch_data.reshape(-1), train_labels : batch_labels.reshape(-1),
                    weights_x:batch_weights,x_ij:batch_xij}
        _, l = session.run([optimizer, loss], feed_dict=feed_dict)
        
        # Update the average loss variable
        average_loss += l
        if step % 2000 == 0:
          if step > 0:
            average_loss = average_loss / 2000
          # The average loss is an estimate of the loss over the last 2000 batches.
          print(‘Average loss at step %d: %f‘ % (step, average_loss))
          glove_loss.append(average_loss)
          average_loss = 0
        
        # Here we compute the top_k closest words for a given validation word
        # in terms of the cosine distance
        # We do this for all the words in the validation set
        # Note: This is an expensive step
        if step % 10000 == 0:
          sim = similarity.eval()
          for i in range(valid_size):
            valid_word = reverse_dictionary[valid_examples[i]]
            top_k = 8 # number of nearest neighbors
            nearest = (-sim[i, :]).argsort()[1:top_k+1]
            log = ‘Nearest to %s:‘ % valid_word
            for k in range(top_k):
              close_word = reverse_dictionary[nearest[k]]
              log = ‘%s %s,‘ % (log, close_word)
            print(log)
            
    final_embeddings = normalized_embeddings.eval()

Initialized
Average loss at step 0: 9.578778
Nearest to it: karol, burgh, destabilise, armchair, crook, roguery, one-sixth, swains,
Nearest to that: wmap, partake, ahmadi, armstrong, memberships, forza, director-general, condo,
Nearest to has: mentality, vastly, approaches, bulwark, enzymes, originally, privatize, reunify,
Nearest to but: inhabited, potrero, trust, memory, curran, philips, p.m.s, pagoda,
Nearest to city: seals, counter-revolution, tubular, kayaking, central, 1568, override, buckland,
Nearest to this: dispersion, intermarriage, dialysis, moguls, aldermen, alcoholic, codes, farallon,
Nearest to UNK: 40.3, tatsam, jupiter, verify, unequal, berliners, march, 1559,
Nearest to by: functionalists, synthesised, palladius, chiapas, synaptic, sumner, raining, valued,
Nearest to or: amherst, ‘mother, epiglottis, wen, stanislaus, trafford, cuticle, reminded,
Nearest to been: 640,961., depression-era, uniquely, mami, 375,000, stickiness, medium-sized, amor,
Nearest to with: anti-statist, pitigliano, branches, reparations, acquittal, frowned, pishpek, left-leaning,
Nearest to be: i-20, kevin, greased, rightly, conductors, hypercholesterolemia, pedro, douaumont,
Nearest to as: gabon, horda, mead, protruding, soundtrack, algeria, 48, macon,
Nearest to at: kambula, tisa, spelled, 130,000, 2008, organisers, |jul_rec_lo_°f, arrows,
Nearest to ,: is, of, its, malton, martin?, retiree, reliant, uri,
Nearest to its: of, ,, galleon, gitlow, rugby-playing, varanasi, fono, clusters,
Average loss at step 2000: 0.739107
Average loss at step 4000: 0.091107
Average loss at step 6000: 0.068614
Average loss at step 8000: 0.076040
Average loss at step 10000: 0.058149
Nearest to it: was, is, that, not, a, in, to, .,
Nearest to that: is, was, the, a, ., ,, to, in,
Nearest to has: is, it, that, a, been, was, to, mentality,
Nearest to but: with, said, trust, mating, not, squamous, war—the, r101,
Nearest to city: of, ‘s, counter-revolution, the, professed, ., equilibrium, seals,
Nearest to this: is, ., for, in, was, the, a, that,
Nearest to UNK: and, ,, (, in, the, ., ), a,
Nearest to by: the, and, ,, ., in, was, of, a,
Nearest to or: UNK, ,, and, a, cuticle, donnchad, ``, ‘mother,
Nearest to been: have, had, to, has, be, was, that, it,
Nearest to with: ,, and, a, the, in, of, for, .,
 
Nearest to by: the, was, ,, in, ., and, a, of,
Nearest to or: (, UNK, ), ``, a, ,, and, with,
Nearest to been: have, has, had, also, be, that, was, to,
Nearest to with: and, ,, a, the, of, in, for, .,
Nearest to be: to, have, can, not, that, from, is, would,
Nearest to as: a, an, ,, such, for, and, is, the,
Nearest to at: of, the, ., in, ‘s, ,, and, by,
Nearest to ,: and, in, the, ., a, with, of, UNK,
Nearest to its: for, and, their, with, his, ,, the, of,
Average loss at step 92000: 0.019305
Average loss at step 94000: 0.019555
Average loss at step 96000: 0.019266
Average loss at step 98000: 0.018803
Average loss at step 100000: 0.018488
Nearest to it: is, was, also, that, not, has, this, a,
Nearest to that: was, is, to, it, the, a, ., ,,
Nearest to has: it, been, was, had, also, is, that, a,
Nearest to but: which, not, ,, it, with, was, and, a,
Nearest to city: of, ‘s, the, ., in, is, new, world,
Nearest to this: is, ., was, it, in, for, the, at,
Nearest to UNK: (, and, ), ,, or, a, the, .,
Nearest to by: the, ., was, ,, and, in, of, a,
Nearest to or: UNK, (, ``, a, ), ,, and, with,
Nearest to been: have, has, had, also, be, was, that, to,
Nearest to with: and, ,, a, the, of, in, for, .,
Nearest to be: to, have, can, not, would, from, that, a,
Nearest to as: a, such, an, ,, for, is, and, to,
Nearest to at: of, ., the, in, ‘s, by, ,, and,
Nearest to ,: and, in, the, ., a, with, UNK, of,
Nearest to its: for, their, and, with, his, ,, to, the,

num_steps = 100001
cbow_loss = []
 
config=tf.ConfigProto(allow_soft_placement=True)
# This is an important setting and with limited GPU memory,
# not using this option might lead to the following error.
# InternalError (see above for traceback): Blas GEMM launch failed : ...
config.gpu_options.allow_growth = True
 
with tf.Session(config=config) as session:
    
    # Initialize the variables in the graph
    tf.global_variables_initializer().run()
    print(‘Initialized‘)
    
    average_loss = 0
    
    # Train the Word2vec model for num_step iterations
    for step in range(num_steps):
        
        # Generate a single batch of data
        batch_data, batch_labels = generate_batch(data, batch_size, window_size)
        
        # Populate the feed_dict and run the optimizer (minimize loss)
        # and compute the loss
        feed_dict = {train_dataset : batch_data, train_labels : batch_labels}
        _, l = session.run([optimizer, loss], feed_dict=feed_dict)
        
        # Update the average loss variable
        average_loss += l
        
        if (step+1) % 2000 == 0:
            if step > 0:
                average_loss = average_loss / 2000
                # The average loss is an estimate of the loss over the last 2000 batches.
            print(‘Average loss at step %d: %f‘ % (step+1, average_loss))
            cbow_loss.append(average_loss)
            average_loss = 0
        
        # Evaluating validation set word similarities
        if (step+1) % 10000 == 0:
            sim = similarity.eval()
            # Here we compute the top_k closest words for a given validation word
            # in terms of the cosine distance
            # We do this for all the words in the validation set
            # Note: This is an expensive step
            for i in range(valid_size):
                valid_word = reverse_dictionary[valid_examples[i]]
                top_k = 8 # number of nearest neighbors
                nearest = (-sim[i, :]).argsort()[1:top_k+1]
                log = ‘Nearest to %s:‘ % valid_word
                for k in range(top_k):
                    close_word = reverse_dictionary[nearest[k]]
                    log = ‘%s %s,‘ % (log, close_word)
                print(log)
    
    # Computing test documents embeddings by averaging word embeddings
    
    # We take batch_size*num_test_steps words from each document
    # to compute document embeddings
    num_test_steps = 100
    
    # Store document embeddings
    # {document_id:embedding} format
    document_embeddings = {}
    print(‘Testing Phase (Compute document embeddings)‘)
    
    # For each test document compute document embeddings
    for k,v in test_data.items():
        print(‘\tCalculating mean embedding for document ‘,k,‘ with ‘, num_test_steps, ‘ steps.‘)
        test_data_index = 0
        topic_mean_batch_embeddings = np.empty((num_test_steps,embedding_size),dtype=np.float32)
        
        # keep averaging mean word embeddings obtained for each step
        for test_step in range(num_test_steps):
            test_batch_labels = generate_test_batch(test_data[k],batch_size)
            batch_mean = session.run(mean_batch_embedding,feed_dict={test_labels:test_batch_labels})
            topic_mean_batch_embeddings[test_step,:] = batch_mean
        document_embeddings[k] = np.mean(topic_mean_batch_embeddings,axis=0)

# Create and fit K-means
kmeans = KMeans(n_clusters=5, random_state=43643, max_iter=10000, n_init=100, algorithm=‘elkan‘)
kmeans.fit(np.array(list(document_embeddings.values())))
 
# Compute items fallen within each cluster
document_classes = {}
for inp, lbl in zip(list(document_embeddings.keys()), kmeans.labels_):
    if lbl not in document_classes:
        document_classes[lbl] = [inp]
    else:
        document_classes[lbl].append(inp)
for k,v in document_classes.items():    
    print(‘\nDocuments in Cluster ‘,k)
    print(‘\t‘,v)

Documents in Cluster  0
     [‘entertainment-216‘, ‘business-240‘, ‘business-44‘, ‘tech-178‘, ‘business-165‘, ‘tech-238‘, ‘business-171‘, ‘business-144‘, ‘business-107‘]
 
Documents in Cluster  1
     [‘tech-34‘, ‘tech-145‘, ‘business-135‘, ‘sport-206‘, ‘tech-216‘, ‘politics-184‘, ‘politics-247‘, ‘politics-171‘, ‘politics-8‘, ‘politics-78‘, ‘entertainment-163‘, ‘politics-16‘, ‘business-141‘, ‘business-215‘, ‘tech-79‘, ‘tech-157‘, ‘sport-231‘, ‘tech-42‘, ‘politics-197‘, ‘politics-98‘, ‘tech-212‘]
 
Documents in Cluster  2
     [‘sport-166‘, ‘entertainment-119‘, ‘business-161‘, ‘sport-129‘, ‘sport-45‘, ‘entertainment-98‘, ‘entertainment-196‘, ‘politics-236‘, ‘sport-26‘, ‘entertainment-1‘, ‘entertainment-74‘, ‘entertainment-244‘, ‘entertainment-154‘]
 
Documents in Cluster  3
     [‘sport-184‘]
 
Documents in Cluster  4
     [‘sport-87‘, ‘sport-32‘, ‘sport-20‘]