1 import numpy as np
 2 import random
 3 import time
 4 import cllm_utils
 5 
 6 data = open("datasets/dinos.txt", ‘r‘).read()
 7 data= data.lower()
 8 chars = list(set(data))             # 转化为无序且不重复的元素列表
 9 print(sorted(chars))
10 data_size, vocab_size = len(data), len(chars)
11 print(‘There are %d total characters and %d unique characters in your data.‘ % (data_size, vocab_size))

1 char_to_ix = { ch:i for i,ch in enumerate(sorted(chars)) }
2 ix_to_char = { i:ch for i,ch in enumerate(sorted(chars)) }
3 print(char_to_ix)
4 print(ix_to_char)

 1 def clip(gradients, maxValue):
 2     ‘‘‘
 3     Clips the gradients‘ values between minimum and maximum.
 4     
 5     Arguments:
 6     gradients -- a dictionary containing the gradients "dWaa", "dWax", "dWya", "db", "dby"
 7     maxValue -- everything above this number is set to this number, and everything less than -maxValue is set to -maxValue
 8     
 9     Returns: 
10     gradients -- a dictionary with the clipped gradients.
11     ‘‘‘
12     dWaa, dWax, dWya, db, dby = gradients[‘dWaa‘], gradients[‘dWax‘], gradients[‘dWya‘], gradients[‘db‘], gradients[‘dby‘]
13    
14     ### START CODE HERE ###
15     # clip to mitigate exploding gradients, loop over [dWax, dWaa, dWya, db, dby]. (≈2 lines)
16     for gradient in [dWaa, dWax, dWya, db, dby]:
17         np.clip(gradient, -maxValue, maxValue, out=gradient)
18     ### END CODE HERE ###
19     
20     gradients = {"dWaa": dWaa, "dWax": dWax, "dWya": dWya, "db": db, "dby": dby}
21     return gradients

 1 def sample(parameters, char_to_ix, seed):
 2     """
 3     Sample a sequence of characters according to a sequence of probability distributions output of the RNN
 4 
 5     Arguments:
 6     parameters -- python dictionary containing the parameters Waa, Wax, Wya, by, and b. 
 7     char_to_ix -- python dictionary mapping each character to an index.
 8     seed -- used for grading purposes. Do not worry about it.
 9 
10     Returns:
11     indices -- a list of length n containing the indices of the sampled characters.
12     """
13     
14     # Retrieve parameters and relevant shapes from "parameters" dictionary
15     Waa, Wax, Wya, by, b = parameters[‘Waa‘], parameters[‘Wax‘], parameters[‘Wya‘], parameters[‘by‘], parameters[‘b‘]
16     vocab_size = by.shape[0]
17     n_a = Waa.shape[1]
18     
19     ### START CODE HERE ###
20     # Step 1: Create the one-hot vector x for the first character (initializing the sequence generation). (≈1 line)
21     x = np.zeros((vocab_size, 1))
22     # Step 1‘: Initialize a_prev as zeros (≈1 line)
23     a_prev = np.zeros((n_a, 1))
24     
25     # Create an empty list of indices, this is the list which will contain the list of indices of the characters to generate (≈1 line)
26     indices = []
27     
28     # Idx is a flag to detect a newline character, we initialize it to -1
29     idx=-1
30     
31     # Loop over time-steps t. At each time-step, sample a character from a probability distribution and append 
32     # its index to "indices". We‘ll stop if we reach 50 characters (which should be very unlikely with a well 
33     # trained model), which helps debugging and prevents entering an infinite loop. 
34     counter = 0
35     newline_character = char_to_ix[‘\n‘]
36     
37     while (idx != newline_character and counter != 50):
38         
39         # Step 2: Forward propagate x using the equations (1), (2) and (3)
40         a = np.tanh(np.dot(Wax, x) + np.dot(Waa, a_prev) + b) 
41         z = np.dot(Wya, a) +by
42         y = softmax(z)
43         # for grading purposes
44         np.random.seed(counter + seed)
45  
46         # Step 3: Sample the index of a character within the vocabulary from the probability distribution y
47         idx = np.random.choice(list(range(vocab_size)), p=y.ravel())
48         # Append the index to "indices"
49         indices.append(idx)       
50         
51         # Step 4: Overwrite the input character as the one corresponding to the sampled index.
52         x = np.zeros((vocab_size, 1))
53         x[idx]=1
54         
55         # Update "a_prev" to be "a"
56         a_prev = a
57         
58         # for grading purpose
59         seed += 1
60         counter += 1
61     ### END CODE HERE ###
62 
63     if (counter == 50):
64         indices.append(char_to_ix[‘\n‘]) 
65     return indices

 1 def optimize(X, Y, a_prev, parameters, learning_rate = 0.01):
 2     """
 3     Execute one step of the optimization to train the model.
 4     
 5     Arguments:
 6     X -- list of integers, where each integer is a number that maps to a character in the vocabulary.
 7     Y -- list of integers, exactly the same as X but shifted one index to the left.
 8     a_prev -- previous hidden state.
 9     parameters -- python dictionary containing:
10                         Wax -- Weight matrix multiplying the input, numpy array of shape (n_a, n_x)
11                         Waa -- Weight matrix multiplying the hidden state, numpy array of shape (n_a, n_a)
12                         Wya -- Weight matrix relating the hidden-state to the output, numpy array of shape (n_y, n_a)
13                         b --  Bias, numpy array of shape (n_a, 1)
14                         by -- Bias relating the hidden-state to the output, numpy array of shape (n_y, 1)
15     learning_rate -- learning rate for the model.
16     
17     Returns:
18     loss -- value of the loss function (cross-entropy)
19     gradients -- python dictionary containing:
20                         dWax -- Gradients of input-to-hidden weights, of shape (n_a, n_x)
21                         dWaa -- Gradients of hidden-to-hidden weights, of shape (n_a, n_a)
22                         dWya -- Gradients of hidden-to-output weights, of shape (n_y, n_a)
23                         db -- Gradients of bias vector, of shape (n_a, 1)
24                         dby -- Gradients of output bias vector, of shape (n_y, 1)
25     a[len(X)-1] -- the last hidden state, of shape (n_a, 1)
26     """
27     ### START CODE HERE ###  
28     # Forward propagate through time (≈1 line)
29     loss, cache = rnn_forward(X, Y, a_prev, parameters)
30     
31     # Backpropagate through time (≈1 line)
32     gradients, a = rnn_backward(X, Y, parameters, cache)
33     
34     # Clip your gradients between -5 (min) and 5 (max) (≈1 line)
35     gradients = clip(gradients, maxValue=5)
36     
37     # Update parameters (≈1 line)
38     parameters = update_parameters(parameters, gradients, learning_rate)  
39     ### END CODE HERE ###
40     
41     return loss, gradients, a[len(X)-1]

1         index = j % len(examples)
2         X = [None] + [char_to_ix[ch] for ch in examples[index]] 
3         Y = X[1:] + [char_to_ix["\n"]]

 1 def model(data, ix_to_char, char_to_ix, num_iterations = 35000, n_a = 50, dino_names = 7, vocab_size = 27):
 2     """
 3     Trains the model and generates dinosaur names. 
 4     
 5     Arguments:
 6     data -- text corpus
 7     ix_to_char -- dictionary that maps the index to a character
 8     char_to_ix -- dictionary that maps a character to an index
 9     num_iterations -- number of iterations to train the model for
10     n_a -- number of units of the RNN cell
11     dino_names -- number of dinosaur names you want to sample at each iteration. 
12     vocab_size -- number of unique characters found in the text, size of the vocabulary
13     
14     Returns:
15     parameters -- learned parameters
16     """   
17     # Retrieve n_x and n_y from vocab_size
18     n_x, n_y = vocab_size, vocab_size
19     
20     # Initialize parameters
21     parameters = initialize_parameters(n_a, n_x, n_y)
22     
23     # Initialize loss (this is required because we want to smooth our loss, don‘t worry about it)
24     loss = get_initial_loss(vocab_size, dino_names)
25     
26     # Build list of all dinosaur names (training examples).
27     with open("datasets/dinos.txt") as f:
28         examples = f.readlines()
29     examples = [x.lower().strip() for x in examples]
30     
31     # Shuffle list of all dinosaur names
32     np.random.seed(0)
33     np.random.shuffle(examples)
34     
35     # Initialize the hidden state of your LSTM
36     a_prev = np.zeros((n_a, 1))
37     
38     # Optimization loop
39     for j in range(num_iterations):
40         
41         ### START CODE HERE ###
42         # Use the hint above to define one training example (X,Y) (≈ 2 lines)
43         index = j % len(examples)
44         X = [None] + [char_to_ix[ch] for ch in examples[index]] 
45         Y = X[1:] + [char_to_ix["\n"]]
46         
47         # Perform one optimization step: Forward-prop -> Backward-prop -> Clip -> Update parameters
48         # Choose a learning rate of 0.01
49         curr_loss, gradients, a_prev = optimize(X, Y, a_prev, parameters, learning_rate = 0.01)
50         ### END CODE HERE ###
51         
52         # Use a latency trick to keep the loss smooth. It happens here to accelerate the training.
53         loss = smooth(loss, curr_loss)
54 
55         # Every 2000 Iteration, generate "n" characters thanks to sample() to check if the model is learning properly
56         if j % 2000 == 0:
57             
58             print(‘Iteration: %d, Loss: %f‘ % (j, loss) + ‘\n‘)
59             
60             # The number of dinosaur names to print
61             seed = 0
62             for name in range(dino_names):
63                 
64                 # Sample indices and print them
65                 sampled_indices = sample(parameters, char_to_ix, seed)
66                 print_sample(sampled_indices, ix_to_char)
67                 
68                 seed += 1  # To get the same result for grading purposed, increment the seed by one. 
69       
70             print(‘\n‘)
71         
72     return parameters

parameters = model(data, ix_to_char, char_to_ix)