BP网络中的反向传播

  1 import numpy as np
  2 
  3 def tanh(x):
  4     return np.tanh(x)
  5 
  6 def tanh_prime(x):
  7     x = np.tanh(x)
  8     return 1.0 - x ** 2
  9 
 10 class Network(object):
 11 
 12     def __init__(self, sizes):
 13         self.num_layers = len(sizes)
 14         self.sizes = sizes
 15         # self.biases is a column vector
 16         # self.weights‘ structure is the same as in the book: http://neuralnetworksanddeeplearning.com/chap2.html
 17         self.biases = [np.random.randn(y, 1) for y in sizes[1:]]
 18         self.weights = [np.random.randn(y, x)
 19                         for x, y in zip(sizes[:-1], sizes[1:])]
 20 
 21     def feedforward(self, a):
 22         """Return the output of the network if "a" is input."""
 23         for b, w in zip(self.biases, self.weights):
 24             a = sigmoid(np.dot(w, a) + b)
 25         return a
 26 
 27     def update_mini_batch(self, mini_batch, learning_rate = 0.2):
 28         """Update the network‘s weights and biases by applying
 29         gradient descent using backpropagation to a single mini batch.
 30         The "mini_batch" is a list of tuples "(x, y)"."""
 31         nabla_b = [np.zeros(b.shape) for b in self.biases]
 32         nabla_w = [np.zeros(w.shape) for w in self.weights]
 33         
 34         # delta_nabla_b is dC/db, delta_nabla_w is dC/dw
 35         for x, y in mini_batch:
 36             delta_nabla_b, delta_nabla_w = self.backprop(x, y)
 37             nabla_b = [nb + dnb for nb, dnb in zip(nabla_b, delta_nabla_b)]
 38             nabla_w = [nw + dnw for nw, dnw in zip(nabla_w, delta_nabla_w)]
 39         self.weights = [w - (learning_rate/len(mini_batch)) * nw
 40                         for w, nw in zip(self.weights, nabla_w)]
 41         self.biases = [b - (learning_rate/len(mini_batch)) * nb
 42                        for b, nb in zip(self.biases, nabla_b)]
 43 
 44     def backprop(self, x, y):
 45         """Return a tuple ``(nabla_b, nabla_w)`` representing the
 46         gradient for the cost function C_x.  ``nabla_b`` and
 47         ``nabla_w`` are layer-by-layer lists of numpy arrays, similar
 48         to ``self.biases`` and ``self.weights``."""
 49         nabla_b = [np.zeros(b.shape) for b in self.biases]
 50         nabla_w = [np.zeros(w.shape) for w in self.weights]
 51         
 52         # feedforward
 53         activation = x
 54         activations = [x]   # list to store all the activations, layer by layer
 55         zs = []             # list to store all the z vectors, layer by layer
 56         
 57         # After this loop, activations = [a0, a1, ..., aL], zs = [z1, z2, ..., zL]
 58         for b, w in zip(self.biases, self.weights):
 59             z = np.dot(w, activation) + b
 60             zs.append(z)
 61             activation = sigmoid(z)
 62             activations.append(activation)
 63         
 64         # backward pass
 65         # delta = deltaL .* sigma‘(zL)
 66         delta = self.cost_derivative(activations[-1], y) *  67                 sigmoid_prime(zs[-1])
 68         
 69         # dC/dbL = delta
 70         # dC/dwL = deltaL * a(L-1)^T
 71         nabla_b[-1] = delta
 72         nabla_w[-1] = np.dot(delta, activations[-2].transpose())
 73 
 74         ‘‘‘Note that the variable l in the loop below is used a little
 75         differently to the notation in Chapter 2 of the book. Here,
 76         l = 1 means the last layer of neurons, l = 2 is the
 77         second-last layer, and so on. It‘s a renumbering of the
 78         scheme in the book, used here to take advantage of the fact
 79         that Python can use negative indices in lists.‘‘‘
 80         # z = z(L-l+1), here, l start from 2, end with self.num_layers-1, namely, L-1
 81         # delta = delta(L-l+1) = w(L-l+2)^T * delta(L-l+2) .* z(L-l+1)
 82         # nabla_b[L-l+1] = delta(L-l+1)
 83         # nabla_w[L-l+1] = delta(L-l+1) * a(L-l)^T
 84         for l in xrange(2, self.num_layers):
 85             z = zs[-l]
 86             sp = sigmoid_prime(z)
 87             delta = np.dot(self.weights[-l + 1].transpose(), delta) * sp
 88             nabla_b[-l] = delta
 89             nabla_w[-l] = np.dot(delta, activations[-l - 1].transpose())
 90         return (nabla_b, nabla_w)
 91 
 92     def evaluate(self, test_data):
 93         """Return the number of test inputs for which the neural
 94         network outputs the correct result. Note that the neural
 95         network‘s output is assumed to be the index of whichever
 96         neuron in the final layer has the highest activation."""
 97         test_results = self.feedforward(test_data)
 98         return test_results
 99 
100     def cost_derivative(self, output_activations, y):
101         return (output_activations - y)
102 
103 #### Miscellaneous functions
104 def sigmoid(z):
105     return 1.0/(1.0 + np.exp(-z))
106 
107 # derivative of the sigmoid function
108 def sigmoid_prime(z):
109     return sigmoid(z) * (1 - sigmoid(z))
110 
111 if __name__ == ‘__main__‘:
112 
113     nn = Network([2, 2, 2])
114 
115     X = np.array([[0, 0],
116                   [0, 1],
117                   [1, 0],
118                   [1, 1]])
119 
120     y = np.array([[1, 1],
121                   [1, 0],
122                   [0, 1],
123                   [0, 0]])
124     
125     for k in range(40000):
126         if k % 10000 == 0:
127             print ‘epochs:‘, k
128         # Randomly select a sample.
129         i = np.random.randint(X.shape[0])
130         nn.update_mini_batch(zip([np.atleast_2d(X[i]).T], [np.atleast_2d(y[i]).T]))
131 
132     for e in X:
133         print(e, nn.evaluate(np.atleast_2d(e).T))