深度残差网络+自适应参数化ReLU激活函数（调参记录8）

  1 #!/usr/bin/env python3
  2 # -*- coding: utf-8 -*-
  3 """
  4 Created on Tue Apr 14 04:17:45 2020
  5 Implemented using TensorFlow 1.0.1 and Keras 2.2.1
  6 
  7 Minghang Zhao, Shisheng Zhong, Xuyun Fu, Baoping Tang, Shaojiang Dong, Michael Pecht,
  8 Deep Residual Networks with Adaptively Parametric Rectifier Linear Units for Fault Diagnosis, 
  9 IEEE Transactions on Industrial Electronics, 2020,  DOI: 10.1109/TIE.2020.2972458 
 10 
 11 @author: Minghang Zhao
 12 """
 13 
 14 from __future__ import print_function
 15 import keras
 16 import numpy as np
 17 from keras.datasets import cifar10
 18 from keras.layers import Dense, Conv2D, BatchNormalization, Activation, Minimum
 19 from keras.layers import AveragePooling2D, Input, GlobalAveragePooling2D, Concatenate, Reshape
 20 from keras.regularizers import l2
 21 from keras import backend as K
 22 from keras.models import Model
 23 from keras import optimizers
 24 from keras.preprocessing.image import ImageDataGenerator
 25 from keras.callbacks import LearningRateScheduler
 26 K.set_learning_phase(1)
 27 
 28 # The data, split between train and test sets
 29 (x_train, y_train), (x_test, y_test) = cifar10.load_data()
 30 
 31 # Noised data
 32 x_train = x_train.astype(‘float32‘) / 255.
 33 x_test = x_test.astype(‘float32‘) / 255.
 34 x_test = x_test-np.mean(x_train)
 35 x_train = x_train-np.mean(x_train)
 36 print(‘x_train shape:‘, x_train.shape)
 37 print(x_train.shape[0], ‘train samples‘)
 38 print(x_test.shape[0], ‘test samples‘)
 39 
 40 # convert class vectors to binary class matrices
 41 y_train = keras.utils.to_categorical(y_train, 10)
 42 y_test = keras.utils.to_categorical(y_test, 10)
 43 
 44 # Schedule the learning rate, multiply 0.1 every 300 epoches
 45 def scheduler(epoch):
 46     if epoch % 300 == 0 and epoch != 0:
 47         lr = K.get_value(model.optimizer.lr)
 48         K.set_value(model.optimizer.lr, lr * 0.1)
 49         print("lr changed to {}".format(lr * 0.1))
 50     return K.get_value(model.optimizer.lr)
 51 
 52 # An adaptively parametric rectifier linear unit (APReLU)
 53 def aprelu(inputs):
 54     # get the number of channels
 55     channels = inputs.get_shape().as_list()[-1]
 56     # get a zero feature map
 57     zeros_input = keras.layers.subtract([inputs, inputs])
 58     # get a feature map with only positive features
 59     pos_input = Activation(‘relu‘)(inputs)
 60     # get a feature map with only negative features
 61     neg_input = Minimum()([inputs,zeros_input])
 62     # define a network to obtain the scaling coefficients
 63     scales_p = GlobalAveragePooling2D()(pos_input)
 64     scales_n = GlobalAveragePooling2D()(neg_input)
 65     scales = Concatenate()([scales_n, scales_p])
 66     scales = Dense(channels, activation=‘linear‘, kernel_initializer=‘he_normal‘, kernel_regularizer=l2(1e-4))(scales)
 67     scales = BatchNormalization()(scales)
 68     scales = Activation(‘relu‘)(scales)
 69     scales = Dense(channels, activation=‘linear‘, kernel_initializer=‘he_normal‘, kernel_regularizer=l2(1e-4))(scales)
 70     scales = BatchNormalization()(scales)
 71     scales = Activation(‘sigmoid‘)(scales)
 72     scales = Reshape((1,1,channels))(scales)
 73     # apply a paramtetric relu
 74     neg_part = keras.layers.multiply([scales, neg_input])
 75     return keras.layers.add([pos_input, neg_part])
 76 
 77 # Residual Block
 78 def residual_block(incoming, nb_blocks, out_channels, downsample=False,
 79                    downsample_strides=2):
 80     
 81     residual = incoming
 82     in_channels = incoming.get_shape().as_list()[-1]
 83     
 84     for i in range(nb_blocks):
 85         
 86         identity = residual
 87         
 88         if not downsample:
 89             downsample_strides = 1
 90         
 91         residual = BatchNormalization()(residual)
 92         residual = aprelu(residual)
 93         residual = Conv2D(out_channels, 3, strides=(downsample_strides, downsample_strides), 
 94                           padding=‘same‘, kernel_initializer=‘he_normal‘, 
 95                           kernel_regularizer=l2(1e-4))(residual)
 96         
 97         residual = BatchNormalization()(residual)
 98         residual = aprelu(residual)
 99         residual = Conv2D(out_channels, 3, padding=‘same‘, kernel_initializer=‘he_normal‘, 
100                           kernel_regularizer=l2(1e-4))(residual)
101         
102         # Downsampling
103         if downsample_strides > 1:
104             identity = AveragePooling2D(pool_size=(1,1), strides=(2,2))(identity)
105             
106         # Zero_padding to match channels
107         if in_channels != out_channels:
108             zeros_identity = keras.layers.subtract([identity, identity])
109             identity = keras.layers.concatenate([identity, zeros_identity])
110             in_channels = out_channels
111         
112         residual = keras.layers.add([residual, identity])
113     
114     return residual
115 
116 
117 # define and train a model
118 inputs = Input(shape=(32, 32, 3))
119 net = Conv2D(16, 3, padding=‘same‘, kernel_initializer=‘he_normal‘, kernel_regularizer=l2(1e-4))(inputs)
120 # net = residual_block(net, 9, 16, downsample=False)
121 net = residual_block(net, 1, 32, downsample=True)
122 # net = residual_block(net, 8, 32, downsample=False)
123 net = residual_block(net, 1, 64, downsample=True)
124 # net = residual_block(net, 8, 64, downsample=False)
125 net = BatchNormalization()(net)
126 net = aprelu(net)
127 net = GlobalAveragePooling2D()(net)
128 outputs = Dense(10, activation=‘softmax‘, kernel_initializer=‘he_normal‘, kernel_regularizer=l2(1e-4))(net)
129 model = Model(inputs=inputs, outputs=outputs)
130 sgd = optimizers.SGD(lr=0.1, decay=0., momentum=0.9, nesterov=True)
131 model.compile(loss=‘categorical_crossentropy‘, optimizer=sgd, metrics=[‘accuracy‘])
132 
133 # data augmentation
134 datagen = ImageDataGenerator(
135     # randomly rotate images in the range (deg 0 to 180)
136     rotation_range=30,
137     # randomly flip images
138     horizontal_flip=True,
139     # randomly shift images horizontally
140     width_shift_range=0.125,
141     # randomly shift images vertically
142     height_shift_range=0.125)
143 
144 reduce_lr = LearningRateScheduler(scheduler)
145 # fit the model on the batches generated by datagen.flow().
146 model.fit_generator(datagen.flow(x_train, y_train, batch_size=100),
147                     validation_data=(x_test, y_test), epochs=1000, 
148                     verbose=1, callbacks=[reduce_lr], workers=4)
149 
150 # get results
151 K.set_learning_phase(0)
152 DRSN_train_score = model.evaluate(x_train, y_train, batch_size=100, verbose=0)
153 print(‘Train loss:‘, DRSN_train_score[0])
154 print(‘Train accuracy:‘, DRSN_train_score[1])
155 DRSN_test_score = model.evaluate(x_test, y_test, batch_size=100, verbose=0)
156 print(‘Test loss:‘, DRSN_test_score[0])
157 print(‘Test accuracy:‘, DRSN_test_score[1])

  1 Epoch 755/1000
  2 16s 33ms/step - loss: 0.2417 - acc: 0.9453 - val_loss: 0.5321 - val_acc: 0.8719
  3 Epoch 756/1000
  4 17s 33ms/step - loss: 0.2436 - acc: 0.9450 - val_loss: 0.5294 - val_acc: 0.8714
  5 Epoch 757/1000
  6 17s 33ms/step - loss: 0.2463 - acc: 0.9435 - val_loss: 0.5351 - val_acc: 0.8710
  7 Epoch 758/1000
  8 17s 33ms/step - loss: 0.2461 - acc: 0.9444 - val_loss: 0.5356 - val_acc: 0.8708
  9 Epoch 759/1000
 10 17s 33ms/step - loss: 0.2440 - acc: 0.9449 - val_loss: 0.5355 - val_acc: 0.8674
 11 Epoch 760/1000
 12 16s 32ms/step - loss: 0.2431 - acc: 0.9447 - val_loss: 0.5329 - val_acc: 0.8711
 13 Epoch 761/1000
 14 16s 32ms/step - loss: 0.2469 - acc: 0.9440 - val_loss: 0.5294 - val_acc: 0.8712
 15 Epoch 762/1000
 16 16s 32ms/step - loss: 0.2420 - acc: 0.9445 - val_loss: 0.5296 - val_acc: 0.8725
 17 Epoch 763/1000
 18 16s 33ms/step - loss: 0.2411 - acc: 0.9469 - val_loss: 0.5353 - val_acc: 0.8712
 19 Epoch 764/1000
 20 17s 33ms/step - loss: 0.2426 - acc: 0.9453 - val_loss: 0.5363 - val_acc: 0.8715
 21 Epoch 765/1000
 22 17s 33ms/step - loss: 0.2415 - acc: 0.9449 - val_loss: 0.5322 - val_acc: 0.8718
 23 Epoch 766/1000
 24 17s 33ms/step - loss: 0.2392 - acc: 0.9450 - val_loss: 0.5321 - val_acc: 0.8696
 25 Epoch 767/1000
 26 17s 34ms/step - loss: 0.2396 - acc: 0.9463 - val_loss: 0.5353 - val_acc: 0.8699
 27 Epoch 768/1000
 28 16s 33ms/step - loss: 0.2457 - acc: 0.9436 - val_loss: 0.5314 - val_acc: 0.8713
 29 Epoch 769/1000
 30 16s 32ms/step - loss: 0.2442 - acc: 0.9440 - val_loss: 0.5327 - val_acc: 0.8740
 31 Epoch 770/1000
 32 16s 32ms/step - loss: 0.2449 - acc: 0.9445 - val_loss: 0.5336 - val_acc: 0.8706
 33 Epoch 771/1000
 34 16s 33ms/step - loss: 0.2408 - acc: 0.9458 - val_loss: 0.5359 - val_acc: 0.8706
 35 Epoch 772/1000
 36 17s 33ms/step - loss: 0.2400 - acc: 0.9454 - val_loss: 0.5362 - val_acc: 0.8690
 37 Epoch 773/1000
 38 17s 33ms/step - loss: 0.2409 - acc: 0.9455 - val_loss: 0.5343 - val_acc: 0.8688
 39 Epoch 774/1000
 40 17s 33ms/step - loss: 0.2390 - acc: 0.9464 - val_loss: 0.5321 - val_acc: 0.8690
 41 Epoch 775/1000
 42 17s 33ms/step - loss: 0.2438 - acc: 0.9439 - val_loss: 0.5363 - val_acc: 0.8700
 43 Epoch 776/1000
 44 16s 33ms/step - loss: 0.2424 - acc: 0.9441 - val_loss: 0.5359 - val_acc: 0.8691
 45 Epoch 777/1000
 46 16s 32ms/step - loss: 0.2398 - acc: 0.9448 - val_loss: 0.5354 - val_acc: 0.8689
 47 Epoch 778/1000
 48 16s 32ms/step - loss: 0.2420 - acc: 0.9450 - val_loss: 0.5385 - val_acc: 0.8681
 49 Epoch 779/1000
 50 16s 33ms/step - loss: 0.2391 - acc: 0.9459 - val_loss: 0.5320 - val_acc: 0.8698
 51 Epoch 780/1000
 52 17s 33ms/step - loss: 0.2425 - acc: 0.9443 - val_loss: 0.5363 - val_acc: 0.8683
 53 Epoch 781/1000
 54 17s 33ms/step - loss: 0.2381 - acc: 0.9457 - val_loss: 0.5345 - val_acc: 0.8680
 55 Epoch 782/1000
 56 17s 33ms/step - loss: 0.2374 - acc: 0.9470 - val_loss: 0.5301 - val_acc: 0.8710
 57 Epoch 783/1000
 58 17s 33ms/step - loss: 0.2389 - acc: 0.9460 - val_loss: 0.5334 - val_acc: 0.8696
 59 Epoch 784/1000
 60 16s 33ms/step - loss: 0.2386 - acc: 0.9473 - val_loss: 0.5286 - val_acc: 0.8691
 61 Epoch 785/1000
 62 16s 32ms/step - loss: 0.2387 - acc: 0.9447 - val_loss: 0.5362 - val_acc: 0.8690
 63 Epoch 786/1000
 64 16s 32ms/step - loss: 0.2386 - acc: 0.9461 - val_loss: 0.5345 - val_acc: 0.8690
 65 Epoch 787/1000
 66 16s 33ms/step - loss: 0.2358 - acc: 0.9464 - val_loss: 0.5344 - val_acc: 0.8709
 67 Epoch 788/1000
 68 17s 33ms/step - loss: 0.2374 - acc: 0.9463 - val_loss: 0.5322 - val_acc: 0.8716
 69 Epoch 789/1000
 70 17s 33ms/step - loss: 0.2388 - acc: 0.9449 - val_loss: 0.5267 - val_acc: 0.8744
 71 Epoch 790/1000
 72 17s 33ms/step - loss: 0.2349 - acc: 0.9471 - val_loss: 0.5347 - val_acc: 0.8706
 73 Epoch 791/1000
 74 17s 33ms/step - loss: 0.2391 - acc: 0.9458 - val_loss: 0.5336 - val_acc: 0.8693
 75 Epoch 792/1000
 76 16s 32ms/step - loss: 0.2385 - acc: 0.9447 - val_loss: 0.5387 - val_acc: 0.8687
 77 Epoch 793/1000
 78 16s 32ms/step - loss: 0.2356 - acc: 0.9471 - val_loss: 0.5374 - val_acc: 0.8686
 79 Epoch 794/1000
 80 16s 33ms/step - loss: 0.2398 - acc: 0.9455 - val_loss: 0.5375 - val_acc: 0.8678
 81 Epoch 795/1000
 82 17s 33ms/step - loss: 0.2365 - acc: 0.9452 - val_loss: 0.5302 - val_acc: 0.8710
 83 Epoch 796/1000
 84 16s 33ms/step - loss: 0.2358 - acc: 0.9469 - val_loss: 0.5374 - val_acc: 0.8711
 85 Epoch 797/1000
 86 17s 33ms/step - loss: 0.2379 - acc: 0.9450 - val_loss: 0.5335 - val_acc: 0.8686
 87 Epoch 798/1000
 88 17s 34ms/step - loss: 0.2369 - acc: 0.9454 - val_loss: 0.5340 - val_acc: 0.8687
 89 Epoch 799/1000
 90 17s 33ms/step - loss: 0.2337 - acc: 0.9470 - val_loss: 0.5360 - val_acc: 0.8698
 91 Epoch 800/1000
 92 16s 33ms/step - loss: 0.2412 - acc: 0.9432 - val_loss: 0.5353 - val_acc: 0.8697
 93 Epoch 801/1000
 94 16s 32ms/step - loss: 0.2357 - acc: 0.9456 - val_loss: 0.5362 - val_acc: 0.8689
 95 Epoch 802/1000
 96 16s 33ms/step - loss: 0.2347 - acc: 0.9464 - val_loss: 0.5371 - val_acc: 0.8698
 97 Epoch 803/1000
 98 16s 33ms/step - loss: 0.2311 - acc: 0.9474 - val_loss: 0.5328 - val_acc: 0.8683
 99 Epoch 804/1000
100 17s 33ms/step - loss: 0.2396 - acc: 0.9449 - val_loss: 0.5358 - val_acc: 0.8699
101 Epoch 805/1000
102 17s 33ms/step - loss: 0.2380 - acc: 0.9459 - val_loss: 0.5351 - val_acc: 0.8689
103 Epoch 806/1000
104 17s 33ms/step - loss: 0.2351 - acc: 0.9452 - val_loss: 0.5362 - val_acc: 0.8693
105 Epoch 807/1000
106 17s 33ms/step - loss: 0.2350 - acc: 0.9463 - val_loss: 0.5281 - val_acc: 0.8711
107 Epoch 808/1000
108 16s 33ms/step - loss: 0.2333 - acc: 0.9467 - val_loss: 0.5347 - val_acc: 0.8703
109 Epoch 809/1000
110 16s 33ms/step - loss: 0.2333 - acc: 0.9463 - val_loss: 0.5338 - val_acc: 0.8709
111 Epoch 810/1000
112 16s 32ms/step - loss: 0.2330 - acc: 0.9478 - val_loss: 0.5351 - val_acc: 0.8704
113 Epoch 811/1000
114 17s 33ms/step - loss: 0.2374 - acc: 0.9439 - val_loss: 0.5400 - val_acc: 0.8696
115 Epoch 812/1000
116 16s 33ms/step - loss: 0.2321 - acc: 0.9467 - val_loss: 0.5361 - val_acc: 0.8709
117 Epoch 813/1000
118 17s 33ms/step - loss: 0.2349 - acc: 0.9457 - val_loss: 0.5307 - val_acc: 0.8706
119 Epoch 814/1000
120 17s 33ms/step - loss: 0.2361 - acc: 0.9457 - val_loss: 0.5368 - val_acc: 0.8686
121 Epoch 815/1000
122 17s 33ms/step - loss: 0.2351 - acc: 0.9459 - val_loss: 0.5344 - val_acc: 0.8692
123 Epoch 816/1000
124 16s 33ms/step - loss: 0.2362 - acc: 0.9464 - val_loss: 0.5297 - val_acc: 0.8693
125 Epoch 817/1000
126 16s 32ms/step - loss: 0.2326 - acc: 0.9471 - val_loss: 0.5344 - val_acc: 0.8688
127 Epoch 818/1000
128 16s 33ms/step - loss: 0.2353 - acc: 0.9448 - val_loss: 0.5418 - val_acc: 0.8698
129 Epoch 819/1000
130 16s 33ms/step - loss: 0.2361 - acc: 0.9439 - val_loss: 0.5353 - val_acc: 0.8705
131 Epoch 820/1000
132 17s 33ms/step - loss: 0.2320 - acc: 0.9468 - val_loss: 0.5411 - val_acc: 0.8701
133 Epoch 821/1000
134 17s 33ms/step - loss: 0.2311 - acc: 0.9466 - val_loss: 0.5360 - val_acc: 0.8683
135 Epoch 822/1000
136 17s 34ms/step - loss: 0.2298 - acc: 0.9464 - val_loss: 0.5369 - val_acc: 0.8722
137 Epoch 823/1000
138 17s 33ms/step - loss: 0.2360 - acc: 0.9450 - val_loss: 0.5409 - val_acc: 0.8657
139 Epoch 824/1000
140 16s 32ms/step - loss: 0.2319 - acc: 0.9471 - val_loss: 0.5340 - val_acc: 0.8689
141 Epoch 825/1000
142 16s 32ms/step - loss: 0.2307 - acc: 0.9483 - val_loss: 0.5338 - val_acc: 0.8695
143 Epoch 826/1000
144 16s 32ms/step - loss: 0.2337 - acc: 0.9465 - val_loss: 0.5364 - val_acc: 0.8676
145 Epoch 827/1000
146 16s 33ms/step - loss: 0.2348 - acc: 0.9454 - val_loss: 0.5367 - val_acc: 0.8676
147 Epoch 828/1000
148 17s 33ms/step - loss: 0.2334 - acc: 0.9453 - val_loss: 0.5284 - val_acc: 0.8699
149 Epoch 829/1000
150 17s 34ms/step - loss: 0.2334 - acc: 0.9459 - val_loss: 0.5325 - val_acc: 0.8689
151 Epoch 830/1000
152 17s 33ms/step - loss: 0.2332 - acc: 0.9462 - val_loss: 0.5346 - val_acc: 0.8701
153 Epoch 831/1000
154 17s 33ms/step - loss: 0.2355 - acc: 0.9438 - val_loss: 0.5329 - val_acc: 0.8687
155 Epoch 832/1000
156 16s 33ms/step - loss: 0.2325 - acc: 0.9459 - val_loss: 0.5325 - val_acc: 0.8694
157 Epoch 833/1000
158 16s 33ms/step - loss: 0.2299 - acc: 0.9480 - val_loss: 0.5328 - val_acc: 0.8683
159 Epoch 834/1000
160 16s 33ms/step - loss: 0.2328 - acc: 0.9462 - val_loss: 0.5345 - val_acc: 0.8686
161 Epoch 835/1000
162 16s 33ms/step - loss: 0.2310 - acc: 0.9468 - val_loss: 0.5396 - val_acc: 0.8684
163 Epoch 836/1000
164 17s 33ms/step - loss: 0.2300 - acc: 0.9477 - val_loss: 0.5312 - val_acc: 0.8680
165 Epoch 837/1000
166 17s 33ms/step - loss: 0.2316 - acc: 0.9470 - val_loss: 0.5363 - val_acc: 0.8696
167 Epoch 838/1000
168 17s 33ms/step - loss: 0.2287 - acc: 0.9466 - val_loss: 0.5400 - val_acc: 0.8691
169 Epoch 839/1000
170 17s 33ms/step - loss: 0.2324 - acc: 0.9456 - val_loss: 0.5354 - val_acc: 0.8691
171 Epoch 840/1000
172 16s 33ms/step - loss: 0.2321 - acc: 0.9456 - val_loss: 0.5269 - val_acc: 0.8701
173 Epoch 841/1000
174 16s 32ms/step - loss: 0.2328 - acc: 0.9440 - val_loss: 0.5319 - val_acc: 0.8713
175 Epoch 842/1000
176 16s 32ms/step - loss: 0.2304 - acc: 0.9454 - val_loss: 0.5295 - val_acc: 0.8697
177 Epoch 843/1000
178 16s 33ms/step - loss: 0.2287 - acc: 0.9459 - val_loss: 0.5329 - val_acc: 0.8720
179 Epoch 844/1000
180 17s 33ms/step - loss: 0.2315 - acc: 0.9452 - val_loss: 0.5334 - val_acc: 0.8709
181 Epoch 845/1000
182 17s 34ms/step - loss: 0.2313 - acc: 0.9461 - val_loss: 0.5357 - val_acc: 0.8691
183 Epoch 846/1000
184 17s 33ms/step - loss: 0.2283 - acc: 0.9470 - val_loss: 0.5292 - val_acc: 0.8743
185 Epoch 847/1000
186 17s 33ms/step - loss: 0.2308 - acc: 0.9447 - val_loss: 0.5303 - val_acc: 0.8713
187 Epoch 848/1000
188 16s 33ms/step - loss: 0.2320 - acc: 0.9458 - val_loss: 0.5297 - val_acc: 0.8675
189 Epoch 849/1000
190 16s 32ms/step - loss: 0.2261 - acc: 0.9473 - val_loss: 0.5278 - val_acc: 0.8712
191 Epoch 850/1000
192 16s 33ms/step - loss: 0.2289 - acc: 0.9466 - val_loss: 0.5329 - val_acc: 0.8710
193 Epoch 851/1000
194 16s 33ms/step - loss: 0.2291 - acc: 0.9474 - val_loss: 0.5331 - val_acc: 0.8715
195 Epoch 852/1000
196 17s 33ms/step - loss: 0.2312 - acc: 0.9463 - val_loss: 0.5269 - val_acc: 0.8727
197 Epoch 853/1000
198 17s 33ms/step - loss: 0.2319 - acc: 0.9458 - val_loss: 0.5287 - val_acc: 0.8701
199 Epoch 854/1000
200 17s 34ms/step - loss: 0.2291 - acc: 0.9461 - val_loss: 0.5300 - val_acc: 0.8731
201 Epoch 855/1000
202 17s 33ms/step - loss: 0.2294 - acc: 0.9469 - val_loss: 0.5342 - val_acc: 0.8703
203 Epoch 856/1000
204 16s 32ms/step - loss: 0.2305 - acc: 0.9456 - val_loss: 0.5324 - val_acc: 0.8703
205 Epoch 857/1000
206 16s 33ms/step - loss: 0.2318 - acc: 0.9448 - val_loss: 0.5338 - val_acc: 0.8677
207 Epoch 858/1000
208 16s 33ms/step - loss: 0.2286 - acc: 0.9466 - val_loss: 0.5299 - val_acc: 0.8688
209 Epoch 859/1000
210 17s 33ms/step - loss: 0.2302 - acc: 0.9467 - val_loss: 0.5329 - val_acc: 0.8686
211 Epoch 860/1000
212 17s 33ms/step - loss: 0.2305 - acc: 0.9457 - val_loss: 0.5350 - val_acc: 0.8687
213 Epoch 861/1000
214 17s 33ms/step - loss: 0.2284 - acc: 0.9457 - val_loss: 0.5376 - val_acc: 0.8689
215 Epoch 862/1000
216 17s 33ms/step - loss: 0.2302 - acc: 0.9460 - val_loss: 0.5317 - val_acc: 0.8705
217 Epoch 863/1000
218 17s 33ms/step - loss: 0.2276 - acc: 0.9462 - val_loss: 0.5327 - val_acc: 0.8694
219 Epoch 864/1000
220 16s 33ms/step - loss: 0.2273 - acc: 0.9471 - val_loss: 0.5338 - val_acc: 0.8706
221 Epoch 865/1000
222 16s 32ms/step - loss: 0.2278 - acc: 0.9462 - val_loss: 0.5311 - val_acc: 0.8703
223 Epoch 866/1000
224 16s 32ms/step - loss: 0.2277 - acc: 0.9455 - val_loss: 0.5312 - val_acc: 0.8727
225 Epoch 867/1000
226 16s 33ms/step - loss: 0.2282 - acc: 0.9467 - val_loss: 0.5315 - val_acc: 0.8707
227 Epoch 868/1000
228 17s 33ms/step - loss: 0.2264 - acc: 0.9458 - val_loss: 0.5371 - val_acc: 0.8694
229 Epoch 869/1000
230 17s 33ms/step - loss: 0.2266 - acc: 0.9470 - val_loss: 0.5354 - val_acc: 0.8684
231 Epoch 870/1000
232 17s 33ms/step - loss: 0.2269 - acc: 0.9473 - val_loss: 0.5325 - val_acc: 0.8674
233 Epoch 871/1000
234 17s 33ms/step - loss: 0.2269 - acc: 0.9481 - val_loss: 0.5349 - val_acc: 0.8706 ETA: 12s - loss: 0.2107 - acc: 0.9536
235 Epoch 872/1000
236 16s 32ms/step - loss: 0.2230 - acc: 0.9476 - val_loss: 0.5338 - val_acc: 0.8709
237 Epoch 873/1000
238 16s 32ms/step - loss: 0.2291 - acc: 0.9455 - val_loss: 0.5291 - val_acc: 0.8692
239 Epoch 874/1000
240 16s 32ms/step - loss: 0.2271 - acc: 0.9459 - val_loss: 0.5248 - val_acc: 0.8708
241 Epoch 875/1000
242 16s 33ms/step - loss: 0.2307 - acc: 0.9456 - val_loss: 0.5285 - val_acc: 0.8705
243 Epoch 876/1000
244 17s 33ms/step - loss: 0.2288 - acc: 0.9452 - val_loss: 0.5331 - val_acc: 0.8694
245 Epoch 877/1000
246 17s 33ms/step - loss: 0.2296 - acc: 0.9455 - val_loss: 0.5343 - val_acc: 0.8671
247 Epoch 878/1000
248 17s 33ms/step - loss: 0.2263 - acc: 0.9469 - val_loss: 0.5348 - val_acc: 0.8703
249 Epoch 879/1000
250 17s 33ms/step - loss: 0.2274 - acc: 0.9462 - val_loss: 0.5333 - val_acc: 0.8678
251 Epoch 880/1000
252 16s 33ms/step - loss: 0.2234 - acc: 0.9483 - val_loss: 0.5364 - val_acc: 0.8684
253 Epoch 881/1000
254 16s 32ms/step - loss: 0.2257 - acc: 0.9469 - val_loss: 0.5303 - val_acc: 0.8693
255 Epoch 882/1000
256 16s 32ms/step - loss: 0.2256 - acc: 0.9474 - val_loss: 0.5301 - val_acc: 0.8691
257 Epoch 883/1000
258 16s 33ms/step - loss: 0.2280 - acc: 0.9457 - val_loss: 0.5285 - val_acc: 0.8670
259 Epoch 884/1000
260 17s 33ms/step - loss: 0.2256 - acc: 0.9479 - val_loss: 0.5231 - val_acc: 0.8709
261 Epoch 885/1000
262 17s 33ms/step - loss: 0.2270 - acc: 0.9444 - val_loss: 0.5311 - val_acc: 0.8693
263 Epoch 886/1000
264 17s 34ms/step - loss: 0.2281 - acc: 0.9458 - val_loss: 0.5255 - val_acc: 0.8707
265 Epoch 887/1000
266 17s 33ms/step - loss: 0.2280 - acc: 0.9457 - val_loss: 0.5289 - val_acc: 0.8727
267 Epoch 888/1000
268 16s 33ms/step - loss: 0.2240 - acc: 0.9470 - val_loss: 0.5342 - val_acc: 0.8698
269 Epoch 889/1000
270 16s 32ms/step - loss: 0.2264 - acc: 0.9464 - val_loss: 0.5375 - val_acc: 0.8672
271 Epoch 890/1000
272 16s 32ms/step - loss: 0.2261 - acc: 0.9469 - val_loss: 0.5372 - val_acc: 0.8686
273 Epoch 891/1000
274 16s 33ms/step - loss: 0.2214 - acc: 0.9472 - val_loss: 0.5297 - val_acc: 0.8692
275 Epoch 892/1000
276 17s 33ms/step - loss: 0.2239 - acc: 0.9473 - val_loss: 0.5325 - val_acc: 0.8705
277 Epoch 893/1000
278 17s 34ms/step - loss: 0.2236 - acc: 0.9470 - val_loss: 0.5261 - val_acc: 0.8673
279 Epoch 894/1000
280 17s 33ms/step - loss: 0.2269 - acc: 0.9465 - val_loss: 0.5368 - val_acc: 0.8674
281 Epoch 895/1000
282 17s 33ms/step - loss: 0.2242 - acc: 0.9469 - val_loss: 0.5361 - val_acc: 0.8684
283 Epoch 896/1000
284 16s 33ms/step - loss: 0.2241 - acc: 0.9470 - val_loss: 0.5322 - val_acc: 0.8689
285 Epoch 897/1000
286 17s 34ms/step - loss: 0.2239 - acc: 0.9466 - val_loss: 0.5413 - val_acc: 0.8645
287 Epoch 898/1000
288 17s 34ms/step - loss: 0.2239 - acc: 0.9467 - val_loss: 0.5379 - val_acc: 0.8674
289 Epoch 899/1000
290 17s 34ms/step - loss: 0.2287 - acc: 0.9460 - val_loss: 0.5365 - val_acc: 0.8663
291 Epoch 900/1000
292 17s 34ms/step - loss: 0.2216 - acc: 0.9482 - val_loss: 0.5382 - val_acc: 0.8695
293 Epoch 901/1000
294 lr changed to 9.999999310821295e-05
295 17s 34ms/step - loss: 0.2176 - acc: 0.9493 - val_loss: 0.5316 - val_acc: 0.8716
296 Epoch 902/1000
297 17s 35ms/step - loss: 0.2117 - acc: 0.9507 - val_loss: 0.5298 - val_acc: 0.8711
298 Epoch 903/1000
299 17s 34ms/step - loss: 0.2125 - acc: 0.9514 - val_loss: 0.5297 - val_acc: 0.8705
300 Epoch 904/1000
301 17s 34ms/step - loss: 0.2120 - acc: 0.9520 - val_loss: 0.5282 - val_acc: 0.8713
302 Epoch 905/1000
303 17s 34ms/step - loss: 0.2125 - acc: 0.9510 - val_loss: 0.5284 - val_acc: 0.8710
304 Epoch 906/1000
305 17s 34ms/step - loss: 0.2103 - acc: 0.9520 - val_loss: 0.5281 - val_acc: 0.8719
306 Epoch 907/1000
307 17s 34ms/step - loss: 0.2075 - acc: 0.9528 - val_loss: 0.5279 - val_acc: 0.8719
308 Epoch 908/1000
309 17s 34ms/step - loss: 0.2074 - acc: 0.9526 - val_loss: 0.5284 - val_acc: 0.8713
310 Epoch 909/1000
311 17s 35ms/step - loss: 0.2070 - acc: 0.9530 - val_loss: 0.5271 - val_acc: 0.8705
312 Epoch 910/1000
313 17s 34ms/step - loss: 0.2093 - acc: 0.9532 - val_loss: 0.5271 - val_acc: 0.8712
314 Epoch 911/1000
315 17s 34ms/step - loss: 0.2071 - acc: 0.9528 - val_loss: 0.5277 - val_acc: 0.8705
316 Epoch 912/1000
317 17s 34ms/step - loss: 0.2052 - acc: 0.9541 - val_loss: 0.5279 - val_acc: 0.8701
318 Epoch 913/1000
319 17s 33ms/step - loss: 0.2076 - acc: 0.9533 - val_loss: 0.5279 - val_acc: 0.8696
320 Epoch 914/1000
321 17s 34ms/step - loss: 0.2070 - acc: 0.9525 - val_loss: 0.5284 - val_acc: 0.8690
322 Epoch 915/1000
323 17s 34ms/step - loss: 0.2040 - acc: 0.9542 - val_loss: 0.5286 - val_acc: 0.8694
324 Epoch 916/1000
325 17s 34ms/step - loss: 0.2035 - acc: 0.9550 - val_loss: 0.5276 - val_acc: 0.8702
326 Epoch 917/1000
327 17s 34ms/step - loss: 0.2032 - acc: 0.9549 - val_loss: 0.5278 - val_acc: 0.8701
328 Epoch 918/1000
329 17s 34ms/step - loss: 0.2041 - acc: 0.9540 - val_loss: 0.5274 - val_acc: 0.8707
330 Epoch 919/1000
331 17s 34ms/step - loss: 0.2039 - acc: 0.9541 - val_loss: 0.5278 - val_acc: 0.8706
332 Epoch 920/1000
333 17s 34ms/step - loss: 0.2024 - acc: 0.9552 - val_loss: 0.5284 - val_acc: 0.8716
334 Epoch 921/1000
335 17s 34ms/step - loss: 0.2046 - acc: 0.9534 - val_loss: 0.5271 - val_acc: 0.8714
336 Epoch 922/1000
337 17s 34ms/step - loss: 0.2062 - acc: 0.9539 - val_loss: 0.5275 - val_acc: 0.8709
338 Epoch 923/1000
339 17s 34ms/step - loss: 0.2046 - acc: 0.9545 - val_loss: 0.5276 - val_acc: 0.8708
340 Epoch 924/1000
341 17s 34ms/step - loss: 0.2058 - acc: 0.9526 - val_loss: 0.5259 - val_acc: 0.8702
342 Epoch 925/1000
343 17s 35ms/step - loss: 0.2047 - acc: 0.9537 - val_loss: 0.5269 - val_acc: 0.8700
344 Epoch 926/1000
345 17s 34ms/step - loss: 0.2053 - acc: 0.9533 - val_loss: 0.5270 - val_acc: 0.8712 ETA: 12s - loss: 0.2096 - acc: 0.9512
346 Epoch 927/1000
347 17s 34ms/step - loss: 0.2059 - acc: 0.9532 - val_loss: 0.5264 - val_acc: 0.8712
348 Epoch 928/1000
349 17s 34ms/step - loss: 0.2047 - acc: 0.9544 - val_loss: 0.5287 - val_acc: 0.8694
350 Epoch 929/1000
351 17s 33ms/step - loss: 0.2071 - acc: 0.9531 - val_loss: 0.5276 - val_acc: 0.8695
352 Epoch 930/1000
353 17s 33ms/step - loss: 0.2050 - acc: 0.9544 - val_loss: 0.5281 - val_acc: 0.8697
354 Epoch 931/1000
355 16s 33ms/step - loss: 0.2042 - acc: 0.9538 - val_loss: 0.5275 - val_acc: 0.8688
356 Epoch 932/1000
357 17s 33ms/step - loss: 0.2022 - acc: 0.9551 - val_loss: 0.5274 - val_acc: 0.8700
358 Epoch 933/1000
359 17s 33ms/step - loss: 0.2035 - acc: 0.9546 - val_loss: 0.5293 - val_acc: 0.8702
360 Epoch 934/1000
361 17s 34ms/step - loss: 0.2042 - acc: 0.9547 - val_loss: 0.5289 - val_acc: 0.8695
362 Epoch 935/1000
363 17s 33ms/step - loss: 0.2026 - acc: 0.9550 - val_loss: 0.5290 - val_acc: 0.8707
364 Epoch 936/1000
365 16s 33ms/step - loss: 0.2031 - acc: 0.9545 - val_loss: 0.5307 - val_acc: 0.8696
366 Epoch 937/1000
367 16s 32ms/step - loss: 0.2034 - acc: 0.9549 - val_loss: 0.5291 - val_acc: 0.8699
368 Epoch 938/1000
369 16s 33ms/step - loss: 0.2042 - acc: 0.9534 - val_loss: 0.5273 - val_acc: 0.8713
370 Epoch 939/1000
371 17s 33ms/step - loss: 0.2067 - acc: 0.9536 - val_loss: 0.5276 - val_acc: 0.8707
372 Epoch 940/1000
373 17s 33ms/step - loss: 0.2035 - acc: 0.9544 - val_loss: 0.5285 - val_acc: 0.8706
374 Epoch 941/1000
375 17s 34ms/step - loss: 0.2011 - acc: 0.9553 - val_loss: 0.5285 - val_acc: 0.8704
376 Epoch 942/1000
377 17s 33ms/step - loss: 0.2026 - acc: 0.9549 - val_loss: 0.5278 - val_acc: 0.8715
378 Epoch 943/1000
379 17s 33ms/step - loss: 0.2005 - acc: 0.9557 - val_loss: 0.5292 - val_acc: 0.8713
380 Epoch 944/1000
381 16s 33ms/step - loss: 0.2029 - acc: 0.9543 - val_loss: 0.5300 - val_acc: 0.8696
382 Epoch 945/1000
383 16s 33ms/step - loss: 0.2020 - acc: 0.9548 - val_loss: 0.5295 - val_acc: 0.8702
384 Epoch 946/1000
385 16s 33ms/step - loss: 0.2020 - acc: 0.9553 - val_loss: 0.5305 - val_acc: 0.8683
386 Epoch 947/1000
387 16s 33ms/step - loss: 0.2012 - acc: 0.9549 - val_loss: 0.5295 - val_acc: 0.8685
388 Epoch 948/1000
389 17s 33ms/step - loss: 0.2003 - acc: 0.9557 - val_loss: 0.5292 - val_acc: 0.8694 ETA: 2s - loss: 0.1982 - acc: 0.9567
390 Epoch 949/1000
391 17s 33ms/step - loss: 0.2058 - acc: 0.9534 - val_loss: 0.5290 - val_acc: 0.8700
392 Epoch 950/1000
393 17s 34ms/step - loss: 0.2018 - acc: 0.9551 - val_loss: 0.5295 - val_acc: 0.8711
394 Epoch 951/1000
395 17s 33ms/step - loss: 0.2009 - acc: 0.9560 - val_loss: 0.5285 - val_acc: 0.8704
396 Epoch 952/1000
397 16s 33ms/step - loss: 0.2016 - acc: 0.9559 - val_loss: 0.5291 - val_acc: 0.8695
398 Epoch 953/1000
399 16s 32ms/step - loss: 0.2033 - acc: 0.9546 - val_loss: 0.5310 - val_acc: 0.8703
400 Epoch 954/1000
401 16s 33ms/step - loss: 0.2022 - acc: 0.9540 - val_loss: 0.5318 - val_acc: 0.8704
402 Epoch 955/1000
403 17s 33ms/step - loss: 0.1997 - acc: 0.9558 - val_loss: 0.5312 - val_acc: 0.8700
404 Epoch 956/1000
405 17s 33ms/step - loss: 0.2007 - acc: 0.9555 - val_loss: 0.5297 - val_acc: 0.8701
406 Epoch 957/1000
407 17s 33ms/step - loss: 0.2035 - acc: 0.9531 - val_loss: 0.5308 - val_acc: 0.8697
408 Epoch 958/1000
409 17s 33ms/step - loss: 0.2018 - acc: 0.9547 - val_loss: 0.5324 - val_acc: 0.8699
410 Epoch 959/1000
411 17s 33ms/step - loss: 0.2006 - acc: 0.9557 - val_loss: 0.5318 - val_acc: 0.8695
412 Epoch 960/1000
413 16s 33ms/step - loss: 0.1996 - acc: 0.9558 - val_loss: 0.5311 - val_acc: 0.8690
414 Epoch 961/1000
415 16s 32ms/step - loss: 0.1991 - acc: 0.9564 - val_loss: 0.5318 - val_acc: 0.8686
416 Epoch 962/1000
417 16s 32ms/step - loss: 0.2013 - acc: 0.9544 - val_loss: 0.5323 - val_acc: 0.8681
418 Epoch 963/1000
419 17s 33ms/step - loss: 0.2013 - acc: 0.9548 - val_loss: 0.5310 - val_acc: 0.8704
420 Epoch 964/1000
421 17s 33ms/step - loss: 0.2026 - acc: 0.9549 - val_loss: 0.5317 - val_acc: 0.8702
422 Epoch 965/1000
423 17s 33ms/step - loss: 0.1990 - acc: 0.9565 - val_loss: 0.5312 - val_acc: 0.8708
424 Epoch 966/1000
425 17s 33ms/step - loss: 0.2021 - acc: 0.9535 - val_loss: 0.5303 - val_acc: 0.8706 ETA: 7s - loss: 0.2001 - acc: 0.9555
426 Epoch 967/1000
427 17s 33ms/step - loss: 0.1996 - acc: 0.9568 - val_loss: 0.5307 - val_acc: 0.8700
428 Epoch 968/1000
429 16s 33ms/step - loss: 0.2007 - acc: 0.9556 - val_loss: 0.5313 - val_acc: 0.8699
430 Epoch 969/1000
431 16s 33ms/step - loss: 0.2004 - acc: 0.9553 - val_loss: 0.5307 - val_acc: 0.8693
432 Epoch 970/1000
433 16s 32ms/step - loss: 0.1988 - acc: 0.9550 - val_loss: 0.5330 - val_acc: 0.8709
434 Epoch 971/1000
435 16s 33ms/step - loss: 0.2025 - acc: 0.9548 - val_loss: 0.5326 - val_acc: 0.8710
436 Epoch 972/1000
437 17s 33ms/step - loss: 0.2042 - acc: 0.9541 - val_loss: 0.5333 - val_acc: 0.8709
438 Epoch 973/1000
439 17s 33ms/step - loss: 0.2011 - acc: 0.9555 - val_loss: 0.5328 - val_acc: 0.8697
440 Epoch 974/1000
441 17s 33ms/step - loss: 0.2018 - acc: 0.9550 - val_loss: 0.5328 - val_acc: 0.8705
442 Epoch 975/1000
443 17s 33ms/step - loss: 0.2047 - acc: 0.9535 - val_loss: 0.5337 - val_acc: 0.8700
444 Epoch 976/1000
445 16s 33ms/step - loss: 0.1987 - acc: 0.9568 - val_loss: 0.5329 - val_acc: 0.8706
446 Epoch 977/1000
447 16s 33ms/step - loss: 0.2009 - acc: 0.9551 - val_loss: 0.5332 - val_acc: 0.8698
448 Epoch 978/1000
449 16s 33ms/step - loss: 0.1967 - acc: 0.9574 - val_loss: 0.5329 - val_acc: 0.8700
450 Epoch 979/1000
451 16s 33ms/step - loss: 0.2011 - acc: 0.9550 - val_loss: 0.5333 - val_acc: 0.8692
452 Epoch 980/1000
453 17s 33ms/step - loss: 0.2017 - acc: 0.9546 - val_loss: 0.5333 - val_acc: 0.8690
454 Epoch 981/1000
455 17s 33ms/step - loss: 0.1982 - acc: 0.9564 - val_loss: 0.5333 - val_acc: 0.8705
456 Epoch 982/1000
457 17s 34ms/step - loss: 0.1995 - acc: 0.9567 - val_loss: 0.5330 - val_acc: 0.8702
458 Epoch 983/1000
459 17s 33ms/step - loss: 0.1985 - acc: 0.9559 - val_loss: 0.5334 - val_acc: 0.8704
460 Epoch 984/1000
461 16s 33ms/step - loss: 0.2007 - acc: 0.9552 - val_loss: 0.5334 - val_acc: 0.8700
462 Epoch 985/1000
463 16s 32ms/step - loss: 0.2023 - acc: 0.9538 - val_loss: 0.5323 - val_acc: 0.8704
464 Epoch 986/1000
465 16s 32ms/step - loss: 0.1995 - acc: 0.9563 - val_loss: 0.5331 - val_acc: 0.8694
466 Epoch 987/1000
467 17s 33ms/step - loss: 0.2004 - acc: 0.9560 - val_loss: 0.5336 - val_acc: 0.8694
468 Epoch 988/1000
469 16s 33ms/step - loss: 0.1985 - acc: 0.9562 - val_loss: 0.5339 - val_acc: 0.8684
470 Epoch 989/1000
471 17s 33ms/step - loss: 0.2021 - acc: 0.9540 - val_loss: 0.5339 - val_acc: 0.8688
472 Epoch 990/1000
473 17s 33ms/step - loss: 0.1997 - acc: 0.9562 - val_loss: 0.5347 - val_acc: 0.8686
474 Epoch 991/1000
475 17s 33ms/step - loss: 0.2011 - acc: 0.9547 - val_loss: 0.5339 - val_acc: 0.8699
476 Epoch 992/1000
477 16s 32ms/step - loss: 0.1998 - acc: 0.9556 - val_loss: 0.5326 - val_acc: 0.8692
478 Epoch 993/1000
479 16s 32ms/step - loss: 0.2015 - acc: 0.9556 - val_loss: 0.5336 - val_acc: 0.8706
480 Epoch 994/1000
481 16s 33ms/step - loss: 0.2020 - acc: 0.9546 - val_loss: 0.5334 - val_acc: 0.8702
482 Epoch 995/1000
483 16s 33ms/step - loss: 0.2054 - acc: 0.9538 - val_loss: 0.5338 - val_acc: 0.8690
484 Epoch 996/1000
485 17s 33ms/step - loss: 0.1983 - acc: 0.9563 - val_loss: 0.5346 - val_acc: 0.8709
486 Epoch 997/1000
487 17s 33ms/step - loss: 0.1989 - acc: 0.9559 - val_loss: 0.5352 - val_acc: 0.8685
488 Epoch 998/1000
489 17s 33ms/step - loss: 0.2007 - acc: 0.9557 - val_loss: 0.5339 - val_acc: 0.8681
490 Epoch 999/1000
491 17s 33ms/step - loss: 0.2015 - acc: 0.9537 - val_loss: 0.5345 - val_acc: 0.8694
492 Epoch 1000/1000
493 16s 33ms/step - loss: 0.2002 - acc: 0.9548 - val_loss: 0.5354 - val_acc: 0.8686
494 Train loss: 0.16600286397337913
495 Train accuracy: 0.968400007367134
496 Test loss: 0.5354112640023232
497 Test accuracy: 0.8685999995470047