深度学习原理与框架- batch_normalize(归一化操作)

   x = conv_layer(x, [5, 5, 3, 64], 1)
   x = batch_normalize(x, is_training)
   x = tf.nn.relu(x)

def batch_normalize(x, is_training, decay=0.99, epsilon=0.001):
    # 定义训练过程中的归一化操作
    def bn_train():
        # 获得前三个通道的均值和标准差
        batch_mean, batch_var = tf.nn.moments(x, axes=[0, 1, 2])
        # 均值获得动量平均的均值，即pop_mean * decay + batch_mean * (1-decay)
        train_mean = tf.assign(pop_mean, pop_mean * decay + batch_mean * (1 - decay))
        # 标准差获得动量平均的标准差，即pop_var * decay + batch_var * (1 - decay)
        train_var = tf.assign(pop_var, pop_var * decay + batch_var * (1 - decay))
        # 使用tf.control_dependencies表示在执行下一步操作前，会进行train_mean和train_var操作，进行参数的更新
        with tf.control_dependencies([train_mean, train_var]):
            # 进行归一化操作
            return tf.nn.batch_normalization(x, batch_mean, batch_var, beta, scale, epsilon)
    # 定义测试阶段的归一化操作
    def bn_inference():
        # 使用训练过程中的动量平均的平均值和标准差，作为均值和标准差的输入
        return tf.nn.batch_normalization(x, pop_mean, pop_var, beta, scale, epsilon)
    # 获得最后一个通道数，即features的个数
    dim = x.get_shape().as_list()[-1]
    # 构造beta， 即偏差， 用于与归一化后的数据进行相加操作， 在训练过程中会进行更新操作
    beta = tf.get_variable(
        name=‘beta‘,
        shape=[dim],
        dtype=tf.float32,
        initializer=tf.truncated_normal_initializer(stddev=0.0),
        trainable=True)
    # 构造scale，用于与归一化后的数据进行相乘操作，在训练过程中进行更新
    scale = tf.get_variable(
        name=‘scale‘,
        shape=[dim],
        dtype=tf.float32,
        initializer=tf.truncated_normal_initializer(stddev=0.1),
        trainable=True)
    # 用于构建动量平均的平均值
    pop_mean = tf.get_variable(
        name=‘pop_mean‘,
        shape=[dim],
        dtype=tf.float32,
        initializer=tf.constant_initializer(0.0),
        trainable=False)
    # 用于构建动量平均的标准差
    pop_var = tf.get_variable(
        name=‘pop_var‘,
        shape=[dim],
        dtype=tf.float32,
        initializer=tf.constant_initializer(1.0),
        trainable=False)
    # 如果is_training 等于 True，执行bn_train操作，否者执行bn_inference操作
    return tf.cond(is_training, bn_train, bn_inference)