Source code for bob.learn.tensorflow.network.MLP

#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>

import tensorflow as tf
from bob.learn.tensorflow.network.utils import is_trainable

slim = tf.contrib.slim


def mlp(
    inputs,
    output_shape,
    hidden_layers=[10],
    hidden_activation=tf.nn.tanh,
    output_activation=None,
    seed=10,
    **kwargs
):
    """An MLP is a representation of a Multi-Layer Perceptron.

    This implementation is feed-forward and fully-connected.
    The implementation allows setting a global and the output activation functions.
    References to fully-connected feed-forward networks: Bishop's Pattern Recognition and Machine Learning, Chapter 5. Figure 5.1 shows what is programmed.

    MLPs normally are multi-layered systems, with 1 or more hidden layers.

    **Parameters**

        output_shape: number of neurons in the output.

        hidden_layers: :py:class:`list` that contains the amount of hidden layers, where each element is the number of neurons

        hidden_activation: Activation function of the hidden layers. Possible values can be seen
                          `here <https://www.tensorflow.org/versions/r0.11/api_docs/python/nn.html#activation-functions>`_.
                           If you set to ``None``, the activation will be linear.

        output_activation: Activation of the output layer.  If you set to `None`, the activation will be linear

        seed:
    """

    initializer = tf.contrib.layers.xavier_initializer(
        uniform=False, dtype=tf.float32, seed=seed
    )

    graph = inputs
    for i in range(len(hidden_layers)):

        weights = hidden_layers[i]
        graph = slim.fully_connected(
            graph,
            weights,
            weights_initializer=initializer,
            activation_fn=hidden_activation,
            scope="fc_{0}".format(i),
        )

    graph = slim.fully_connected(
        graph,
        output_shape,
        weights_initializer=initializer,
        activation_fn=output_activation,
        scope="fc_output",
    )

    return graph


[docs]def mlp_with_batchnorm_and_dropout( inputs, fully_connected_layers, mode=tf.estimator.ModeKeys.TRAIN, trainable_variables=None, **kwargs ): if trainable_variables is not None: raise ValueError( "The batch_norm layers selectable training is not implemented!" ) end_points = {} net = slim.flatten(inputs) weight_decay = 1e-5 dropout_keep_prob = 0.5 batch_norm_params = { # Decay for the moving averages. "decay": 0.995, # epsilon to prevent 0s in variance. "epsilon": 0.001, # force in-place updates of mean and variance estimates "updates_collections": None, "is_training": (mode == tf.estimator.ModeKeys.TRAIN), } with slim.arg_scope( [slim.fully_connected], weights_initializer=tf.truncated_normal_initializer(stddev=0.1), weights_regularizer=slim.l2_regularizer(weight_decay), normalizer_fn=slim.batch_norm, normalizer_params=batch_norm_params, ), tf.name_scope("MLP"): # hidden layers for i, n in enumerate(fully_connected_layers): name = "fc_{:0d}".format(i) trainable = is_trainable(name, trainable_variables, mode=mode) with slim.arg_scope( [slim.batch_norm], is_training=trainable, trainable=trainable ): net = slim.fully_connected(net, n, scope=name, trainable=trainable) end_points[name] = net name = "dropout_{:0d}".format(i) net = slim.dropout( net, dropout_keep_prob, is_training=(mode == tf.estimator.ModeKeys.TRAIN), scope=name, ) end_points[name] = net return net, end_points