Flex-Convolution/layers.py

# -*- coding: utf-8 -*-

# Copyright 2017 ComputerGraphics Tuebingen. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# Authors: Fabian Groh, Patrick Wieschollek, Hendrik P.A. Lensch


import tensorflow as tf
from user_ops import flex_convolution as _flex_convolution
from user_ops import flex_pooling as _flex_pooling
from user_ops import flex_convolution_transpose as _flex_convolution_transpose

from tensorflow.python.keras import activations
from tensorflow.python.keras import initializers
from tensorflow.python.keras.engine.base_layer import Layer
from tensorflow.python.util.tf_export import tf_export
from tensorflow.python.framework import tensor_shape
from tensorflow.python.framework import ops


all = ['FlexPooling', 'FlexConvolution', 'FlexConvolutionTranspose',
       'flex_pooling', 'flex_convolution', 'flex_convolution_transpose']


def _remove_dim(x, axis=2):
  return tf.squeeze(x, axis=axis)


@tf_export('keras.layers.FlexPooling')
class FlexPooling(Layer):
  """flex pooling layer.

  This layer performs a max-pooling operation over elements in arbitrary
  neighborhoods. When `data_format` is 'simple', the input shape should
  have rank 3, otherwise rank 4 and dimension 2 should be 1.

  Remarks:
      In contrast to traditional pooling, this operation has no option for
      sub-sampling.

  Arguments:
      features: A `Tensor` of the format [B, Din, (1), N].
      neighborhoods: A `Tensor` of the format [B, K, (1), N] (tf.int32).
      name: A string, the name of the layer.

  """

  def __init__(self,
               features,
               neighborhoods,
               data_format='simple',
               name=None):

    super(FlexPooling, self).__init__(name=name)
    self.features = features
    self.neighborhoods = neighborhoods
    self.data_format = data_format

  def compute_output_shape(self, input_shape):
    return tensor_shape.TensorShape(input_shape)

  def build(self, input_shape):
    self.built = True

  def call(self, inputs):
    if not isinstance(inputs, list):
      raise ValueError('A flexconv layer should be called '
                       'on a list of inputs.')

    features = ops.convert_to_tensor(inputs[0], dtype=self.dtype)
    neighborhoods = ops.convert_to_tensor(inputs[1], dtype=tf.int32)

    if self.data_format == 'expanded':
      features = _remove_dim(features, 2)
      neighborhoods = _remove_dim(neighborhoods, 2)

    y, _ = _flex_pooling(features, neighborhoods)

    if self.data_format == 'expanded':
      y = tf.expand_dims(y, axis=2)

    return y


def flex_pooling(features,
                 neighborhoods,
                 data_format='simple',
                 name=None):

  layer = FlexPooling(features,
                      neighborhoods,
                      data_format=data_format,
                      name=name)

  return layer.apply([features, neighborhoods])


@tf_export('keras.layers.FlexConvolution')
class FlexConvolution(Layer):
  """flex convolution layer.

  This layer convolves elements in arbitrary neighborhoods with a kernel to
  produce a tensor of outputs.
  If `use_feature_bias` is True (and a `features_bias_initializer` is provided),
  a bias vector is created and added to the outputs after te convolution.
  Finally, if `activation` is not `None`, it is applied to the outputs as well.
  When `data_format` is 'simple', the input shape should have rank 3,
  otherwise rank 4 and dimension 2 should be 1.

  Remarks:
      In contrast to traditional convolutions, this operation has two
      bias terms:
        - bias term when dynamically computing the weight [Din, Dout]
        - bias term which is added tot the features [Dout]

  Arguments:
      features: A `Tensor` of the format [B, Din, (1), N].
      positions: A `Tensor` of the format [B, Dp, (1), N].
      neighborhoods: A `Tensor` of the format [B, K, (1), N] (tf.int32).
      filters: Integer, the dimensionality of the output space (i.e. the number
        of filters in the convolution).
      activation: Activation function. Set it to None to maintain a
        linear activation.
      kernel_initializer: An initializer for the convolution kernel.
      position_bias_initializer: An initializer for the bias vector within
        the convolution. If None, the default initializer will be used.
      features_bias_initializer: An initializer for the bias vector after
        the convolution. If None, the default initializer will be used.
      use_feature_bias: Boolean, whether the layer uses a bias.
      data_format: A string, one of `simple` (default) or `expaned`.
        If `simple` the shapes are [B, Din, N], when `expanded` the shapes
        are assumed to be [B, Din, 1, N] to match `channels_first` in trad
        convolutions.
      trainable: Boolean, if `True` also add variables to the graph collection
        `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).
      name: A string, the name of the layer.

  """

  def __init__(self,
               features,
               positions,
               neighborhoods,
               filters,
               activation=None,
               kernel_initializer=None,
               position_bias_initializer=tf.zeros_initializer(),
               features_bias_initializer=tf.zeros_initializer(),
               use_feature_bias=True,
               data_format='simple',
               trainable=True,
               name=None):

    super(FlexConvolution, self).__init__(trainable=trainable,
                                          name=name)
    self.features = features
    self.positions = positions
    self.neighborhoods = neighborhoods

    self.filters = int(filters)
    self.activation = activations.get(activation)
    self.use_feature_bias = use_feature_bias
    self.data_format = data_format
    self.kernel_initializer = initializers.get(kernel_initializer)
    self.position_bias_initializer = initializers.get(position_bias_initializer)
    self.features_bias_initializer = initializers.get(features_bias_initializer)

  def compute_output_shape(self, input_shape):
    input_shape = tensor_shape.TensorShape(input_shape)
    input_shape[1] = self.filters
    return input_shape

  def build(self, input_shape):
    if self.data_format == 'expanded':
      features = _remove_dim(self.features, 2)
      positions = _remove_dim(self.positions, 2)
    else:
      features = self.features
      positions = self.positions
    [B, Din, N] = features.shape
    Din = int(Din)
    N = int(N)
    Dp = int(positions.shape[1])
    Dout = self.filters

    self.position_theta = self.add_weight(
        'position_theta',
        shape=[1, Dp, Din, Dout],
        initializer=self.kernel_initializer,
        dtype=self.dtype,
        trainable=True)

    self.position_bias = self.add_weight(
        'position_bias',
        shape=[Din, Dout],
        initializer=self.position_bias_initializer,
        dtype=self.dtype,
        trainable=True)

    if self.use_feature_bias:
      self.feature_bias = self.add_weight(
          'feature_bias',
          shape=[Dout, 1],
          initializer=self.features_bias_initializer,
          dtype=self.dtype,
          trainable=True)
    else:
      self.feature_bias = None
    self.built = True

  def call(self, inputs):

    if not isinstance(inputs, list):
      raise ValueError('A flexconv layer should be called '
                       'on a list of inputs.')

    features = ops.convert_to_tensor(inputs[0], dtype=self.dtype)
    positions = ops.convert_to_tensor(inputs[1], dtype=self.dtype)
    neighborhoods = ops.convert_to_tensor(inputs[2], dtype=tf.int32)

    if self.data_format == 'expanded':
      features = _remove_dim(features, 2)
      positions = _remove_dim(positions, 2)
      neighborhoods = _remove_dim(neighborhoods, 2)

    y = _flex_convolution(features, positions, neighborhoods,
                          self.position_theta, self.position_bias)

    if self.use_feature_bias:
      y = tf.add(y, self.feature_bias)

    if self.activation is not None:
      y = self.activation(y)

    if self.data_format == 'expanded':
      y = tf.expand_dims(y, axis=2)

    return y


def flex_convolution(features,
                     positions,
                     neighborhoods,
                     filters,
                     activation=None,
                     kernel_initializer=None,
                     position_bias_initializer=tf.zeros_initializer(),
                     features_bias_initializer=tf.zeros_initializer(),
                     use_feature_bias=True,
                     data_format='simple',
                     trainable=True,
                     name=None):

  layer = FlexConvolution(features,
                          positions,
                          neighborhoods,
                          filters,
                          activation=activation,
                          kernel_initializer=kernel_initializer,
                          position_bias_initializer=position_bias_initializer,
                          features_bias_initializer=features_bias_initializer,
                          use_feature_bias=use_feature_bias,
                          data_format=data_format,
                          trainable=trainable,
                          name=name)

  return layer.apply([features, positions, neighborhoods])


@tf_export('keras.layers.FlexConvolutionTranspose')
class FlexConvolutionTranspose(FlexConvolution):
  """flex convolution-transpose layer.

  This layer applies a transpose convolution to elements in arbitrary
  neighborhoods.
  If `use_feature_bias` is True (and a `features_bias_initializer` is provided),
  a bias vector is created and added to the outputs after te convolution.
  Finally, if `activation` is not `None`, it is applied to the outputs as well.
  When `data_format` is 'simple', the input shape should have rank 3,
  otherwise rank 4 and dimension 2 should be 1.

  Remarks:
      In contrast to traditional transposed convolutions, this operation has two
      bias terms:
        - bias term when dynamically computing the weight [Din, Dout]
        - bias term which is added tot the features [Dout]

  Arguments:
      features: A `Tensor` of the format [B, Din, (1), N].
      positions: A `Tensor` of the format [B, Dp, (1), N].
      neighborhoods: A `Tensor` of the format [B, K, (1), N] (tf.int32).
      filters: Integer, the dimensionality of the output space (i.e. the number
        of filters in the convolution).
      activation: Activation function. Set it to None to maintain a
        linear activation.
      kernel_initializer: An initializer for the convolution kernel.
      position_bias_initializer: An initializer for the bias vector within
        the convolution. If None, the default initializer will be used.
      features_bias_initializer: An initializer for the bias vector after
        the convolution. If None, the default initializer will be used.
      use_feature_bias: Boolean, whether the layer uses a bias.
      data_format: A string, one of `simple` (default) or `expaned`.
        If `simple` the shapes are [B, Din, N], when `expanded` the shapes
        are assumed to be [B, Din, 1, N] to match `channels_first` in trad
        convolutions.
      trainable: Boolean, if `True` also add variables to the graph collection
        `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).
      name: A string, the name of the layer.

  """

  def call(self, inputs):

    if not isinstance(inputs, list):
      raise ValueError('A flexconv layer should be called '
                       'on a list of inputs.')

    features = ops.convert_to_tensor(inputs[0], dtype=self.dtype)
    positions = ops.convert_to_tensor(inputs[1], dtype=self.dtype)
    neighborhoods = ops.convert_to_tensor(inputs[2], dtype=tf.int32)

    if self.data_format == 'expanded':
      features = _remove_dim(features, 2)
      positions = _remove_dim(positions, 2)
      neighborhoods = _remove_dim(neighborhoods, 2)

    y = _flex_convolution_transpose(features, positions, neighborhoods,
                                    self.position_theta, self.position_bias)

    if self.use_feature_bias:
      y = tf.add(y, self.feature_bias)

    if self.activation is not None:
      y = self.activation(y)

    if self.data_format == 'expanded':
      y = tf.expand_dims(y, axis=2)

    return y


def flex_convolution_transpose(features,
                               positions,
                               neighborhoods,
                               filters,
                               activation=None,
                               kernel_initializer=None,
                               position_bias_initializer=tf.zeros_initializer(),
                               features_bias_initializer=tf.zeros_initializer(),
                               use_feature_bias=True,
                               data_format='simple',
                               trainable=True,
                               name=None):

  layer = FlexConvolutionTranspose(features,
                                   positions,
                                   neighborhoods,
                                   filters,
                                   activation=activation,
                                   kernel_initializer=kernel_initializer,
                                   position_bias_initializer=position_bias_initializer,
                                   features_bias_initializer=features_bias_initializer,
                                   use_feature_bias=use_feature_bias,
                                   data_format=data_format,
                                   trainable=trainable,
                                   name=name)

  return layer.apply([features, positions, neighborhoods])