diff --git a/.github/inference_time.png b/.github/inference_time.png index bd56ca2..5d2b04b 100644 Binary files a/.github/inference_time.png and b/.github/inference_time.png differ diff --git a/README.md b/README.md index 09fe613..d2fe833 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ The following figure shows the *raw* network semantic segmentation prediction on

-This repository contains contains the source code of our FlexConv Layer from our 2018 ACCV paper "Flex-Convolution (Million-Scale Point-Cloud Learning Beyond Grid-Worlds)". +This repository contains the source code of our FlexConv Layer from our 2018 ACCV paper "Flex-Convolution (Million-Scale Point-Cloud Learning Beyond Grid-Worlds)".

@@ -22,7 +22,7 @@ This repository contains contains the source code of our FlexConv Layer from our Example - Usage ------------------- -We provide GPU-tailored cuda implementation of our novel FlexConv in TensorFlow. +We provide GPU-tailored CUDA implementations of our novel FlexConv, FlexPool, FlexDeconv operations in TensorFlow. ```console user@host $ cd user_ops @@ -35,13 +35,13 @@ user@host $ python example.py Experiments ------------------- -Deep learning on point-clouds is a complex matter and our codebase reflect that complexity. +Deep learning on point-clouds is a complex matter and our codebase reflects that complexity. We are currently working on refactoring our research implementation to ease the usage. Therefore, `layers.py` contains a Keras/tf.layers compatible implementation. We will add the models later. ### Benchmark -We benchmarked the inference time of entire network on the 2D-3D-S dataset and with recent tests on V100, we were able to process ~18 Million Points. +We benchmarked the inference time of *entire* network on the 2D-3D-S dataset and with a recent test on a NVIDIA V100 GPU, we were able to process ~18 Million Points.

@@ -50,14 +50,14 @@ We benchmarked the inference time of entire network on the 2D-3D-S dataset and w ShapeNet part segmentation results per category and mIoU (%) for different methods and inference speed (on a NVIDIA GeForce GTX 1080 Ti). -|| Airplane | Bag | Cap | Car | Chair | Earphones | Guitar | Knife | Lamp | Laptop | Motorbike | Mug | Pistol | Rocket | Skateboard | Table | mIoU | shapes/sec | -| ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | --- | ---- | ---- | --- | -|Kd-Network [4] | 80.1 | 74.6 | 74.3 | 70.3 | 88.6 | 73.5 | 90.2 | **87.2** | 81.0 | 94.9 | 57.4 | 86.7 | 78.1 | 51.8 | 69.9 | 80.3 | 77.4 | n.a.| -|PointNet [1] | 83.4 | 78.7 | 82.5 | 74.9 | 89.6 | 73.0 | 91.5 | 85.9 | 80.8 | 95.3 | 65.2 | 93.0 | 81.2 | 57.9 | 72.8 | 80.6 | 80.4 | n.a. | -|PointNet++ [2] | 82.4 | 79.0 | 87.7 | 77.3 | **90.8** | 71.8 | 91.0 | 85.9 | 83.7 | 95.3 | 71.6 | 94.1 | 81.3 | 58.7 | 76.4 | 82.6 | 81.9 | 2.7 | -|SPLATNet3D [3] | 81.9 | 83.9 | 88.6 | **79.5** | 90.1 | 73.5 | 91.3 | 84.7 | **84.5** | 96.3 | 69.7 | 95.0 | 81.7 | 59.2 | 70.4 | 81.3 | 82.0 | 9.4 | -|SGPN [5] | 80.4 | 78.6 | 78.8 | 71.5 | 88.6 | **78.0** | 90.9 | 83.0 | 78.8 | 95.8 | **77.8** | 93.8 | **87.4** | 60.1 | **92.3** | **89.4** | 82.8 | n.a. | -Ours | **83.6** | **91.2** | **96.7** | **79.5** | 84.7 | 71.7 | **92.0** | 86.5 | 83.2 | **96.6** | 71.7 | **95.7** | 86.1 | **74.8** | 81.4 | 84.5 | **85.0** | **489.3** | +| | mIoU | shapes/sec | Airplane | Bag | Cap | Car | Chair | Earphones | Guitar | Knife | Lamp | Laptop | Motorbike | Mug | Pistol | Rocket | Skateboard | Table | +| ---- | ---- | --- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | --- | ---- | +| Kd-Network [4] | 77.4 | n.a. | 80.1 | 74.6 | 74.3 | 70.3 | 88.6 | 73.5 | 90.2 | **87.2** | 81.0 | 94.9 | 57.4 | 86.7 | 78.1 | 51.8 | 69.9 | 80.3 | +| PointNet [1] | 80.4 | n.a. | 83.4 | 78.7 | 82.5 | 74.9 | 89.6 | 73.0 | 91.5 | 85.9 | 80.8 | 95.3 | 65.2 | 93.0 | 81.2 | 57.9 | 72.8 | 80.6 | +| PointNet++ [2] | 81.9 | 2.7 | 82.4 | 79.0 | 87.7 | 77.3 | **90.8** | 71.8 | 91.0 | 85.9 | 83.7 | 95.3 | 71.6 | 94.1 | 81.3 | 58.7 | 76.4 | 82.6 | +| SPLATNet3D [3] | 82.0 | 9.4 | 81.9 | 83.9 | 88.6 | **79.5** | 90.1 | 73.5 | 91.3 | 84.7 | **84.5** | 96.3 | 69.7 | 95.0 | 81.7 | 59.2 | 70.4 | 81.3 | +| SGPN [5] | 82.8 | n.a. | 80.4 | 78.6 | 78.8 | 71.5 | 88.6 | **78.0** | 90.9 | 83.0 | 78.8 | 95.8 | **77.8** | 93.8 | **87.4** | 60.1 | **92.3** | **89.4** | +| Ours | **85.0** | **489.3** | **83.6** | **91.2** | **96.7** | **79.5** | 84.7 | 71.7 | **92.0** | 86.5 | 83.2 | **96.6** | 71.7 | **95.7** | 86.1 | **74.8** | 81.4 | 84.5 | ![example segmentation](./.github/shapenet.png) @@ -65,14 +65,14 @@ Ours | **83.6** | **91.2** | **96.7** | **79.5** | 84. Class specific average precision (AP) on the 2D-3D-S dataset. -| | Table | Chair | Sofa | Bookcase | Board | Ceiling | Floor | Wall | Beam | Col. | Wind. | Door | mAP | -|---|---|---|---|---|---|---|---|---|---|---|---|---|---| -Armeni [6] | 46.02 | 16.15 | 6.78 | 54.71 | 3.91 | 71.61 | 88.70 | 72.86 | 66.67 | **91.77** | 25.92 | 54.11 | 49.93| -Armeni [6] | 39.87 | 11.43 | 4.91 | **57.76** | 3.73 | 50.74 | 80.48 | 65.59 | 68.53 | 85.08 | 21.17 | 45.39 | 44.19| -PointNet [1] | 46.67 | 33.80 | 4.76 | n.a. | 11.72 | n.a. | n.a. | n.a. | n.a. | n.a. | n.a. | n.a. | n.a. | -SGPN [5] | 46.90 | 40.77 | 6.38 | 47.61 | 11.05 | 79.44 | 66.29 | **88.77** | **77.98** | 60.71 | **66.62** | **56.75** | 54.35 | -Ours | 66.03 | 51.75 | 15.59 | 39.03 | 43.50 | 87.20 | 96.00 | 65.53 | 54.76 | 52.74 | 55.34 | 35.81 | 55.27| -Ours** | **67.02** | **52.75** | **16.61** | 39.26 | **47.68** | **87.33** | **96.10** | 65.52 | 56.83 | 55.10 | 57.66 | 36.76 | **56.55**| +| | mAP | Table | Chair | Sofa | Bookcase | Board | Ceiling | Floor | Wall | Beam | Col. | Wind. | Door | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| Armeni [6] | 49.93 | 46.02 | 16.15 | 6.78 | 54.71 | 3.91 | 71.61 | 88.70 | 72.86 | 66.67 | **91.77** | 25.92 | 54.11 | +| Armeni [6] | 44.19 | 39.87 | 11.43 | 4.91 | **57.76** | 3.73 | 50.74 | 80.48 | 65.59 | 68.53 | 85.08 | 21.17 | 45.39 | +| PointNet [1] | n.a. | 46.67 | 33.80 | 4.76 | n.a. | 11.72 | n.a. | n.a. | n.a. | n.a. | n.a. | n.a. | n.a. | +| SGPN [5] | 54.35 | 46.90 | 40.77 | 6.38 | 47.61 | 11.05 | 79.44 | 66.29 | **88.77** | **77.98** | 60.71 | **66.62** | **56.75** | +| Ours | 55.27 | 66.03 | 51.75 | 15.59 | 39.03 | 43.50 | 87.20 | 96.00 | 65.53 | 54.76 | 52.74 | 55.34 | 35.81 | +| Ours** | **56.55** | **67.02** | **52.75** | **16.61** | 39.26 | **47.68** | **87.33** | **96.10** | 65.52 | 56.83 | 55.10 | 57.66 | 36.76 | ![example segmentation](./.github/2d3ds-results-large-1.jpg) diff --git a/example.py b/example.py index 1d07355..fd411a5 100644 --- a/example.py +++ b/example.py @@ -19,29 +19,50 @@ """ -Demonstration of using FlexConvolution Layer. +Demonstration of using FlexConvolution, FlexPooling Layer. """ import numpy as np import tensorflow as tf -from layers import flex_convolution +from tabulate import tabulate +from layers import flex_convolution, flex_convolution_transpose, flex_pooling -B, Din, Dout, Dp, N, K = 1, 2, 4, 3, 10, 5 +B, Din, Dout, Dout2, Dp, N, N2, K, K2 = 1, 2, 4, 8, 3, 10, 5, 5, 3 features = np.random.randn(B, Din, N).astype(np.float32) positions = np.random.randn(B, Dp, N).astype(np.float32) neighbors = np.random.randint(0, N, [B, K, N]).astype(np.int32) +neighbors2 = np.random.randint(0, N, [B, K2, N2]).astype(np.int32) + +features = tf.convert_to_tensor(features, name='features') +positions = tf.convert_to_tensor(positions, name='positions') +neighbors = tf.convert_to_tensor(neighbors, name='neighbors') +neighbors2 = tf.convert_to_tensor(neighbors2, name='neighbors2') + +net = [features] +# use our FlexConv similar to a traditional convolution layer +net.append(flex_convolution(net[-1], positions, neighbors, Dout)) +# pool and sub-sampling are different operations +net.append(flex_pooling(net[-1], neighbors)) + +# when ordering the points beforehand sub-sampling is simply +features = net[-1][:, :, :N2] +positions = positions[:, :, :N2] + +net.append(features) +# we didn't notice any improvements using the transposed version vs. pooling +net.append(flex_convolution_transpose(net[-1], positions, neighbors2, Dout2)) +# of course any commonly used arguments work here as well +net.append(flex_convolution(net[-1], positions, + neighbors2, Dout2, trainable=False)) -features = tf.convert_to_tensor(features) -positions = tf.convert_to_tensor(positions) -neighbors = tf.convert_to_tensor(neighbors) -features2 = flex_convolution(features, positions, neighbors, Dout) -features3 = flex_convolution(features2, positions, neighbors, Dout, trainable=False) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) - sess.run(features2) - sess.run(features3) + sess.run(net[-1]) - print(tf.trainable_variables()) + print(tabulate([[v.name, v.shape] for v in tf.trainable_variables()], + headers=["Name", "Shape"])) + + print(tabulate([[n.name, n.shape] for n in net], headers=["Name", "Shape"])) diff --git a/layers.py b/layers.py index ebbb13d..da84707 100644 --- a/layers.py +++ b/layers.py @@ -29,18 +29,88 @@ from tensorflow.python.util.tf_export import tf_export from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import ops -flex_pooling = _flex_pooling -flex_convolution_transpose = _flex_convolution_transpose + +all = ['FlexPooling', 'FlexConvolution', 'FlexConvolutionTranspose', + 'flex_pooling', 'flex_convolution', 'flex_convolution_transpose'] -def _remove_dim(x, axis): - shape = x.shape.as_list() - assert shape[axis] == 1 - del shape[axis] - return tf.reshape(x, shape) +def _remove_dim(x, axis=2): + return tf.squeeze(x, axis=axis) -@tf_export('keras.layers.Dense') +@tf_export('keras.layers.FlexPooling') +class FlexPooling(Layer): + """flex pooling layer. + + This layer performs a max-pooling operation over elements in arbitrary + neighborhoods. When `data_format` is 'simple', the input shape should + have rank 3, otherwise rank 4 and dimension 2 should be 1. + + Remarks: + In contrast to traditional pooling, this operation has no option for + sub-sampling. + + Arguments: + features: A `Tensor` of the format [B, Din, (1), N]. + neighborhoods: A `Tensor` of the format [B, K, (1), N] (tf.int32). + name: A string, the name of the layer. + + """ + + def __init__(self, + features, + neighborhoods, + data_format='simple', + name=None): + + super(FlexPooling, self).__init__(name=name) + self.features = features + self.neighborhoods = neighborhoods + self.data_format = data_format + + def compute_output_shape(self, input_shape): + return tensor_shape.TensorShape(input_shape) + + def build(self, input_shape): + self.built = True + + def call(self, inputs): + if not isinstance(inputs, list): + raise ValueError('A flexconv layer should be called ' + 'on a list of inputs.') + + features = ops.convert_to_tensor(inputs[0], dtype=self.dtype) + neighborhoods = ops.convert_to_tensor(inputs[1], dtype=tf.int32) + + if self.data_format == 'expanded': + features = _remove_dim(inputs[0], 2) + neighborhoods = _remove_dim(inputs[1], 2) + else: + features = inputs[0] + neighborhoods = inputs[1] + + y, _ = _flex_pooling(features, neighborhoods) + + if self.data_format == 'expanded': + y = tf.expand_dims(y, axis=2) + + return y + + +def flex_pooling(features, + neighborhoods, + data_format='simple', + name=None): + + layer = FlexPooling(features, + neighborhoods, + data_format=data_format, + name=name) + + return layer.apply([features, neighborhoods]) + + +@tf_export('keras.layers.FlexConvolution') class FlexConvolution(Layer): """flex convolution layer. @@ -49,6 +119,8 @@ class FlexConvolution(Layer): If `use_feature_bias` is True (and a `features_bias_initializer` is provided), a bias vector is created and added to the outputs after te convolution. Finally, if `activation` is not `None`, it is applied to the outputs as well. + When `data_format` is 'simple', the input shape should have rank 3, + otherwise rank 4 and dimension 2 should be 1. Remarks: In contrast to traditional convolutions, this operation has two @@ -120,7 +192,6 @@ class FlexConvolution(Layer): else: features = self.features positions = self.positions - [B, Din, N] = features.shape Din = int(Din) N = int(N) @@ -213,3 +284,108 @@ def flex_convolution(features, name=name) return layer.apply([features, positions, neighborhoods]) + + +@tf_export('keras.layers.FlexConvolutionTranspose') +class FlexConvolutionTranspose(FlexConvolution): + """flex convolution-transpose layer. + + This layer applies a transpose convolution to elements in arbitrary + neighborhoods. + If `use_feature_bias` is True (and a `features_bias_initializer` is provided), + a bias vector is created and added to the outputs after te convolution. + Finally, if `activation` is not `None`, it is applied to the outputs as well. + When `data_format` is 'simple', the input shape should have rank 3, + otherwise rank 4 and dimension 2 should be 1. + + Remarks: + In contrast to traditional transposed convolutions, this operation has two + bias terms: + - bias term when dynamically computing the weight [Din, Dout] + - bias term which is added tot the features [Dout] + + Arguments: + features: A `Tensor` of the format [B, Din, (1), N]. + positions: A `Tensor` of the format [B, Dp, (1), N]. + neighborhoods: A `Tensor` of the format [B, K, (1), N] (tf.int32). + filters: Integer, the dimensionality of the output space (i.e. the number + of filters in the convolution). + activation: Activation function. Set it to None to maintain a + linear activation. + kernel_initializer: An initializer for the convolution kernel. + position_bias_initializer: An initializer for the bias vector within + the convolution. If None, the default initializer will be used. + features_bias_initializer: An initializer for the bias vector after + the convolution. If None, the default initializer will be used. + use_feature_bias: Boolean, whether the layer uses a bias. + data_format: A string, one of `simple` (default) or `expaned`. + If `simple` the shapes are [B, Din, N], when `expanded` the shapes + are assumed to be [B, Din, 1, N] to match `channels_first` in trad + convolutions. + trainable: Boolean, if `True` also add variables to the graph collection + `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). + name: A string, the name of the layer. + + """ + + def call(self, inputs): + + if not isinstance(inputs, list): + raise ValueError('A flexconv layer should be called ' + 'on a list of inputs.') + + features = ops.convert_to_tensor(inputs[0], dtype=self.dtype) + positions = ops.convert_to_tensor(inputs[1], dtype=self.dtype) + neighborhoods = ops.convert_to_tensor(inputs[2], dtype=tf.int32) + + if self.data_format == 'expanded': + features = _remove_dim(inputs[0], 2) + positions = _remove_dim(inputs[1], 2) + neighborhoods = _remove_dim(inputs[2], 2) + else: + features = inputs[0] + positions = inputs[1] + neighborhoods = inputs[2] + + y = _flex_convolution_transpose(features, positions, neighborhoods, + self.position_theta, self.position_bias) + + if self.use_feature_bias: + y = tf.add(y, self.feature_bias) + + if self.activation is not None: + y = self.activation(y) + + if self.data_format == 'expanded': + y = tf.expand_dims(y, axis=2) + + return y + + +def flex_convolution_transpose(features, + positions, + neighborhoods, + filters, + activation=None, + kernel_initializer=None, + position_bias_initializer=tf.zeros_initializer(), + features_bias_initializer=tf.zeros_initializer(), + use_feature_bias=True, + data_format='simple', + trainable=True, + name=None): + + layer = FlexConvolutionTranspose(features, + positions, + neighborhoods, + filters, + activation=activation, + kernel_initializer=kernel_initializer, + position_bias_initializer=position_bias_initializer, + features_bias_initializer=features_bias_initializer, + use_feature_bias=use_feature_bias, + data_format=data_format, + trainable=trainable, + name=name) + + return layer.apply([features, positions, neighborhoods]) diff --git a/user_ops/CMakeLists.txt b/user_ops/CMakeLists.txt index 7b40ae9..33e0de0 100644 --- a/user_ops/CMakeLists.txt +++ b/user_ops/CMakeLists.txt @@ -10,7 +10,7 @@ project( FlexConv ) list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}) -find_package(CUDA 9 REQUIRED) +find_package(CUDA REQUIRED) find_package(TensorFlow REQUIRED) diff --git a/user_ops/FindTensorFlow.cmake b/user_ops/FindTensorFlow.cmake index d70a4b3..41c9b04 100644 --- a/user_ops/FindTensorFlow.cmake +++ b/user_ops/FindTensorFlow.cmake @@ -122,7 +122,7 @@ else() # However, only TensorFlow versions 1.9, 1.10 support all header files # for custom ops. set(_TensorFlow_KNOWN_VERSIONS ${TensorFlow_ADDITIONAL_VERSIONS} - "1.9" "1.9.0" "1.10" "1.10.0") + "1.9" "1.9.0" "1.10" "1.10.0", "1.11", "1.11.0") set(_TensorFlow_TEST_VERSIONS) if(TF_FIND_VERSION) @@ -270,4 +270,4 @@ SET(TensorFlow_INCLUDE_DIR ${TensorFlow_INCLUDE_DIR} CACHE PATH "path to tensorf SET(TensorFlow_VERSION ${TensorFlow_VERSION} CACHE INTERNAL "The Python executable Version") SET(TensorFlow_ABI ${TensorFlow_ABI} CACHE STRING "The Python executable Version") SET(TensorFlow_LIBRARY ${TensorFlow_LIBRARY} CACHE PATH "The Python executable Version") -SET(TensorFlow_FOUND ${TensorFlow_FOUND} CACHE BOOL "The Python executable Version") \ No newline at end of file +SET(TensorFlow_FOUND ${TensorFlow_FOUND} CACHE BOOL "The Python executable Version") diff --git a/user_ops/__init__.py b/user_ops/__init__.py index 5e1862b..78449c8 100644 --- a/user_ops/__init__.py +++ b/user_ops/__init__.py @@ -41,9 +41,8 @@ flex_pool_grad = _flex_pooling_op_so.flex_pool_grad flex_deconv = _flex_deconvolution_op_so.flex_deconv flex_deconv_grad = _flex_deconvolution_op_so.flex_deconv_grad + # pylint: disable=redefined-builtin - - def flex_convolution(features, position, neighborhood,