Refactoring to more easily reference TF code. Add MNIST test.

mnist/context.py

@@ -0,0 +1,4 @@
+import sys, os
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+from lib import pointcnn

mnist/data_utils.py

@@ -0,0 +1,145 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import h5py
+import plyfile
+import numpy as np
+from matplotlib import cm
+import scipy.spatial.distance as distance
+
+
+def save_ply(points, filename, colors=None, normals=None):
+    vertex = np.array([tuple(p) for p in points], dtype=[('x', 'f4'), ('y', 'f4'), ('z', 'f4')])
+    n = len(vertex)
+    desc = vertex.dtype.descr
+
+    if normals is not None:
+        vertex_normal = np.array([tuple(n) for n in normals], dtype=[('nx', 'f4'), ('ny', 'f4'), ('nz', 'f4')])
+        assert len(vertex_normal) == n
+        desc = desc + vertex_normal.dtype.descr
+
+    if colors is not None:
+        vertex_color = np.array([tuple(c * 255) for c in colors],
+                                dtype=[('red', 'u1'), ('green', 'u1'), ('blue', 'u1')])
+        assert len(vertex_color) == n
+        desc = desc + vertex_color.dtype.descr
+
+    vertex_all = np.empty(n, dtype=desc)
+
+    for prop in vertex.dtype.names:
+        vertex_all[prop] = vertex[prop]
+
+    if normals is not None:
+        for prop in vertex_normal.dtype.names:
+            vertex_all[prop] = vertex_normal[prop]
+
+    if colors is not None:
+        for prop in vertex_color.dtype.names:
+            vertex_all[prop] = vertex_color[prop]
+
+    ply = plyfile.PlyData([plyfile.PlyElement.describe(vertex_all, 'vertex')], text=False)
+    if not os.path.exists(os.path.dirname(filename)):
+        os.makedirs(os.path.dirname(filename))
+    ply.write(filename)
+
+
+def save_ply_property(points, property, property_max, filename, cmap_name='Set1'):
+    point_num = points.shape[0]
+    colors = np.full(points.shape, 0.5)
+    cmap = cm.get_cmap(cmap_name)
+    for point_idx in range(point_num):
+        colors[point_idx] = cmap(property[point_idx] / property_max)[:3]
+    save_ply(points, filename, colors)
+
+
+def save_ply_batch(points_batch, file_path, points_num=None):
+    batch_size = points_batch.shape[0]
+    if type(file_path) != list:
+        basename = os.path.splitext(file_path)[0]
+        ext = '.ply'
+    for batch_idx in range(batch_size):
+        point_num = points_batch.shape[1] if points_num is None else points_num[batch_idx]
+        if type(file_path) == list:
+            save_ply(points_batch[batch_idx][:point_num], file_path[batch_idx])
+        else:
+            save_ply(points_batch[batch_idx][:point_num], '%s_%04d%s' % (basename, batch_idx, ext))
+
+
+def save_ply_property_batch(points_batch, property_batch, file_path, points_num=None, property_max=None,
+                            cmap_name='Set1'):
+    batch_size = points_batch.shape[0]
+    if type(file_path) != list:
+        basename = os.path.splitext(file_path)[0]
+        ext = '.ply'
+    property_max = np.max(property_batch) if property_max is None else property_max
+    for batch_idx in range(batch_size):
+        point_num = points_batch.shape[1] if points_num is None else points_num[batch_idx]
+        if type(file_path) == list:
+            save_ply_property(points_batch[batch_idx][:point_num], property_batch[batch_idx][:point_num],
+                              property_max, file_path[batch_idx], cmap_name)
+        else:
+            save_ply_property(points_batch[batch_idx][:point_num], property_batch[batch_idx][:point_num],
+                              property_max, '%s_%04d%s' % (basename, batch_idx, ext), cmap_name)
+
+
+def save_ply_point_with_normal(data_sample, folder):
+    for idx, sample in enumerate(data_sample):
+        filename_pts = os.path.join(folder, '{:08d}.ply'.format(idx))
+        save_ply(sample[..., :3], filename_pts, normals=sample[..., 3:])
+
+
+def grouped_shuffle(inputs):
+    for idx in range(len(inputs) - 1):
+        assert (len(inputs[idx]) == len(inputs[idx + 1]))
+
+    shuffle_indices = np.arange(inputs[0].shape[0])
+    np.random.shuffle(shuffle_indices)
+    outputs = []
+    for idx in range(len(inputs)):
+        outputs.append(inputs[idx][shuffle_indices, ...])
+    return outputs
+
+
+def load_cls(filelist):
+    points = []
+    labels = []
+
+    folder = os.path.dirname(filelist)
+    for line in open(filelist):
+        filename = os.path.basename(line.rstrip())
+        data = h5py.File(os.path.join(folder, filename))
+        if 'normal' in data:
+            points.append(np.concatenate([data['data'][...], data['data'][...]], axis=-1).astype(np.float32))
+        else:
+            points.append(data['data'][...].astype(np.float32))
+        labels.append(np.squeeze(data['label'][:]).astype(np.int32))
+    return (np.concatenate(points, axis=0),
+            np.concatenate(labels, axis=0))
+
+
+def load_cls_train_val(filelist, filelist_val):
+    data_train, label_train = grouped_shuffle(load_cls(filelist))
+    data_val, label_val = load_cls(filelist_val)
+    return data_train, label_train, data_val, label_val
+
+
+def load_seg(filelist):
+    points = []
+    labels = []
+    point_nums = []
+    labels_seg = []
+
+    folder = os.path.dirname(filelist)
+    for line in open(filelist):
+        filename = os.path.basename(line.rstrip())
+        data = h5py.File(os.path.join(folder, filename))
+        points.append(data['data'][...].astype(np.float32))
+        labels.append(data['label'][...].astype(np.int32))
+        point_nums.append(data['data_num'][...].astype(np.int32))
+        labels_seg.append(data['label_seg'][...].astype(np.int32))
+    return (np.concatenate(points, axis=0),
+            np.concatenate(labels, axis=0),
+            np.concatenate(point_nums, axis=0),
+            np.concatenate(labels_seg, axis=0))

mnist/download_datasets.py

@@ -0,0 +1,140 @@
+#!/usr/bin/python3
+'''Download datasets for this project.'''
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import sys
+import gzip
+import html
+import shutil
+import tarfile
+import zipfile
+import requests
+import argparse
+from tqdm import tqdm
+
+
+# from https://gist.github.com/hrouault/1358474
+def query_yes_no(question, default="yes"):
+    """Ask a yes/no question via raw_input() and return their answer.
+    "question" is a string that is presented to the user.
+    "default" is the presumed answer if the user just hits <Enter>.
+        It must be "yes" (the default), "no" or None (meaning
+        an answer is required of the user).
+    The "answer" return value is one of "yes" or "no".
+    """
+    valid = {"yes": True, "y": True, "ye": True,
+             "no": False, "n": False}
+    if default == None:
+        prompt = " [y/n] "
+    elif default == "yes":
+        prompt = " [Y/n] "
+    elif default == "no":
+        prompt = " [y/N] "
+    else:
+        raise ValueError("invalid default answer: '%s'" % default)
+
+    while True:
+        sys.stdout.write(question + prompt)
+        choice = input().lower()
+        if default is not None and choice == '':
+            return valid[default]
+        elif choice in valid:
+            return valid[choice]
+        else:
+            sys.stdout.write("Please respond with 'yes' or 'no' (or 'y' or 'n').\n")
+
+
+def download_from_url(url, dst):
+    download = True
+    if os.path.exists(dst):
+        download = query_yes_no('Seems you have downloaded %s to %s, overwrite?' % (url, dst), default='no')
+        if download:
+            os.remove(dst)
+
+    if download:
+        response = requests.get(url, stream=True)
+        total_size = int(response.headers.get('content-length', 0))
+        chunk_size = 1024 * 1024
+        bars = total_size // chunk_size
+        with open(dst, "wb") as handle:
+            for data in tqdm(response.iter_content(chunk_size=chunk_size), total=bars, desc=url.split('/')[-1],
+                             unit='M'):
+                handle.write(data)
+
+
+def download_and_unzip(url, root, dataset):
+    folder = os.path.join(root, dataset)
+    folder_zips = os.path.join(folder, 'zips')
+    if not os.path.exists(folder_zips):
+        os.makedirs(folder_zips)
+    filename_zip = os.path.join(folder_zips, url.split('/')[-1])
+
+    download_from_url(url, filename_zip)
+
+    if filename_zip.endswith('.zip'):
+        zip_ref = zipfile.ZipFile(filename_zip, 'r')
+        zip_ref.extractall(folder)
+        zip_ref.close()
+    elif filename_zip.endswith(('.tar.gz', '.tgz')):
+        tarfile.open(name=filename_zip, mode="r:gz").extractall(folder)
+    elif filename_zip.endswith('.gz'):
+        filename_no_gz = filename_zip[:-3]
+        with gzip.open(filename_zip, 'rb') as f_in, open(filename_no_gz, 'wb') as f_out:
+            shutil.copyfileobj(f_in, f_out)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--folder', '-f', help='Path to data folder.')
+    parser.add_argument('--dataset', '-d', help='Dataset to download.')
+    args = parser.parse_args()
+    print(args)
+
+    root = args.folder if args.folder else '../../data'
+    if args.dataset == 'tu_berlin':
+        download_and_unzip('http://cybertron.cg.tu-berlin.de/eitz/projects/classifysketch/sketches_svg.zip', root,
+                           args.dataset)
+    elif args.dataset == 'modelnet':
+        download_and_unzip('https://shapenet.cs.stanford.edu/media/modelnet40_ply_hdf5_2048.zip', root, args.dataset)
+        folder = os.path.join(root, args.dataset)
+        folder_h5 = os.path.join(folder, 'modelnet40_ply_hdf5_2048')
+        for filename in os.listdir(folder_h5):
+            shutil.move(os.path.join(folder_h5, filename), os.path.join(folder, filename))
+        shutil.rmtree(folder_h5)
+    elif args.dataset == 'shapenet_partseg':
+        download_and_unzip('https://shapenet.cs.stanford.edu/iccv17/partseg/train_data.zip', root, args.dataset)
+        download_and_unzip('https://shapenet.cs.stanford.edu/iccv17/partseg/train_label.zip', root, args.dataset)
+        download_and_unzip('https://shapenet.cs.stanford.edu/iccv17/partseg/val_data.zip', root, args.dataset)
+        download_and_unzip('https://shapenet.cs.stanford.edu/iccv17/partseg/val_label.zip', root, args.dataset)
+        download_and_unzip('https://shapenet.cs.stanford.edu/iccv17/partseg/test_data.zip', root, args.dataset)
+        download_and_unzip('https://shapenet.cs.stanford.edu/iccv17/partseg/test_label.zip', root, args.dataset)
+    elif args.dataset == 'mnist':
+        download_and_unzip('http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz', root, args.dataset)
+        download_and_unzip('http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz', root, args.dataset)
+        download_and_unzip('http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz', root, args.dataset)
+        download_and_unzip('http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz', root, args.dataset)
+    elif args.dataset == 'cifar10':
+        download_and_unzip('https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz', root, args.dataset)
+    elif args.dataset == 'quick_draw':
+        url_categories = 'https://raw.githubusercontent.com/googlecreativelab/quickdraw-dataset/master/categories.txt'
+        folder = os.path.join(root, args.dataset)
+        folder_zips = os.path.join(folder, 'zips')
+        if not os.path.exists(folder_zips):
+            os.makedirs(folder_zips)
+        filename_categories = os.path.join(folder_zips, url_categories.split('/')[-1])
+        download_from_url(url_categories, filename_categories)
+
+        categories = [line.strip() for line in open(filename_categories, 'r')]
+        url_base = 'https://storage.googleapis.com/quickdraw_dataset/sketchrnn/'
+        for category in categories:
+            url = url_base + html.escape(category) + '.npz'
+            filename_category = os.path.join(folder_zips, category + '.npz')
+            download_from_url(url, filename_category)
+
+
+if __name__ == '__main__':
+    main()

mnist/model.py

@@ -0,0 +1,120 @@
+import math
+import data_utils
+import time
+
+import torch
+from torch import nn
+from torch.autograd import Variable
+from torch.utils.data import Dataset, DataLoader
+
+from pointcnn.core import rPointCNN
+from pointcnn.util import knn_indices_func
+from pointcnn.layers import MLP
+
+x = 2
+
+# N_neighbors, dilution, N_rep, C_out
+# 8          , 1,        all  , 16 * x
+# 8          , 2,        all  , 32 * x
+# 8          , 4,        all  , 48 * x
+# 12         , 4,        120  , 64 * x
+# 12         , 6,        120  , 80 * x
+
+# Data_dim = 3
+
+class mnist_dataset(Dataset):
+
+    def __init__(self, data, labels):
+        self.data = data
+        self.labels = labels
+
+    def __len__(self):
+        return len(self.data)
+    
+    def __getitem__(self, i):
+        return self.data[i], self.labels[i]
+
+# C_in, C_out, D, N_neighbors, dilution, N_rep, r_indices_func, C_lifted = None, mlp_width = 2
+# (a, b, c, d, e) == (C_in, C_out, N_neighbors, dilution, N_rep)
+paPointCNN = lambda a,b,c,d,e: rPointCNN(a, b, 3, c, d, e, knn_indices_func)
+
+class Classifier(nn.Module):
+    
+    def __init__(self):
+        super(Classifier, self).__init__()
+
+        self.pcnn = nn.Sequential(
+            paPointCNN(  1,  32,  8, 1, 256),
+            paPointCNN( 32,  64,  8, 2, 256),
+            paPointCNN( 64,  96,  8, 4, 256),
+            paPointCNN( 96, 128, 12, 4, 120),
+            paPointCNN(128, 160, 12, 6, 120),
+        )
+
+        self.fcn = nn.Sequential(
+            nn.Linear(160, 128),
+            nn.ReLU(),
+            nn.Dropout(0.0),
+            nn.Linear(128,  64),  # throw in some batch normalization
+            nn.ReLU(),
+            nn.Dropout(0.5),
+            nn.Linear(64, 10),  # 10 digits
+        )
+
+        self.log_softmax = nn.LogSoftmax()
+
+    def forward(self, x):
+        x = self.pcnn(x)[1]  # grab features
+        logits = self.fcn(x)
+        logits = torch.mean(logits, dim = 1)
+        log_probs = self.log_softmax(logits)
+        return log_probs
+
+model = Classifier().cuda()
+
+num_class = 10
+sample_num = 160
+batch_size = 32
+num_epochs = 2048
+jitter = 0.01
+jitter_val = 0.01
+
+rotation_range = [0, math.pi / 18, 0, 'g']
+rotation_rage_val = [0, 0, 0, 'u']
+order = 'rxyz'
+
+scaling_range = [0.05, 0.05, 0.05, 'g']
+scaling_range_val = [0, 0, 0, 'u']
+
+data_train, label_train, data_val, label_val = data_utils.load_cls_train_val("./mnist/zips/train_files.txt", "./mnist/zips/test_files.txt")
+
+num_train = data_train.shape[0]
+point_num = data_train.shape[1]
+
+batch_num_per_epoch = int(math.ceil(num_train / batch_size))
+batch_num = batch_num_per_epoch * num_epochs
+
+dataset = mnist_dataset(data_train, label_train)
+loader = DataLoader(dataset, batch_size = batch_size)
+
+optimizer = torch.optim.SGD(model.parameters(), lr = 0.1, momentum = 0.9)
+loss_fn = nn.NLLLoss()
+
+for _ in range(num_epochs):
+    for data, label in loader:
+
+        data = Variable(data).cuda()
+        label = Variable(label.long()).cuda()
+        P = data[:,:,:3]
+        F = data[:,:,3:]
+
+        optimizer.zero_grad()
+
+        t0 = time.time()
+        out = model((P, F))
+
+        loss = loss_fn(out, label)
+        loss.backward()
+        optimizer.step()
+        
+        # print(loss.data[0])

mnist/pointcnn

@@ -0,0 +1 @@
+/home/austin/ISEE/PointCNN/pointcnn

mnist/prepare_mnist_data.py

@@ -0,0 +1,92 @@
+#!/usr/bin/python3
+'''Convert MNIST to points.'''
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import sys
+import h5py
+import random
+import argparse
+import numpy as np
+from mnist import MNIST
+from datetime import datetime
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import data_utils
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--folder', '-f', help='Path to data folder')
+    parser.add_argument('--point_num', '-p', help='Point number for each sample', type=int, default=256)
+    parser.add_argument('--save_ply', '-s', help='Convert .pts to .ply', action='store_true')
+    args = parser.parse_args()
+    print(args)
+
+    batch_size = 2048
+
+    folder_mnist = args.folder if args.folder else '../../data/mnist/zips'
+    folder_pts = os.path.join(os.path.dirname(folder_mnist), 'pts')
+
+    mnist_data = MNIST(folder_mnist)
+    mnist_train_test = [(mnist_data.load_training(), 'train'), (mnist_data.load_testing(), 'test')]
+
+    data = np.zeros((batch_size, args.point_num, 4))
+    label = np.zeros((batch_size), dtype=np.int32)
+    for ((images, labels), tag) in mnist_train_test:
+        idx_h5 = 0
+        filename_filelist_h5 = os.path.join(os.path.dirname(folder_mnist), '%s_files.txt' % tag)
+        point_num_total = 0
+        with open(filename_filelist_h5, 'w') as filelist_h5:
+            for idx_img, image in enumerate(images):
+                points = []
+                pixels = []
+                for idx_pixel, pixel in enumerate(image):
+                    if pixel == 0:
+                        continue
+                    x = idx_pixel // 28
+                    z = idx_pixel % 28
+                    points.append((x, random.random() * 1e-6, z))
+                    pixels.append(pixel)
+                point_num_total = point_num_total + len(points)
+                pixels_sum = sum(pixels)
+                probs = [pixel / pixels_sum for pixel in pixels]
+                indices = np.random.choice(list(range(len(points))), size=args.point_num,
+                                           replace=(len(points) < args.point_num), p=probs)
+                points_array = np.array(points)[indices]
+                pixels_array_1d = (np.array(pixels)[indices].astype(np.float32) / 255) - 0.5
+                pixels_array = np.expand_dims(pixels_array_1d, axis=-1)
+
+                points_min = np.amin(points_array, axis=0)
+                points_max = np.amax(points_array, axis=0)
+                points_center = (points_min + points_max) / 2
+                scale = np.amax(points_max - points_min) / 2
+                points_array = (points_array - points_center) * (0.8 / scale)
+
+                if args.save_ply:
+                    filename_pts = os.path.join(folder_pts, tag, '{:06d}.ply'.format(idx_img))
+                    data_utils.save_ply(points_array, filename_pts, colors=np.tile(pixels_array, (1, 3)) + 0.5)
+
+                idx_in_batch = idx_img % batch_size
+                data[idx_in_batch, ...] = np.concatenate((points_array, pixels_array), axis=-1)
+                label[idx_in_batch] = labels[idx_img]
+                if ((idx_img + 1) % batch_size == 0) or idx_img == len(images) - 1:
+                    item_num = idx_in_batch + 1
+                    filename_h5 = os.path.join(os.path.dirname(folder_mnist), '%s_%d.h5' % (tag, idx_h5))
+                    print('{}-Saving {}...'.format(datetime.now(), filename_h5))
+                    filelist_h5.write('./%s_%d.h5\n' % (tag, idx_h5))
+
+                    file = h5py.File(filename_h5, 'w')
+                    file.create_dataset('data', data=data[0:item_num, ...])
+                    file.create_dataset('label', data=label[0:item_num, ...])
+                    file.close()
+
+                    idx_h5 = idx_h5 + 1
+        print('Average point number in each sample is : %f!' % (point_num_total / len(images)))
+
+
+if __name__ == '__main__':
+    main()

mnist/train.py

@@ -0,0 +1,9 @@
+import h5py
+
+f = h5py.File("./mnist/zips/train_0.h5", 'r')
+
+data = f["data"]
+label = f["label"]
+
+for r in data[0]:
+    print(r)

pointcnn/context.py

@@ -1,5 +1,5 @@
 import sys, os
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
 
-from lib import timed
-from lib import pytorch_knn_cuda
+# from lib import timed
+# from lib import pytorch_knn_cuda

pointcnn/core.py

@@ -8,19 +8,20 @@ import matplotlib.pyplot as plt
 
 try:
     from .util import knn_indices_func
-    from .layers import MLP, LayerNorm, DepthwiseSeparableConv2d, endchannels
-    from .context import timed
+    from .layers import MLP, LayerNorm, Conv, SepConv, endchannels
+    # from .context import timed
 except SystemError:
     from util import knn_indices_func
-    from layers import MLP, LayerNorm, DepthwiseSeparableConv2d, endchannels
-    from context import timed
+    from layers import MLP, LayerNorm, Conv, SepConv, endchannels
+    # from context import timed
 
 class XConv(nn.Module):
     """
     Vectorized pointwise convolution.
     """
 
-    def __init__(self, C_in, C_out, D, N_neighbors, N_rep, C_lifted = None, mlp_width = 2):
+    def __init__(self, C_in, C_out, D, N_neighbors, N_rep, C_lifted = None,
+                 mlp_width = 2):
         """
         :param C_in: Input dimension of the points' features.
         :param C_out: Output dimension of the representative point features.
@@ -48,47 +49,30 @@ class XConv(nn.Module):
 
         # Main dense linear layers
         self.mlp_lift = MLP([D] + [self.C_lifted] * (mlp_width - 1), batch_norm = False)
-        # self.mid_conv = endchannels(nn.Conv2d(D, N_neighbors, 1).cuda())
 
         # Layers to generate X
-        self.mid_conv = endchannels(nn.Sequential(
-            nn.Conv2d(D, N_neighbors**2, (1, N_neighbors)).cuda(),
-            nn.ReLU(),
-            nn.BatchNorm2d(N_neighbors**2).cuda(),
-        ))
-        self.mid_dwconv1 = endchannels(nn.Sequential(
-            DepthwiseSeparableConv2d(
-                in_channels = N_neighbors,
-                out_channels = N_neighbors**2,
-                kernel_size = (1, N_neighbors),
-                depth_multiplier = N_neighbors
-            ).cuda(),
-            nn.ReLU(),
-            nn.BatchNorm2d(N_neighbors ** 2).cuda()
-        ))
-        self.mid_dwconv2 = endchannels(nn.Sequential(
-            DepthwiseSeparableConv2d(
-                in_channels = N_neighbors,
-                out_channels = N_neighbors**2,
-                kernel_size = (1, N_neighbors),
-                depth_multiplier = N_neighbors
-            ).cuda(),
-            nn.ReLU(),
-            nn.BatchNorm2d(N_neighbors**2).cuda()
-        ))
+        self.mid_conv = endchannels(Conv(D, N_neighbors**2, (1, N_neighbors))).cuda()
+        self.mid_dwconv1 = endchannels(SepConv(
+            in_channels = N_neighbors,
+            out_channels = N_neighbors**2,
+            kernel_size = (1, N_neighbors),
+            depth_multiplier = N_neighbors
+        )).cuda()
+        self.mid_dwconv2 = endchannels(SepConv(
+            in_channels = N_neighbors,
+            out_channels = N_neighbors**2,
+            kernel_size = (1, N_neighbors),
+            depth_multiplier = N_neighbors
+        )).cuda()
 
         # Final 
         self.mlp = MLP([N_neighbors] * mlp_width, batch_norm = False)
-        self.end_conv = endchannels(nn.Sequential(
-            DepthwiseSeparableConv2d(
-                in_channels = C_lifted + C_in,
-                out_channels = C_out,
-                kernel_size = (1, N_neighbors),
-                depth_multiplier = 4
-            ).cuda(),
-            nn.ReLU(),
-            nn.BatchNorm2d(C_out).cuda()
-        ))
+        self.end_conv = endchannels(SepConv(
+            in_channels = C_lifted + C_in,
+            out_channels = C_out,
+            kernel_size = (1, N_neighbors),
+            depth_multiplier = 4
+        )).cuda()
 
     # @timed.timed
     def forward(self, x):
@@ -136,6 +120,7 @@ class XConv(nn.Module):
         # Weight and permute F_cat with the learned X.
         F_X = torch.matmul(X, F_cat)
         F_p = self.end_conv(F_X).squeeze(dim = 2)
+        time.sleep(5)
         return F_p
 
 class PointCNN(nn.Module):
@@ -143,18 +128,22 @@ class PointCNN(nn.Module):
     TODO: Insert documentation
     """
 
-    def __init__(self, C_in, C_out, D, N_neighbors, N_rep, r_indices_func, C_lifted = None, mlp_width = 4):
+    def __init__(self, C_in, C_out, D, N_neighbors, dilution, N_rep,
+                 r_indices_func, C_lifted = None, mlp_width = 2):
         """
         :param C_in: Input dimension of the points' features.
         :param C_out: Output dimension of the representative point features.
         :param D: Spatial dimensionality of points.
         :param N_neighbors: Number of neighbors to convolve over.
+        :param N_rep: Number of representative points.
+        :param dilution: "Spread" of neighboring points.
         :param r_indices_func: Selector function of the type,
             INP
             ======
             ps : (N, N_rep, D) Representative points
             P  : (N, *, D) Point cloud
             N_neighbors : Number of points for each region.
+            dilution : "Spread" of neighboring points (analogous to stride).
 
             OUT
             ======
@@ -173,6 +162,7 @@ class PointCNN(nn.Module):
 
         self.r_indices_func = r_indices_func
         self.x_conv = XConv(C_in, C_out, D, N_neighbors, N_rep, C_lifted, mlp_width)
+        self.dilution = dilution
 
     def select_region(self, P, P_idx):
         """
@@ -204,12 +194,12 @@ class PointCNN(nn.Module):
         :return:
         """
         ps, P, F = x
-        P_idx = self.r_indices_func(ps.cpu(), P.cpu(), self.x_conv.N_neighbors).cuda()  # This step takes ~97% of the time.
-        P_regional = self.select_region(P, P_idx)                           # Prime target for optimization: KNN on GPU.
+        P_idx = self.r_indices_func(ps.cpu(), P.cpu(), self.x_conv.N_neighbors, self.dilution).cuda()  # This step takes ~97% of the time.
+        P_regional = self.select_region(P, P_idx)  # Prime target for optimization: KNN on GPU.
         if False:
             # Draw neighborhood points, for debugging.
-            t = 23
-            n = 3
+            t = 15
+            n = 0
             test_point = ps[n,t,:].cpu().data.numpy()
             neighborhood = P_regional[n,t,:,:].cpu().data.numpy()
             plt.scatter(P[n][:,0], P[n][:,1])
@@ -218,7 +208,27 @@ class PointCNN(nn.Module):
             plt.show()
         F_regional = self.select_region(F, P_idx)
         # ps, P, F_P -> ps_F
-        return self.x_conv((ps, P_regional, F_regional))
+        F_p = self.x_conv((ps, P_regional, F_regional))
+        return F_p
+
+class rPointCNN(nn.Module):
+    """ PointCNN with randomly sampled representative points. """
+
+    def __init__(self, *args, **kwargs):
+        super(rPointCNN, self).__init__()
+        self.pointcnn = PointCNN(*args, **kwargs)
+        self.N_rep = args[5]  # Exists because PointCNN requires it.
+
+    def forward(self, x):
+        P, F = x
+        if self.N_rep < P.size()[1]:
+            idx = np.random.choice(P.size()[1], self.N_rep, replace = False).tolist()
+            ps = P[:,idx,:]
+        else:
+            # All input points are representative points.
+            ps = P
+        ps_F = self.pointcnn((ps, P, F))
+        return ps, ps_F
 
 if __name__ == "__main__":
     np.random.seed(0)
@@ -242,14 +252,15 @@ if __name__ == "__main__":
 
     elif TESTING == PointCNN:
         N = 4
-        num_points = 4000
-        N_rep = 4000
+        num_points = 10000
+        N_rep = 5000
         D = 3
-        C_in = 768
-        C_out = 7
-        N_neighbors = 5
+        C_in = 128
+        C_out = 256
+        N_neighbors = 10
+        dilution = 2
 
-        model = PointCNN(C_in, C_out, D, N_neighbors, N_rep, knn_indices_func).cuda()
+        model = PointCNN(C_in, C_out, D, N_neighbors, dilution, N_rep, knn_indices_func).cuda()
 
         test_P  = np.random.rand(N,num_points,D).astype(np.float32)
         test_F  = np.random.rand(N,num_points,C_in).astype(np.float32)

pointcnn/layers.py

@@ -13,21 +13,78 @@ def endchannels(f, make_contiguous = False):
     class wrapped_layer(nn.Module):
         def __init__(self):
             super(wrapped_layer, self).__init__()
+            self.f = f
         def forward(self, x):
             x = x.permute(0,3,1,2)
-            x = f(x)
+            x = self.f(x)
             x = x.permute(0,2,3,1)
             return x
     return wrapped_layer()
 
-def DepthwiseSeparableConv2d(in_channels, out_channels, kernel_size, depth_multiplier):
+class Dense(nn.Module):
+
+    def __init__(self, in_features, out_features, drop_rate = 0, with_bn = True,
+                 activation = nn.ReLU()):
+        super(Dense, self).__init__()
+
+        self.linear = nn.Linear(in_features, out_features)
+        self.activation = activation
+        # self.bn = LayerNorm(out_channels) if with_bn else None
+        if drop_rate > 0:
+            self.drop = nn.Dropout(drop_rate)
+
+    def forward(self, x):
+        x = self.linear(x)
+        if self.activation:
+            x = self.activation(x)
+        # if self.bn:
+        #     x = self.bn(x)
+        if self.drop:
+            x = self.drop(x)
+        return x
+
+class Conv(nn.Module):
+
+    def __init__(self, in_channels, out_channels, kernel_size, with_bn = True,
+                 activation = nn.ReLU()):
+        super(Conv, self).__init__()
+
+        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size, bias = not with_bn)
+        self.activation = activation
+        self.bn = nn.BatchNorm2d(out_channels) if with_bn else None
+
+    def forward(self, x):
+        x = self.conv(x)
+        if self.activation:
+            x = self.activation(x)
+        if self.bn:
+            x = self.bn(x)
+        return x
+
+class SepConv(nn.Module):
     """
     Depthwise separable convolution
     """
-    return nn.Sequential(
-        nn.Conv2d(in_channels, in_channels * depth_multiplier, kernel_size, groups = in_channels),
-        nn.Conv2d(in_channels * depth_multiplier, out_channels, 1)
-    )
+
+    def __init__(self, in_channels, out_channels, kernel_size, depth_multiplier = 1,
+                 with_bn = True, activation = nn.ReLU()):
+        super(SepConv, self).__init__()
+
+        self.conv = nn.Sequential(
+            nn.Conv2d(in_channels, in_channels * depth_multiplier, kernel_size, groups = in_channels),
+            nn.Conv2d(in_channels * depth_multiplier, out_channels, 1, bias = not with_bn)
+        )
+
+        self.activation = activation
+        self.bn = nn.BatchNorm2d(out_channels) if with_bn else None
+
+    def forward(self, x):
+        x = self.conv(x)
+        if self.activation:
+            x = self.activation(x)
+        if self.bn:
+            x = self.bn(x)
+        return x
 
 class LayerNorm(nn.Module):
     """

pointcnn/util.py

@@ -6,9 +6,11 @@ from sklearn.neighbors import NearestNeighbors
 torch.CUDA_LAUNCH_BLOCKING = 1
 
 try:
-    from .context import pytorch_knn_cuda
+    # from .context import pytorch_knn_cuda
+    pass
 except SystemError:
-    from context import pytorch_knn_cuda
+    # from context import pytorch_knn_cuda
+    pass
 
 def apply_along_dim(xs, f, dim):
     """
@@ -31,7 +33,7 @@ def zipwith_matmul(xs, ys):
     N = len(xs)
     return torch.stack([torch.mm(xs[i], ys[i]) for i in range(N)], dim = 0)
 
-def knn_indices_func(ps, P, k):
+def knn_indices_func(ps, P, k, d):
     """
     Indexing function based on K-Nearest Neighbors search.
     :type ps: FloatTensor (N, N_rep, D)
@@ -48,9 +50,9 @@ def knn_indices_func(ps, P, k):
     P = P.data.numpy()
 
     def single_batch_knn(p, P_particular):
-        nbrs = NearestNeighbors(k + 1, algorithm = "ball_tree").fit(P_particular)
+        nbrs = NearestNeighbors(d*k + 1, algorithm = "ball_tree").fit(P_particular)
         indices = nbrs.kneighbors(p)[1]
-        return indices[:,1:]
+        return indices[:,1::d]
 
     region_idx = np.stack([
         single_batch_knn(p, P[n]) for n, p in enumerate(ps)

requirements.txt

@@ -4,3 +4,6 @@ numpy
 libKMCUDA
 http://download.pytorch.org/whl/cu80/torch-0.3.1-cp35-cp35m-linux_x86_64.whl
 torchvision
+h5py
+mnist
+plyfile