Refactor match_descriptors and fix small bugs in ORB

This commit is contained in:
Johannes Schönberger
2013-11-30 03:55:40 +01:00
parent 2705907270
commit f22e00e693
10 changed files with 152 additions and 317 deletions
+43 -58
View File
@@ -1,78 +1,63 @@
import numpy as np
from skimage import data
from skimage import transform as tf
from skimage.feature import (pairwise_hamming_distance,
match_binary_descriptors, corner_harris,
corner_peaks, keypoints_orb, descriptor_orb)
from skimage.feature import (match_descriptors, corner_harris,
corner_peaks, ORB)
from skimage.color import rgb2gray
from skimage import img_as_float
import matplotlib.pyplot as plt
# Initializing parameters for transformation
rotate = 0.5
translate = (-100, -200)
scaling = (1.5, 1.5)
# Creating a transformed image from the original Lena image by scaling and
# rotating it
img_color = data.lena()
tform = tf.AffineTransform(scale = scaling, rotation=rotate,
translation=translate)
transformed_img_color = tf.warp(img_color, tform)
img = rgb2gray(img_color)
transformed_img = rgb2gray(transformed_img_color)
img1_color = data.lena()
img2_color = tf.rotate(img1_color, 180)
tform = tf.AffineTransform(scale=(1.3, 1.1), rotation=0.5,
translation=(0, -200))
img3_color = tf.warp(img1_color, tform)
img1 = rgb2gray(img1_color)
img2 = rgb2gray(img2_color)
img3 = rgb2gray(img3_color)
# Extracting oFAST keypoints and computing their rBRIEF descriptors
keypoints1 = keypoints_orb(img, n_keypoints=500)
keypoints1.shape
descriptors1, keypoints1 = descriptor_orb(img, keypoints1)
keypoints1.shape
descriptors1.shape
descriptor_extractor = ORB(n_keypoints=200)
keypoints1, descriptors1 = descriptor_extractor.detect_and_extract(img1)
keypoints2, descriptors2 = descriptor_extractor.detect_and_extract(img2)
keypoints3, descriptors3 = descriptor_extractor.detect_and_extract(img3)
keypoints2 = keypoints_orb(transformed_img,
n_keypoints=500)
keypoints2.shape
descriptors2, keypoints2 = descriptor_orb(transformed_img, keypoints2)
keypoints2.shape
descriptors2.shape
idxs1, idxs2 = match_descriptors(descriptors1, descriptors2, cross_check=True)
src12 = keypoints1[idxs1]
dst12 = keypoints2[idxs2]
#Initializing parameters for Descriptor matching
match_threshold = 0.3
match_cross_check = True
idxs1, idxs3 = match_descriptors(descriptors1, descriptors3, cross_check=True)
src13 = keypoints1[idxs1]
dst13 = keypoints3[idxs3]
pairwise_hamming_distance(descriptors1, descriptors2)
matched_keypoints, mask1, mask2 = match_binary_descriptors(keypoints1,
descriptors1,
keypoints2,
descriptors2,
cross_check=match_cross_check,
threshold=match_threshold)
img12 = np.concatenate((img_as_float(img1_color),
img_as_float(img2_color)), axis=1)
img13 = np.concatenate((img_as_float(img1_color),
img_as_float(img3_color)), axis=1)
matched_keypoints.shape
imgs = (img12, img13)
srcs = (src12, src13)
dsts = (dst12, dst13)
# Plotting the matched correspondences in both the images using matplotlib
src = matched_keypoints[:, 0, :]
dst = matched_keypoints[:, 1, :]
src_scale = 10 * (keypoints1.octave[mask1] + 1) ** 1.5
dst_scale = 10 * (keypoints2.octave[mask2] + 1) ** 1.5
offset = img1.shape
img_combined = np.concatenate((img_as_float(img_color),
img_as_float(transformed_img_color)), axis=1)
offset = img.shape
fig, ax = plt.subplots(nrows=2, ncols=1)
fig, ax = plt.subplots(nrows=1, ncols=1)
plt.gray()
for i in range(2):
ax.imshow(img_combined, interpolation='nearest')
ax.axis('off')
ax.axis((0, 2 * offset[1], offset[0], 0))
ax.set_title('Matched correspondences : Rotation = %f; Scale = %s; Translation = %s; threshold = %f; cross_check = %r' % (rotate, scaling, translate, match_threshold, match_cross_check))
ax[i].imshow(imgs[i], interpolation='nearest')
ax[i].axis('off')
ax[i].axis((0, 2 * offset[1], offset[0], 0))
for m in range(len(src)):
c = np.random.rand(3,1)
ax.plot((src[m, 1], dst[m, 1] + offset[1]), (src[m, 0], dst[m, 0]), '-', color=c)
ax.scatter(src[m, 1], src[m, 0], src_scale[m], facecolors='none', edgecolors=c)
ax.scatter(dst[m, 1] + offset[1], dst[m, 0], dst_scale[m], facecolors='none', edgecolors=c)
src = srcs[i]
dst = dsts[i]
for m in range(len(src)):
color = np.random.rand(3, 1)
ax[i].plot((src[m, 1], dst[m, 1] + offset[1]), (src[m, 0], dst[m, 0]),
'-', color=color)
ax[i].scatter(src[m, 1], src[m, 0], facecolors='none', edgecolors=color)
ax[i].scatter(dst[m, 1] + offset[1], dst[m, 0], facecolors='none',
edgecolors=color)
plt.show()
+2 -4
View File
@@ -12,8 +12,7 @@ from .template import match_template
from .brief import BRIEF
from .censure import CenSurE
from .orb import ORB
from .match import match_binary_descriptors
from .util import pairwise_hamming_distance
from .match import match_descriptors
__all__ = ['daisy',
@@ -39,5 +38,4 @@ __all__ = ['daisy',
'BRIEF',
'CenSurE',
'ORB',
'pairwise_hamming_distance',
'match_binary_descriptors']
'match_descriptors']
+5 -4
View File
@@ -7,7 +7,7 @@ cimport numpy as cnp
from libc.float cimport DBL_MAX
from libc.math cimport atan2
from skimage.util import img_as_float
from skimage.util import img_as_float, pad
from skimage.color import rgb2grey
from .util import _prepare_grayscale_input_2D
@@ -240,7 +240,8 @@ def corner_orientations(image, Py_ssize_t[:, :] corners, mask):
if mask.shape[0] % 2 != 1 or mask.shape[1] % 2 != 1:
raise ValueError("Size of mask must be uneven.")
cdef double[:, :] cimage = image
cdef double[:, :] cimage = pad(image, 16, mode='constant',
constant_values=0)
cdef char[:, ::1] cmask = np.ascontiguousarray(mask != 0, dtype=np.uint8)
cdef Py_ssize_t i, r, c, r0, c0
@@ -253,8 +254,8 @@ def corner_orientations(image, Py_ssize_t[:, :] corners, mask):
cdef double m01, m10
for i in range(corners.shape[0]):
r0 = corners[i, 0] - mrows2
c0 = corners[i, 1] - mcols2
r0 = corners[i, 0] - mrows2 + 16
c0 = corners[i, 1] - mcols2 + 16
m01 = 0
m10 = 0
+43 -59
View File
@@ -1,81 +1,65 @@
import numpy as np
from .util import pairwise_hamming_distance
from .match_cy import _binary_cross_check_loop
from scipy.spatial.distance import cdist
def match_binary_descriptors(keypoints1, descriptors1, keypoints2,
descriptors2, threshold=0.40, cross_check=True):
"""Match keypoints described using binary descriptors in one image to
those in second image.
def match_descriptors(descriptors1, descriptors2, metric=None, p=2,
threshold=0, cross_check=True):
"""Brute-force matching of descriptors.
For each descriptor in the first set this matcher finds the closest
descriptor in the second set (and vice-versa in the case of enabled
cross-checking).
Parameters
----------
keypoints1 : record array with M rows
Record array with fields row, col, octave, orientation, response.
Octave, orientation and response can be None.
descriptors1 : (M, P) ndarray
descriptors1 : (M, P) array
Binary descriptors of size P about M keypoints in the first image.
keypoints2 : record array with N rows
Record array with fields row, col, octave, orientation, response.
Octave, orientation and response can be None.
descriptors2 : (N, P) ndarray
descriptors2 : (N, P) array
Binary descriptors of size P about N keypoints in the second image.
threshold : float in range [0, 1]
Maximum allowable hamming distance between descriptors of two keypoints
metric : {'euclidean', 'cityblock', 'minkowski', 'hamming', ...}
The metric to compute the distance between two descriptors. See
`scipy.spatial.distance.cdist` for all possible types. The hamming
distance should be used for binary descriptors. By default the L2-norm
is used for all descriptors of dtype float or double and the Hamming
distance is used for binary descriptors automatically.
p : int
The p-norm to apply for ``metric='minkowski'``.
threshold : float
Maximum allowed distance between descriptors of two keypoints
in separate images to be regarded as a match.
cross_check : bool
If True, the matched keypoints are returned after cross checking i.e. a
matched pair (keypoint1, keypoint2) is returned iff keypoint2 is the best
match for keypoint1 in second image and keypoint1 is the best match for
keypoint2 in first image.
matched pair (keypoint1, keypoint2) is returned if keypoint2 is the
best match for keypoint1 in second image and keypoint1 is the best
match for keypoint2 in first image.
Returns
-------
matches : (Q, 2, 2) ndarray
Location of Q matched keypoint pairs from two images.
idxs1 : (Q,) ndarray
Indices of keypoints in keypoints1 that have been matched.
idxs2 : (Q,) ndarray
Indices of keypoints in keypoints2 that have been matched.
indices1 : (Q, ) array
Indices of corresponding matches for first set of descriptors.
indices2 : (Q, ) array
Indices of corresponding matches for second set of descriptors.
"""
if (keypoints1.shape[0] != descriptors1.shape[0]
or keypoints2.shape[0] != descriptors2.shape[0]):
raise ValueError("The number of keypoints and number of described "
"keypoints do not match.")
if descriptors1.shape[1] != descriptors2.shape[1]:
raise ValueError("Descriptor sizes for matching keypoints in both "
"the images should be equal.")
raise ValueError("Descriptor length must equal.")
# Get hamming distances between keypoints1 and keypoints2
distance = pairwise_hamming_distance(descriptors1, descriptors2)
if metric is None:
if np.issubdtype(descriptors1.dtype, np.bool):
metric = 'hamming'
else:
metric = 'euclidean'
distances = cdist(descriptors1, descriptors2, metric=metric, p=p)
indices1 = np.arange(descriptors1.shape[0])
indices2 = np.argmin(distances, axis=1)
if cross_check:
matched_keypoints1_index = np.argmin(distance, axis=1)
matched_keypoints2_index = np.argmin(distance, axis=0)
matches1 = np.argmin(distances, axis=0)
mask = indices1 == matches1[indices2]
indices1 = indices1[mask]
indices2 = indices2[mask]
matched_idxs = _binary_cross_check_loop(matched_keypoints1_index,
matched_keypoints2_index,
distance, threshold)
matches = np.zeros((matched_idxs.shape[0], 2, 2),
dtype=np.intp)
idxs1 = matched_idxs[:, 0]
idxs2 = matched_idxs[:, 1]
matches[:, 0, :] = keypoints1[idxs1]
matches[:, 1, :] = keypoints2[idxs2]
else:
temp = distance > threshold
row_check = np.any(~temp, axis=1)
matched_keypoints2 = keypoints2[np.argmin(distance, axis=1)]
matches = np.zeros((np.sum(row_check), 2, 2),
dtype=np.intp)
matches[:, 0, :] = keypoints1[row_check]
matches[:, 1, :] = matched_keypoints2[row_check]
idxs1 = np.where(row_check == True)[0]
idxs2 = np.argmin(distance, axis=1)[row_check]
return matches, mask1, mask2
return indices1, indices2
-18
View File
@@ -1,18 +0,0 @@
import numpy as np
def _binary_cross_check_loop(Py_ssize_t[:] matched_keypoints1_index,
Py_ssize_t[:] matched_keypoints2_index,
double[:, ::1] distance, double threshold):
cdef Py_ssize_t i
cdef Py_ssize_t count = 0
cdef Py_ssize_t[:, ::1] matched_index = np.zeros((len(matched_keypoints1_index), 2), dtype=np.intp)
for i in range(len(matched_keypoints1_index)):
if (matched_keypoints2_index[matched_keypoints1_index[i]] == i and
distance[i, matched_keypoints1_index[i]] < threshold):
matched_index[count, 0] = i
matched_index[count, 1] = matched_keypoints1_index[i]
count += 1
return np.asarray(matched_index[:count, :])
+4 -4
View File
@@ -151,7 +151,7 @@ class ORB(FeatureDetector, DescriptorExtractor):
def _extract_octave(self, octave_image, keypoints, orientations):
mask = _mask_border_keypoints(octave_image.shape, keypoints,
distance=16)
distance=20)
keypoints = np.array(keypoints[mask], dtype=np.intp, order='C',
copy=False)
orientations = np.array(orientations[mask], dtype=np.double, order='C',
@@ -260,8 +260,8 @@ class ORB(FeatureDetector, DescriptorExtractor):
descriptors, mask = self._extract_octave(octave_image, keypoints,
orientations)
keypoints_list.append(keypoints * self.downscale ** octave)
responses_list.append(responses)
keypoints_list.append(keypoints[mask] * self.downscale ** octave)
responses_list.append(responses[mask])
descriptors_list.append(descriptors)
keypoints = np.vstack(keypoints_list)
@@ -273,4 +273,4 @@ class ORB(FeatureDetector, DescriptorExtractor):
else:
# Choose best n_keypoints according to Harris corner response
best_indices = responses.argsort()[::-1][:self.n_keypoints]
return (keypoints[best_indices], descriptors[best_indices])
return keypoints[best_indices], descriptors[best_indices]
-3
View File
@@ -16,7 +16,6 @@ def configuration(parent_package='', top_path=None):
cython(['censure_cy.pyx'], working_path=base_path)
cython(['orb_cy.pyx'], working_path=base_path)
cython(['brief_cy.pyx'], working_path=base_path)
cython(['match_cy.pyx'], working_path=base_path)
cython(['_texture.pyx'], working_path=base_path)
cython(['_template.pyx'], working_path=base_path)
@@ -28,8 +27,6 @@ def configuration(parent_package='', top_path=None):
include_dirs=[get_numpy_include_dirs()])
config.add_extension('brief_cy', sources=['brief_cy.c'],
include_dirs=[get_numpy_include_dirs()])
config.add_extension('match_cy', sources=['match_cy.c'],
include_dirs=[get_numpy_include_dirs()])
config.add_extension('_texture', sources=['_texture.c'],
include_dirs=[get_numpy_include_dirs(), '../_shared'])
config.add_extension('_template', sources=['_template.c'],
+55 -140
View File
@@ -1,45 +1,32 @@
import numpy as np
from numpy.testing import assert_array_equal, assert_raises
from numpy.testing import assert_equal, assert_raises
from skimage import data
from skimage import transform as tf
from skimage.color import rgb2gray
from skimage.feature import (descriptor_brief, match_binary_descriptors,
corner_peaks, corner_harris,
create_keypoint_recarray)
from skimage.feature import (BRIEF, match_descriptors,
corner_peaks, corner_harris)
def test_match_binary_descriptors_unequal_descriptor_keypoints_error():
"""Number of descriptors should be equal to the number of keypoints."""
kp1 = np.array([[40, 50],
[60, 40],
[30, 70]])
keypoints1 = create_keypoint_recarray(kp1[:, 0], kp1[:, 1])
des1 = np.array([[True, True, False, True],
[False, True, False, True]])
kp2 = np.array([[60, 50],
[50, 80]])
keypoints2 = create_keypoint_recarray(kp2[:, 0], kp2[:, 1])
des2 = np.array([[True, False, False, True],
[False, True, True, True]])
assert_raises(ValueError, match_binary_descriptors, keypoints1, des1, keypoints2, des2)
def test_match_binary_descriptors_unequal_descriptor_sizes_error():
def test_binary_descriptors_unequal_descriptor_sizes_error():
"""Sizes of descriptors of keypoints to be matched should be equal."""
kp1 = np.array([[40, 50],
[60, 40]])
keypoints1 = create_keypoint_recarray(kp1[:, 0], kp1[:, 1])
des1 = np.array([[True, True, False, True],
[False, True, False, True]])
kp2 = np.array([[60, 50],
[50, 80]])
keypoints2 = create_keypoint_recarray(kp2[:, 0], kp2[:, 1])
des2 = np.array([[True, False, False, True, False],
[False, True, True, True, False]])
assert_raises(ValueError, match_binary_descriptors, keypoints1, des1, keypoints2, des2)
assert_raises(ValueError, match_descriptors, des1, des2)
def test_match_binary_descriptors_lena_rotation_crosscheck_false():
def test_binary_descriptors():
des1 = np.array([[True, True, False, True, True],
[False, True, False, True, True]])
des2 = np.array([[True, False, False, True, False],
[False, False, True, True, True]])
indices1, indices2 = match_descriptors(des1, des2)
assert_equal(indices1, [0, 1])
assert_equal(indices2, [0, 1])
def test_binary_descriptors_lena_rotation_crosscheck_false():
"""Verify matched keypoints and their corresponding masks results between
lena image and its rotated version with the expected keypoint pairs with
cross_check disabled."""
@@ -48,72 +35,35 @@ def test_match_binary_descriptors_lena_rotation_crosscheck_false():
tform = tf.SimilarityTransform(scale=1, rotation=0.15, translation=(0, 0))
rotated_img = tf.warp(img, tform)
kp1 = corner_peaks(corner_harris(img), min_distance=5)
keypoints1 = create_keypoint_recarray(kp1[:, 0], kp1[:, 1])
descriptors1, keypoints1 = descriptor_brief(img, keypoints1,
descriptor_size=512)
descriptor = BRIEF(descriptor_size=512)
kp2 = corner_peaks(corner_harris(rotated_img), min_distance=5)
keypoints2 = create_keypoint_recarray(kp2[:, 0], kp2[:, 1])
descriptors2, keypoints2 = descriptor_brief(rotated_img, keypoints2,
descriptor_size=512)
keypoints1 = corner_peaks(corner_harris(img), min_distance=5)
descriptors1, mask1 = descriptor.extract(img, keypoints1)
keypoints1 = keypoints1[mask1]
matched_keypoints, m1, m2 = match_binary_descriptors(keypoints1,
descriptors1,
keypoints2,
descriptors2,
threshold=0.13,
cross_check=False)
keypoints2 = corner_peaks(corner_harris(rotated_img), min_distance=5)
descriptors2, mask2 = descriptor.extract(rotated_img, keypoints2)
keypoints2 = keypoints1[mask2]
expected_mask1 = np.array([11, 12, 16, 20, 24, 26, 27, 29, 35, 39, 40,
42, 45])
expected_mask2 = np.array([ 1, 3, 0, 4, 6, 7, 8, 9, 10, 10, 11,
12, 13])
expected = np.array([[[245, 141],
[221, 176]],
m1, m2 = match_descriptors(descriptors1, descriptors2, threshold=0.13,
cross_check=False)
[[247, 130],
[225, 165]],
print m1
print m2
[[263, 272],
[219, 309]],
[[271, 120],
[250, 159]],
[[311, 174],
[282, 218]],
[[323, 164],
[294, 210]],
[[327, 147],
[301, 195]],
[[377, 157],
[349, 211]],
[[414, 70],
[399, 131]],
[[425, 67],
[399, 131]],
[[435, 181],
[403, 244]],
[[454, 176],
[423, 242]],
[[467, 166],
[437, 234]]])
assert_array_equal(matched_keypoints, expected)
assert_array_equal(m1, expected_mask1)
assert_array_equal(m2, expected_mask2)
expected_mask1 = np.array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46])
expected_mask2 = np.array([33, 0, 35, 7, 1, 35, 3, 2, 3, 6, 4, 9,
11, 10, 28, 7, 8, 5, 31, 14, 13, 15, 21, 16,
16, 13, 17, 18, 19, 21, 22, 23, 0, 24, 1, 24,
23, 0, 26, 27, 25, 34, 28, 14, 29, 30, 21])
assert_equal(m1, expected_mask1)
assert_equal(m2, expected_mask2)
def test_match_binary_descriptors_lena_rotation_crosscheck_true():
def test_binary_descriptors_lena_rotation_crosscheck_true():
"""Verify matched keypoints and their corresponding masks results between
lena image and its rotated version with the expected keypoint pairs with
cross_check enabled."""
@@ -122,62 +72,27 @@ def test_match_binary_descriptors_lena_rotation_crosscheck_true():
tform = tf.SimilarityTransform(scale=1, rotation=0.15, translation=(0, 0))
rotated_img = tf.warp(img, tform)
kp1 = corner_peaks(corner_harris(img), min_distance=5)
keypoints1 = create_keypoint_recarray(kp1[:, 0], kp1[:, 1])
descriptors1, keypoints1 = descriptor_brief(img, keypoints1, descriptor_size=512)
descriptor = BRIEF(descriptor_size=512)
kp2 = corner_peaks(corner_harris(rotated_img), min_distance=5)
keypoints2 = create_keypoint_recarray(kp2[:, 0], kp2[:, 1])
descriptors2, keypoints2 = descriptor_brief(rotated_img, keypoints2,
descriptor_size=512)
keypoints1 = corner_peaks(corner_harris(img), min_distance=5)
descriptors1, mask1 = descriptor.extract(img, keypoints1)
keypoints1 = keypoints1[mask1]
matched_keypoints, m1, m2 = match_binary_descriptors(keypoints1,
descriptors1,
keypoints2,
descriptors2,
threshold=0.13)
keypoints2 = corner_peaks(corner_harris(rotated_img), min_distance=5)
descriptors2, mask2 = descriptor.extract(rotated_img, keypoints2)
keypoints2 = keypoints1[mask2]
expected = np.array([[[245, 141],
[221, 176]],
m1, m2 = match_descriptors(descriptors1, descriptors2, threshold=0.13,
cross_check=True)
[[247, 130],
[225, 165]],
[[263, 272],
[219, 309]],
[[271, 120],
[250, 159]],
[[311, 174],
[282, 218]],
[[323, 164],
[294, 210]],
[[327, 147],
[301, 195]],
[[377, 157],
[349, 211]],
[[414, 70],
[399, 131]],
[[435, 181],
[403, 244]],
[[454, 176],
[423, 242]],
[[467, 166],
[437, 234]]])
expected_mask1 = np.array([11, 12, 16, 20, 24, 26, 27, 29, 35, 40, 42, 45])
expected_mask2 = np.array([ 1, 3, 0, 4, 6, 7, 8, 9, 10, 11, 12, 13])
assert_array_equal(matched_keypoints, expected)
assert_array_equal(m1, expected_mask1)
assert_array_equal(m2, expected_mask2)
expected_mask1 = np.array([ 0, 1, 2, 4, 6, 7, 9, 10, 11, 12, 13, 15,
16, 17, 19, 20, 21, 24, 26, 27, 28, 29, 30, 35,
36, 38, 39, 40, 42, 44, 45])
expected_mask2 = np.array([33, 0, 35, 1, 3, 2, 6, 4, 9, 11, 10, 7,
8, 5, 14, 13, 15, 16, 17, 18, 19, 21, 22, 24,
23, 26, 27, 25, 28, 29, 30])
assert_equal(m1, expected_mask1)
assert_equal(m2, expected_mask2)
if __name__ == '__main__':
-2
View File
@@ -105,8 +105,6 @@ def test_descriptor_orb():
keypoints2, descriptors2 = detector_extractor.detect_and_extract(img)
assert_array_equal(exp_descriptors, descriptors2[100:120, 10:20])
assert_array_equal(keypoints1[mask1], keypoints2)
if __name__ == '__main__':
from numpy import testing
-25
View File
@@ -76,28 +76,3 @@ def _mask_border_keypoints(image_shape, keypoints, distance):
& (keypoints[:, 1] < (cols - distance + 1)))
return mask
def pairwise_hamming_distance(array1, array2):
"""**Experimental function**.
Calculate hamming dissimilarity measure between two sets of
vectors.
Parameters
----------
array1 : (P1, D) array
P1 vectors of size D.
array2 : (P2, D) array
P2 vectors of size D.
Returns
-------
distance : (P1, P2) array of dtype float
2D ndarray with value at an index (i, j) representing the hamming
distance in the range [0, 1] between ith vector in array1 and jth
vector in array2.
"""
distance = (array1[:, None] != array2[None]).mean(axis=2)
return distance