From 7c389a625f38bb597fbd13d2d497635089ef362e Mon Sep 17 00:00:00 2001 From: Jacob Wasserman Date: Wed, 16 Oct 2013 00:17:52 -0400 Subject: [PATCH 1/4] Initial commit of geocoding functionality Geocode a Series or list of strings and get back a GeoDataFrame with the Point objects and a full geocoded address. --- geopandas/geocode.py | 84 +++++++++++++++++++++++++++++++++++++++++++ requirements.test.txt | 1 + tests/test_geocode.py | 36 +++++++++++++++++++ 3 files changed, 121 insertions(+) create mode 100644 geopandas/geocode.py create mode 100644 tests/test_geocode.py diff --git a/geopandas/geocode.py b/geopandas/geocode.py new file mode 100644 index 0000000..335311b --- /dev/null +++ b/geopandas/geocode.py @@ -0,0 +1,84 @@ +from collections import defaultdict + +import fiona +import geopy +import numpy as np +import pandas as pd +from shapely.geometry import Point + +import geopandas as gpd + +def geocode(strings, provider='googlev3', **kwargs): + """ + Geocode a set of strings and get a GeoDataFrame of the resulting points + + Parameters + ---------- + strings : list or Series of addresses to geocode + provider : geopy geocoder to use, default 'googlev3' + Some providers require additional arguments such as access keys + * googlev3 + * bing + * google + * yahoo + * mapquest + * openmapquest + + Consult the terms of service for each provider to ensure proper use + of the results. + + Example + ------- + >>> df = geocode(['boston, ma', '1600 pennsylvania ave. washington, dc']) + address geometry + 0 Boston, MA, USA POINT (-71.0597731999999951 42.3584308000000007) + 1 1600 Pennsylvania Avenue Northwest, President'... POINT (-77.0365122999999983 38.8978377999999978) + + """ + if not isinstance(strings, pd.Series): + strings = pd.Series(strings) + + coders = {'googlev3': geopy.geocoders.GoogleV3, + 'bing': geopy.geocoders.Bing, + 'google': geopy.geocoders.Google, + 'yahoo': geopy.geocoders.Yahoo, + 'mapquest': geopy.geocoders.MapQuest, + 'openmapquest': geopy.geocoders.OpenMapQuest} + + if provider not in coders: + raise ValueError('Unknown geocoding provider: {}'.format(provider)) + + coder = coders[provider](**kwargs) + results = {} + for i, s in strings.iteritems(): + # Probably want some try/catch here, but what to do on exception? + results[i] = coder.geocode(s) + + df = _prepare_geocode_result(results) + return df + +def _prepare_geocode_result(results): + """ + Helper function for the geocode function + + Takes a dict where keys are index entries, values are tuples containing: + (address, (lat, lon)) + + """ + # Prepare the data for the DataFrame as a dict of lists + d = defaultdict(list) + index = [] + + for i, s in results.iteritems(): + address, loc = s + + # loc is lat, lon and we want lon, lat + p = Point(loc[1], loc[0]) + d['geometry'].append(p) + d['address'].append(address) + index.append(i) + + df = gpd.GeoDataFrame(d, index=index) + df.crs = fiona.crs.from_epsg(4326) + + return df diff --git a/requirements.test.txt b/requirements.test.txt index 7f762a0..2e9afd0 100644 --- a/requirements.test.txt +++ b/requirements.test.txt @@ -1 +1,2 @@ psycopg2>=2.5.1 +geopy>=0.95.1 diff --git a/tests/test_geocode.py b/tests/test_geocode.py new file mode 100644 index 0000000..8bc1772 --- /dev/null +++ b/tests/test_geocode.py @@ -0,0 +1,36 @@ +import unittest + +import fiona +from shapely.geometry import Point +import geopandas as gpd + +from geopandas.geocode import geocode, _prepare_geocode_result + +class TestGeocode(unittest.TestCase): + def test_prepare_result(self): + # Calls _prepare_result with sample results from the geocoder call + # loop + p0 = Point(12.3, -45.6) # Treat these as lat/lon + p1 = Point(-23.4, 56.7) + d = {'a': ('address0', p0.coords[0]), + 'b': ('address1', p1.coords[0])} + + df = _prepare_geocode_result(d) + assert type(df) is gpd.GeoDataFrame + self.assertEqual(fiona.crs.from_epsg(4326), df.crs) + self.assertEqual(len(df), 2) + self.assertIn('address', df) + + coords = df.loc['a']['geometry'].coords[0] + test = p0.coords[0] + # Output from the df should be lon/lat + self.assertAlmostEqual(coords[0], test[1]) + self.assertAlmostEqual(coords[1], test[0]) + + coords = df.loc['b']['geometry'].coords[0] + test = p1.coords[0] + self.assertAlmostEqual(coords[0], test[1]) + self.assertAlmostEqual(coords[1], test[0]) + + def test_bad_provider(self): + self.assertRaises(ValueError, geocode, ['cambridge, ma'], 'badprovider') From 01cea97dad211746d2d5ba4ae5c03aa06121a544 Mon Sep 17 00:00:00 2001 From: Jacob Wasserman Date: Sun, 20 Oct 2013 19:27:50 -0400 Subject: [PATCH 2/4] Change assertIn to assert_ for python 2.6 support --- tests/test_geocode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_geocode.py b/tests/test_geocode.py index 8bc1772..e0df4d8 100644 --- a/tests/test_geocode.py +++ b/tests/test_geocode.py @@ -19,7 +19,7 @@ class TestGeocode(unittest.TestCase): assert type(df) is gpd.GeoDataFrame self.assertEqual(fiona.crs.from_epsg(4326), df.crs) self.assertEqual(len(df), 2) - self.assertIn('address', df) + self.assert_('address' in df) coords = df.loc['a']['geometry'].coords[0] test = p0.coords[0] From 2b02fc3dffc6c1d1db81d4fc02ceee3260fe3a00 Mon Sep 17 00:00:00 2001 From: Jacob Wasserman Date: Sun, 20 Oct 2013 20:37:39 -0400 Subject: [PATCH 3/4] Catch geocoding erros and fill with empty rows Catch GeocoderResultError and ValueError. Fill the resulting GeoDataFrame with an empty row. --- geopandas/geocode.py | 16 +++++++++++++--- tests/test_geocode.py | 16 ++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/geopandas/geocode.py b/geopandas/geocode.py index 335311b..3853ea1 100644 --- a/geopandas/geocode.py +++ b/geopandas/geocode.py @@ -2,6 +2,7 @@ from collections import defaultdict import fiona import geopy +from geopy.geocoders.base import GeocoderResultError import numpy as np import pandas as pd from shapely.geometry import Point @@ -51,8 +52,10 @@ def geocode(strings, provider='googlev3', **kwargs): coder = coders[provider](**kwargs) results = {} for i, s in strings.iteritems(): - # Probably want some try/catch here, but what to do on exception? - results[i] = coder.geocode(s) + try: + results[i] = coder.geocode(s) + except (GeocoderResultError, ValueError): + results[i] = (None, None) df = _prepare_geocode_result(results) return df @@ -73,7 +76,14 @@ def _prepare_geocode_result(results): address, loc = s # loc is lat, lon and we want lon, lat - p = Point(loc[1], loc[0]) + if loc is None: + p = Point() + else: + p = Point(loc[1], loc[0]) + + if address is None: + address = pd.np.nan + d['geometry'].append(p) d['address'].append(address) index.append(i) diff --git a/tests/test_geocode.py b/tests/test_geocode.py index e0df4d8..de5e777 100644 --- a/tests/test_geocode.py +++ b/tests/test_geocode.py @@ -1,6 +1,7 @@ import unittest import fiona +import pandas as pd from shapely.geometry import Point import geopandas as gpd @@ -32,5 +33,20 @@ class TestGeocode(unittest.TestCase): self.assertAlmostEqual(coords[0], test[1]) self.assertAlmostEqual(coords[1], test[0]) + def test_prepare_result_none(self): + p0 = Point(12.3, -45.6) # Treat these as lat/lon + d = {'a': ('address0', p0.coords[0]), + 'b': (None, None)} + + df = _prepare_geocode_result(d) + assert type(df) is gpd.GeoDataFrame + self.assertEqual(fiona.crs.from_epsg(4326), df.crs) + self.assertEqual(len(df), 2) + self.assert_('address' in df) + + row = df.loc['b'] + self.assertEqual(len(row['geometry'].coords), 0) + self.assert_(pd.np.isnan(row['address'])) + def test_bad_provider(self): self.assertRaises(ValueError, geocode, ['cambridge, ma'], 'badprovider') From e3f99438876226d0bf53b8fbc4e11911d573545e Mon Sep 17 00:00:00 2001 From: Jacob Wasserman Date: Sun, 20 Oct 2013 20:38:54 -0400 Subject: [PATCH 4/4] Add some more documentation about geopy --- geopandas/geocode.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/geopandas/geocode.py b/geopandas/geocode.py index 3853ea1..f6da451 100644 --- a/geopandas/geocode.py +++ b/geopandas/geocode.py @@ -11,22 +11,26 @@ import geopandas as gpd def geocode(strings, provider='googlev3', **kwargs): """ - Geocode a set of strings and get a GeoDataFrame of the resulting points + Geocode a set of strings and get a GeoDataFrame of the resulting points. Parameters ---------- strings : list or Series of addresses to geocode provider : geopy geocoder to use, default 'googlev3' Some providers require additional arguments such as access keys - * googlev3 + See each geocoder's specific parameters in geopy.geocoders + * googlev3, default * bing * google * yahoo * mapquest * openmapquest - Consult the terms of service for each provider to ensure proper use - of the results. + Ensure proper use of the results by consulting the Terms of Service for + your provider. + + Geocoding requires geopy. Install it using 'pip install geopy'. See also + https://github.com/geopy/geopy Example -------