From 437b1ac95d532136c4e4351eafa58fb9912edf47 Mon Sep 17 00:00:00 2001 From: "Alan D. Snow" Date: Fri, 23 Sep 2022 15:46:35 -0500 Subject: [PATCH] ENH: Add where filter to read_file (#2552) --- CHANGELOG.md | 2 ++ doc/source/docs/user_guide/io.rst | 17 +++++++++++++++++ geopandas/io/file.py | 27 +++++++++++++++++++++------ geopandas/io/tests/test_file.py | 19 +++++++++++++++++++ 4 files changed, 59 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fd3f56..4253ca1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ Development version New features and improvements: + - Add where filter to ``read_file`` (#2552) + Deprecations and compatibility notes: Bug fixes: diff --git a/doc/source/docs/user_guide/io.rst b/doc/source/docs/user_guide/io.rst index f0c6371..0675413 100644 --- a/doc/source/docs/user_guide/io.rst +++ b/doc/source/docs/user_guide/io.rst @@ -177,6 +177,23 @@ Skip loading geometry from the file: ) +SQL WHERE Filter +^^^^^^^^^^^^^^^^^ + +.. versionadded:: 0.12 + +Load in a subset of data with a `SQL WHERE clause `__. + +.. note:: Requires Fiona 1.9+ or the pyogrio engine. + +.. code-block:: python + + gdf = geopandas.read_file( + geopandas.datasets.get_path("naturalearth_lowres"), + where="continent='Africa'", + ) + + Writing Spatial Data --------------------- diff --git a/geopandas/io/file.py b/geopandas/io/file.py index 1e20abf..3cfb795 100644 --- a/geopandas/io/file.py +++ b/geopandas/io/file.py @@ -26,12 +26,14 @@ _VALID_URLS.discard("") fiona = None fiona_env = None fiona_import_error = None +FIONA_GE_19 = False def _import_fiona(): global fiona global fiona_env global fiona_import_error + global FIONA_GE_19 if fiona is None: try: @@ -48,6 +50,9 @@ def _import_fiona(): except ImportError: fiona_env = None + FIONA_GE_19 = Version(Version(fiona.__version__).base_version) >= Version( + "1.9.0" + ) except ImportError as err: fiona = False fiona_import_error = str(err) @@ -263,8 +268,11 @@ def _read_file(filename, bbox=None, mask=None, rows=None, engine=None, **kwargs) def _read_file_fiona( - path_or_bytes, from_bytes, bbox=None, mask=None, rows=None, **kwargs + path_or_bytes, from_bytes, bbox=None, mask=None, rows=None, where=None, **kwargs ): + if where is not None and not FIONA_GE_19: + raise NotImplementedError("where requires fiona 1.9+") + if not from_bytes: # Opening a file via URL or file-like-object above automatically detects a # zipped file. In order to match that behavior, attempt to add a zip scheme @@ -319,17 +327,24 @@ def _read_file_fiona( mask = mapping(mask.to_crs(crs).unary_union) elif isinstance(mask, BaseGeometry): mask = mapping(mask) + + filters = {} + if bbox is not None: + filters["bbox"] = bbox + if mask is not None: + filters["mask"] = mask + if where is not None: + filters["where"] = where + # setup the data loading filter if rows is not None: if isinstance(rows, int): rows = slice(rows) elif not isinstance(rows, slice): raise TypeError("'rows' must be an integer or a slice.") - f_filt = features.filter( - rows.start, rows.stop, rows.step, bbox=bbox, mask=mask - ) - elif any((bbox, mask)): - f_filt = features.filter(bbox=bbox, mask=mask) + f_filt = features.filter(rows.start, rows.stop, rows.step, **filters) + elif filters: + f_filt = features.filter(**filters) else: f_filt = features # get list of columns diff --git a/geopandas/io/tests/test_file.py b/geopandas/io/tests/test_file.py index 2ccbd94..df21f74 100644 --- a/geopandas/io/tests/test_file.py +++ b/geopandas/io/tests/test_file.py @@ -37,10 +37,12 @@ try: FIONA_GE_1814 = Version(fiona.__version__) >= Version("1.8.14") # invalid datetime handling FIONA_GE_1821 = Version(fiona.__version__) >= Version("1.8.21") + FIONA_GE_19 = Version(Version(fiona.__version__).base_version) >= Version("1.9.0") except ImportError: fiona = False FIONA_GE_1814 = False FIONA_GE_1821 = False + FIONA_GE_19 = False PYOGRIO_MARK = pytest.mark.skipif(not pyogrio, reason="pyogrio not installed") @@ -766,6 +768,23 @@ def test_read_file__ignore_all_fields(engine): assert gdf.columns.tolist() == ["geometry"] +def test_read_file__where_filter(engine): + if FIONA_GE_19 or engine == "pyogrio": + gdf = geopandas.read_file( + geopandas.datasets.get_path("naturalearth_lowres"), + where="continent='Africa'", + engine=engine, + ) + assert gdf.continent.unique().tolist() == ["Africa"] + else: + with pytest.raises(NotImplementedError): + geopandas.read_file( + geopandas.datasets.get_path("naturalearth_lowres"), + where="continent='Africa'", + engine="fiona", + ) + + @PYOGRIO_MARK def test_read_file__columns(): # TODO: this is only support for pyogrio, but we could mimic it for fiona as well