From 17af94de01bb8b7dd6bdef879bb4a9a7381e4011 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Mon, 6 Jun 2022 17:55:12 +0200 Subject: [PATCH] Fix read_parquet/feather to read files written by GDAL (#2443) --- ci/envs/38-latest-conda-forge.yaml | 2 +- geopandas/io/arrow.py | 46 ++++++++++++++++-- .../tests/data/arrow/test_data_gdal350.arrow | Bin 0 -> 4570 bytes .../data/arrow/test_data_gdal350.parquet | Bin 0 -> 3584 bytes geopandas/io/tests/test_arrow.py | 38 ++++++++++++++- 5 files changed, 80 insertions(+), 6 deletions(-) create mode 100644 geopandas/io/tests/data/arrow/test_data_gdal350.arrow create mode 100644 geopandas/io/tests/data/arrow/test_data_gdal350.parquet diff --git a/ci/envs/38-latest-conda-forge.yaml b/ci/envs/38-latest-conda-forge.yaml index c6ea712..e1d2c74 100644 --- a/ci/envs/38-latest-conda-forge.yaml +++ b/ci/envs/38-latest-conda-forge.yaml @@ -14,7 +14,7 @@ dependencies: - pytest - pytest-cov - pytest-xdist - - fsspec + # - fsspec # to have one non-minimal build without fsspec # optional - rtree - matplotlib diff --git a/geopandas/io/arrow.py b/geopandas/io/arrow.py index 256474d..2a174ad 100644 --- a/geopandas/io/arrow.py +++ b/geopandas/io/arrow.py @@ -311,13 +311,13 @@ def _to_feather(df, path, index=None, compression=None, version=None, **kwargs): feather.write_feather(table, path, compression=compression, **kwargs) -def _arrow_to_geopandas(table): +def _arrow_to_geopandas(table, metadata=None): """ Helper function with main, shared logic for read_parquet/read_feather. """ df = table.to_pandas() - metadata = table.schema.metadata + metadata = metadata or table.schema.metadata if metadata is None or b"geo" not in metadata: raise ValueError( """Missing geo metadata in Parquet/Feather file. @@ -410,6 +410,29 @@ def _get_filesystem_path(path, filesystem=None, storage_options=None): return filesystem, path +def _ensure_arrow_fs(filesystem): + """ + Simplified version of pyarrow.fs._ensure_filesystem. This is only needed + below because `pyarrow.parquet.read_metadata` does not yet accept a + filesystem keyword (https://issues.apache.org/jira/browse/ARROW-16719) + """ + from pyarrow import fs + + if isinstance(filesystem, fs.FileSystem): + return filesystem + + # handle fsspec-compatible filesystems + try: + import fsspec + except ImportError: + pass + else: + if isinstance(filesystem, fsspec.AbstractFileSystem): + return fs.PyFileSystem(fs.FSSpecHandler(filesystem)) + + return filesystem + + def _read_parquet(path, columns=None, storage_options=None, **kwargs): """ Load a Parquet object from the file path, returning a GeoDataFrame. @@ -487,7 +510,24 @@ def _read_parquet(path, columns=None, storage_options=None, **kwargs): kwargs["use_pandas_metadata"] = True table = parquet.read_table(path, columns=columns, filesystem=filesystem, **kwargs) - return _arrow_to_geopandas(table) + # read metadata separately to get the raw Parquet FileMetaData metadata + # (pyarrow doesn't properly exposes those in schema.metadata for files + # created by GDAL - https://issues.apache.org/jira/browse/ARROW-16688) + metadata = None + if table.schema.metadata is None or b"geo" not in table.schema.metadata: + try: + # read_metadata does not accept a filesystem keyword, so need to + # handle this manually (https://issues.apache.org/jira/browse/ARROW-16719) + if filesystem is not None: + pa_filesystem = _ensure_arrow_fs(filesystem) + with pa_filesystem.open_input_file(path) as source: + metadata = parquet.read_metadata(source).metadata + else: + metadata = parquet.read_metadata(path).metadata + except Exception: + pass + + return _arrow_to_geopandas(table, metadata) def _read_feather(path, columns=None, **kwargs): diff --git a/geopandas/io/tests/data/arrow/test_data_gdal350.arrow b/geopandas/io/tests/data/arrow/test_data_gdal350.arrow new file mode 100644 index 0000000000000000000000000000000000000000..41bc5c32f9d84e92770dfbeb22c59e0f1435149b GIT binary patch literal 4570 zcmeHK&2QsG6rX&w4OLVkwn#(VU*fr>EhOrDInr91amd&sfTA*0+MxRDE8S2wXYQrpajD7dl^>qqu!X&`Q+-1L^ zaUqX-Cwyofe$2g*?K*N*&gaxzURLB&&mLOd`C-o;JR6b$V}{)K&S495Be{CnL zNxYYqwF%&ws#gTuXf!r=+Re@N+sCo2m&&SMQnX4rrxi<@rWcoWrD1HFyBmAko4df$ z(bGYw-R{ll)#~-VR;~U~N6Y1*-f3;(n1D4s&Ye%}-U;w|rM7K0jL=J;AHzoKTwW~| zin>x|=uun&>-M0pc)>UecHz{rqdO>syEZ_E@^|_R7Sai;c<9hyy z(rm3GI8@;8Jl9K}UFSG*?K5#G&=2ibBDrv!}aV>UB|ZuQqLWp zx(;`I$$cv6>(WSsdXA#C)-q}Ui@P8cYirHNokx|t@<=V`6-Wr|kT^DDr)^?=1zlV2 zc5%y&v02sVb{_iYryQ4VXrJ-Ee0g~(@<0ZO;AhQVA(PCBoRDFQ&=AFv zi}*ePQ@#kWK|7iOYLQncm!vS7at>jMCGgwe8s)BPm7n>z!^kUD#~O0zDfb{Xumj|@ zufXt>pu8r5DPIx20Y-v3d^&I~A1kC)&I(JMy$Hs{$H&ZtV{FFw!A9iVyw1 z+rKu*rLh^3_rcNJk)M(8K}UZ<{@uX-(Y_F+u`59aPK&)`iKOtc1589|Xfha0YoUF5 z0j7D#M;shcG>6FLk#bBv2>aPkz7xugFjZAWF(xOHWomL-&fhw0r+@6?H-!x>wc_Awvqz@W#&q)N%}NuA+gj#$nji zx8D%`uoB2$R|N7u*7XyN(ygPt>4fX@VgEix0ZQ}65;JTUas~@|_xF`~7LT(eA|b_+ zah66S&Ssa`n_b2pq*foY1oXn1UJ4ttSz*1m_yN0@dCB^GaTU%%nN=a^gDZi16vgL) zh)#+-ih0@;itYjq?mqa>L=fxrd+03B{s`U$6qhM*luFQKzW{qNeFls_O0wqk88CeY z(39x(kD34d8E|yup2^h?URLzFQi=h0=?Q>j#MtMkA&8>?Q;`g`(RW__znaz&(>mf6 c1zFad))8Yh%e0P|))9BBBc7wM5p|LJH!qSH<^TWy literal 0 HcmV?d00001 diff --git a/geopandas/io/tests/data/arrow/test_data_gdal350.parquet b/geopandas/io/tests/data/arrow/test_data_gdal350.parquet new file mode 100644 index 0000000000000000000000000000000000000000..6baa7e8d0a44d960aaf84dc764e6c253a10ba5d0 GIT binary patch literal 3584 zcmeHKF>l*O6h6wdL%TLA2c|#)0!k2$>Y}tNl9FW^LF|(((@JDpmTV-cEd-RrC`2hz zCF$5s5Wqu+{)J}h(4pv@B}2QUL%U@tI`k)W%G9ywJ4&={x68%~5rZ(!e&(hu}?9)JQ1) zB^6^QqiGtAMY(GY+kD`94jYQlk)y~76~7s0`V*sFvJe7lnk5t1M9==H&lSXW$ zosWTI0t?3k*a(^%qJ_zWQT{R#W9h*t4a@VR{E3RZrs6+_;$MVfS786=RGxoD+RNpx z{CW9eDuIP#0xU`8qV`-WzmLV(iC!w>WWgVrwl<&qmax+rnm49P3ASt{z?Tq z^KEGM6@)>J!7D_KqY-K}4Bb&QP0??brzgg#>F_i&P2CEECd%EeA(~m4`f!RN>RTKl zB07g^)XTLW5Sp2~kHTkcvok9|2Q9pEex0D~UQu1gV5dgN3!YJUy6w$!&^tSRoBk{s zp+0U1IkVD(_hc{*Bs!-L#)0hPxY`i{w(%Tx#Aa?Rb==A=(fl4vhP?ipK zaN8hu0!IA<+nsT`Hz-0!W6{#TyH4Z({WTZ=KdztnyYfZn%Q6@Ta_AKRMK?r$v3w(c zPDSWA6Z!hR8HA!DH;F5it-DLzYUWNxJ|XR>mBScs!$aB~j1O_Hh1W1{;e5;2apwT zZC=VQrg+_Q^>*FVVZsK$3zffz&+Ryx~c`$Fw<6V!~7yaIYLqg(+kN$IPQOADFQ zRoF8N@)h8+q-1?~p;RiCt6Rm@!R=U3GK-Ru5#{V+LQZF7SxGM_VyUpH?yT=_7Iy$k zzS_w~NVVGO?%{lXw}SKc_vAzp)a6PMwh6GRqwCh8(Rc*-q=+}wQlZDv)LUR9>O@k? zq|%CzeEJm`Nm;X5Ou0wXj#sZ;{7)DxsyaixF(?4Jc+k;`q}+ndiO#d&ElDS`w6E97rmpUH}_SlWXnCCe(NyH8CNz%_V-CG!t04)j|2c~BP!`icF&k*-PM{_= Version("3.0.0") + + expected = geopandas.GeoDataFrame( + {"col_str": ["a", "b"], "col_int": [1, 2], "col_float": [0.1, 0.2]}, + geometry=[MultiPolygon([box(0, 0, 1, 1), box(2, 2, 3, 3)]), box(4, 4, 5, 5)], + crs="EPSG:4326", + ) + + df = geopandas.read_parquet(DATA_PATH / "arrow" / "test_data_gdal350.parquet") + assert_geodataframe_equal(df, expected, check_crs=check_crs) + + df = geopandas.read_feather(DATA_PATH / "arrow" / "test_data_gdal350.arrow") + assert_geodataframe_equal(df, expected, check_crs=check_crs)