diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d543f3..2fd3f56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,8 @@ Small bug-fix release: casted to GeometryDtype (#2482). - Fix a crash in datetime column reading where the file contains mixed timezone offsets (#2479). These will be read as UTC localized values. +- Fix a crash in datetime column reading where the file contains datetimes + outside the range supported by [ns] precision (#2505). - Fix regression in passing the Parquet or Feather format ``version`` in ``to_parquet`` and ``to_feather``. As a result, the ``version`` parameter for the ``to_parquet`` and ``to_feather`` methods has been replaced with diff --git a/geopandas/io/file.py b/geopandas/io/file.py index 3b4336e..8c62af5 100644 --- a/geopandas/io/file.py +++ b/geopandas/io/file.py @@ -341,13 +341,14 @@ def _read_file_fiona( f_filt, crs=crs, columns=columns + ["geometry"] ) for k in datetime_fields: - as_dt = pd.to_datetime(df[k]) + as_dt = pd.to_datetime(df[k], errors="ignore") # if to_datetime failed, try again for mixed timezone offsets if as_dt.dtype == "object": - # This can fail if there are invalid datetimes - as_dt = pd.to_datetime(df[k], utc=True) + # This can still fail if there are invalid datetimes + as_dt = pd.to_datetime(df[k], errors="ignore", utc=True) # if to_datetime succeeded, round datetimes as - # fiona only supports up to ms precision + # fiona only supports up to ms precision (any microseconds are + # floating point rounding error) if not (as_dt.dtype == "object"): df[k] = as_dt.dt.round(freq="ms") return df diff --git a/geopandas/io/tests/test_file.py b/geopandas/io/tests/test_file.py index 6ef5858..2ccbd94 100644 --- a/geopandas/io/tests/test_file.py +++ b/geopandas/io/tests/test_file.py @@ -33,12 +33,14 @@ except ImportError: try: import fiona - FIONA_GE_1814 = Version(fiona.__version__) >= Version( - "1.8.14" - ) # datetime roundtrip + # datetime roundtrip + FIONA_GE_1814 = Version(fiona.__version__) >= Version("1.8.14") + # invalid datetime handling + FIONA_GE_1821 = Version(fiona.__version__) >= Version("1.8.21") except ImportError: fiona = False FIONA_GE_1814 = False + FIONA_GE_1821 = False PYOGRIO_MARK = pytest.mark.skipif(not pyogrio, reason="pyogrio not installed") @@ -232,7 +234,63 @@ def test_to_file_datetime(tmpdir, driver, ext, time, engine): assert_series_equal(df["b"], df_read["b"]) -def test_read_file_mixed_datetimes(tmpdir): +dt_exts = ["gpkg", "geojson"] + + +def write_invalid_date_file(date_str, tmpdir, ext, engine): + tempfilename = os.path.join(str(tmpdir), f"test_invalid_datetime.{ext}") + df = GeoDataFrame( + { + "date": ["2014-08-26T10:01:23", "2014-08-26T10:01:23", date_str], + "geometry": [Point(1, 1), Point(1, 1), Point(1, 1)], + } + ) + # Schema not required for GeoJSON since not typed, but needed for GPKG + if ext == "geojson": + df.to_file(tempfilename) + else: + schema = {"geometry": "Point", "properties": {"date": "datetime"}} + if engine == "pyogrio" and not fiona: + # (use schema to write the invalid date without pandas datetimes + pytest.skip("test requires fiona kwarg schema") + df.to_file(tempfilename, schema=schema, engine="fiona") + return tempfilename + + +@pytest.mark.parametrize("ext", dt_exts) +def test_read_file_datetime_invalid(tmpdir, ext, engine): + # https://github.com/geopandas/geopandas/issues/2502 + if not FIONA_GE_1821 and ext == "gpkg": + # https://github.com/Toblerity/Fiona/issues/1035 + pytest.skip("Invalid datetime throws in Fiona<1.8.21") + + date_str = "9999-99-99T00:00:00" # invalid date handled by GDAL + tempfilename = write_invalid_date_file(date_str, tmpdir, ext, engine) + res = read_file(tempfilename) + if ext == "gpkg": + assert is_datetime64_any_dtype(res["date"]) + assert pd.isna(res["date"].iloc[-1]) + else: + assert res["date"].dtype == "object" + assert isinstance(res["date"].iloc[-1], str) + + +@pytest.mark.parametrize("ext", dt_exts) +def test_read_file_datetime_out_of_bounds_ns(tmpdir, ext, engine): + # https://github.com/geopandas/geopandas/issues/2502 + if ext == "geojson": + skip_pyogrio_not_supported(engine) + + date_str = "9999-12-31T00:00:00" # valid to GDAL, not to [ns] format + tempfilename = write_invalid_date_file(date_str, tmpdir, ext, engine) + res = read_file(tempfilename) + # Pandas invalid datetimes are read in as object dtype (strings) + assert res["date"].dtype == "object" + assert isinstance(res["date"].iloc[0], str) + + +def test_read_file_datetime_mixed_offsets(tmpdir): + # https://github.com/geopandas/geopandas/issues/2478 tempfilename = os.path.join(str(tmpdir), "test_mixed_datetime.geojson") df = GeoDataFrame( { @@ -244,10 +302,11 @@ def test_read_file_mixed_datetimes(tmpdir): } ) df.to_file(tempfilename) - res = read_file(tempfilename) # check mixed tz don't crash GH2478 - assert is_datetime64_any_dtype(res["date"]) + # check mixed tz don't crash GH2478 + res = read_file(tempfilename) if FIONA_GE_1814: # Convert mixed timezones to UTC equivalent + assert is_datetime64_any_dtype(res["date"]) assert res["date"].dt.tz == pytz.utc else: # old fiona and pyogrio ignore timezones and read as datetimes successfully