BUG: Fix out of bounds datetime (#2505)

Co-authored-by: Joris Van den Bossche <jorisvandenbossche@gmail.com>

CHANGELOG.md

@@ -26,6 +26,8 @@ Small bug-fix release:
   casted to GeometryDtype (#2482). 
 - Fix a crash in datetime column reading where the file contains mixed timezone
   offsets (#2479). These will be read as UTC localized values.
+- Fix a crash in datetime column reading where the file contains datetimes
+  outside the range supported by [ns] precision (#2505).
 - Fix regression in passing the Parquet or Feather format ``version`` in
   ``to_parquet`` and ``to_feather``. As a result, the ``version`` parameter
   for the ``to_parquet`` and ``to_feather`` methods has been replaced with

geopandas/io/file.py

@@ -341,13 +341,14 @@ def _read_file_fiona(
                     f_filt, crs=crs, columns=columns + ["geometry"]
                 )
             for k in datetime_fields:
-                as_dt = pd.to_datetime(df[k])
+                as_dt = pd.to_datetime(df[k], errors="ignore")
                 # if to_datetime failed, try again for mixed timezone offsets
                 if as_dt.dtype == "object":
-                    # This can fail if there are invalid datetimes
-                    as_dt = pd.to_datetime(df[k], utc=True)
+                    # This can still fail if there are invalid datetimes
+                    as_dt = pd.to_datetime(df[k], errors="ignore", utc=True)
                 # if to_datetime succeeded, round datetimes as
-                # fiona only supports up to ms precision
+                # fiona only supports up to ms precision (any microseconds are
+                # floating point rounding error)
                 if not (as_dt.dtype == "object"):
                     df[k] = as_dt.dt.round(freq="ms")
             return df

geopandas/io/tests/test_file.py

@@ -33,12 +33,14 @@ except ImportError:
 try:
     import fiona
 
-    FIONA_GE_1814 = Version(fiona.__version__) >= Version(
-        "1.8.14"
-    )  # datetime roundtrip
+    # datetime roundtrip
+    FIONA_GE_1814 = Version(fiona.__version__) >= Version("1.8.14")
+    # invalid datetime handling
+    FIONA_GE_1821 = Version(fiona.__version__) >= Version("1.8.21")
 except ImportError:
     fiona = False
     FIONA_GE_1814 = False
+    FIONA_GE_1821 = False
 
 
 PYOGRIO_MARK = pytest.mark.skipif(not pyogrio, reason="pyogrio not installed")
@@ -232,7 +234,63 @@ def test_to_file_datetime(tmpdir, driver, ext, time, engine):
         assert_series_equal(df["b"], df_read["b"])
 
 
-def test_read_file_mixed_datetimes(tmpdir):
+dt_exts = ["gpkg", "geojson"]
+
+
+def write_invalid_date_file(date_str, tmpdir, ext, engine):
+    tempfilename = os.path.join(str(tmpdir), f"test_invalid_datetime.{ext}")
+    df = GeoDataFrame(
+        {
+            "date": ["2014-08-26T10:01:23", "2014-08-26T10:01:23", date_str],
+            "geometry": [Point(1, 1), Point(1, 1), Point(1, 1)],
+        }
+    )
+    # Schema not required for GeoJSON since not typed, but needed for GPKG
+    if ext == "geojson":
+        df.to_file(tempfilename)
+    else:
+        schema = {"geometry": "Point", "properties": {"date": "datetime"}}
+        if engine == "pyogrio" and not fiona:
+            # (use schema to write the invalid date without pandas datetimes
+            pytest.skip("test requires fiona kwarg schema")
+        df.to_file(tempfilename, schema=schema, engine="fiona")
+    return tempfilename
+
+
+@pytest.mark.parametrize("ext", dt_exts)
+def test_read_file_datetime_invalid(tmpdir, ext, engine):
+    # https://github.com/geopandas/geopandas/issues/2502
+    if not FIONA_GE_1821 and ext == "gpkg":
+        # https://github.com/Toblerity/Fiona/issues/1035
+        pytest.skip("Invalid datetime throws in Fiona<1.8.21")
+
+    date_str = "9999-99-99T00:00:00"  # invalid date handled by GDAL
+    tempfilename = write_invalid_date_file(date_str, tmpdir, ext, engine)
+    res = read_file(tempfilename)
+    if ext == "gpkg":
+        assert is_datetime64_any_dtype(res["date"])
+        assert pd.isna(res["date"].iloc[-1])
+    else:
+        assert res["date"].dtype == "object"
+        assert isinstance(res["date"].iloc[-1], str)
+
+
+@pytest.mark.parametrize("ext", dt_exts)
+def test_read_file_datetime_out_of_bounds_ns(tmpdir, ext, engine):
+    # https://github.com/geopandas/geopandas/issues/2502
+    if ext == "geojson":
+        skip_pyogrio_not_supported(engine)
+
+    date_str = "9999-12-31T00:00:00"  # valid to GDAL, not to [ns] format
+    tempfilename = write_invalid_date_file(date_str, tmpdir, ext, engine)
+    res = read_file(tempfilename)
+    # Pandas invalid datetimes are read in as object dtype (strings)
+    assert res["date"].dtype == "object"
+    assert isinstance(res["date"].iloc[0], str)
+
+
+def test_read_file_datetime_mixed_offsets(tmpdir):
+    # https://github.com/geopandas/geopandas/issues/2478
     tempfilename = os.path.join(str(tmpdir), "test_mixed_datetime.geojson")
     df = GeoDataFrame(
         {
@@ -244,10 +302,11 @@ def test_read_file_mixed_datetimes(tmpdir):
         }
     )
     df.to_file(tempfilename)
-    res = read_file(tempfilename)  # check mixed tz don't crash GH2478
-    assert is_datetime64_any_dtype(res["date"])
+    # check mixed tz don't crash GH2478
+    res = read_file(tempfilename)
     if FIONA_GE_1814:
         # Convert mixed timezones to UTC equivalent
+        assert is_datetime64_any_dtype(res["date"])
         assert res["date"].dt.tz == pytz.utc
     else:
         # old fiona and pyogrio ignore timezones and read as datetimes successfully