BUG: fix mixed datetime regression (#2479)

This commit is contained in:
Matt Richards
2022-07-24 11:10:03 +02:00
committed by GitHub
parent 470fba5c2e
commit 2ce0068c7c
3 changed files with 34 additions and 3 deletions
+2
View File
@@ -14,6 +14,8 @@ Deprecations and compatibility notes:
`schema_version` if `version` is one of 0.1.0 or 0.4.0 (#2496).
Bug fixes:
- Fix a crash in datetime column reading where the file contains mixed timezone
offsets (#2479). These will be read as UTC localized values.
- Fix regression (RecursionError) in reshape methods such as ``unstack()``
and ``pivot()`` involving MultiIndex, or GeoDataFrame construction with
+9 -3
View File
@@ -341,9 +341,15 @@ def _read_file_fiona(
f_filt, crs=crs, columns=columns + ["geometry"]
)
for k in datetime_fields:
# fiona only supports up to ms precision, any microseconds are
# floating point rounding error
df[k] = pd.to_datetime(df[k]).dt.round(freq="ms")
as_dt = pd.to_datetime(df[k])
# if to_datetime failed, try again for mixed timezone offsets
if as_dt.dtype == "object":
# This can fail if there are invalid datetimes
as_dt = pd.to_datetime(df[k], utc=True)
# if to_datetime succeeded, round datetimes as
# fiona only supports up to ms precision
if not (as_dt.dtype == "object"):
df[k] = as_dt.dt.round(freq="ms")
return df
+23
View File
@@ -10,6 +10,7 @@ import numpy as np
import pandas as pd
import pytz
from pandas.api.types import is_datetime64_any_dtype
from pandas.testing import assert_series_equal
from shapely.geometry import Point, Polygon, box
@@ -231,6 +232,28 @@ def test_to_file_datetime(tmpdir, driver, ext, time, engine):
assert_series_equal(df["b"], df_read["b"])
def test_read_file_mixed_datetimes(tmpdir):
tempfilename = os.path.join(str(tmpdir), "test_mixed_datetime.geojson")
df = GeoDataFrame(
{
"date": [
"2014-08-26 10:01:23.040001+02:00",
"2019-03-07 17:31:43.118999+01:00",
],
"geometry": [Point(1, 1), Point(1, 1)],
}
)
df.to_file(tempfilename)
res = read_file(tempfilename) # check mixed tz don't crash GH2478
assert is_datetime64_any_dtype(res["date"])
if FIONA_GE_1814:
# Convert mixed timezones to UTC equivalent
assert res["date"].dt.tz == pytz.utc
else:
# old fiona and pyogrio ignore timezones and read as datetimes successfully
assert is_datetime64_any_dtype(res["date"])
@pytest.mark.parametrize("driver,ext", driver_ext_pairs)
def test_to_file_with_point_z(tmpdir, ext, driver, engine):
"""Test that 3D geometries are retained in writes (GH #612)."""