BUG: move nan-handling to tslib and clean up

AlexKirko · AlexKirko · commit 620506d1857b · 2020-01-01T11:02:33.000+03:00
diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx
@@ -296,17 +296,34 @@ def format_array_from_datetime(ndarray[int64_t] values, object tz=None,
     return result
 
 
-def array_with_unit_to_datetime(ndarray values, object unit,
+def array_with_unit_to_datetime(ndarray values, ndarray mask, object unit,
                                 str errors='coerce'):
     """
-    convert the ndarray according to the unit
+    Convert the ndarray to datetime according to the time unit.
+
+    This function converts an array of objects into a numpy array of
+    datetime64[ns]. It returns the converted array
+    and also returns the timezone offset
+
     if errors:
       - raise: return converted values or raise OutOfBoundsDatetime
           if out of range on the conversion or
           ValueError for other conversions (e.g. a string)
       - ignore: return non-convertible values as the same unit
       - coerce: NaT for non-convertibles
 
+    Parameters
+    ----------
+    values : ndarray of object
+         Date-like objects to convert
+    mask : ndarray of bool
+         Not-a-time mask for non-nullable integer types conversion,
+         can be None
+    unit : object
+         Time unit to use during conversion
+    errors : str, default 'raise'
+         Error behavior when parsing
+
     Returns
     -------
     result : ndarray of m8 values
@@ -316,7 +333,6 @@ def array_with_unit_to_datetime(ndarray values, object unit,
         Py_ssize_t i, j, n=len(values)
         int64_t m
         ndarray[float64_t] fvalues
-        ndarray mask
         bint is_ignore = errors=='ignore'
         bint is_coerce = errors=='coerce'
         bint is_raise = errors=='raise'
@@ -329,9 +345,13 @@ def array_with_unit_to_datetime(ndarray values, object unit,
 
     if unit == 'ns':
         if issubclass(values.dtype.type, np.integer):
-            return values.astype('M8[ns]'), tz
-        # This will return a tz
-        return array_to_datetime(values.astype(object), errors=errors)
+            result = values.astype('M8[ns]')
+        else:
+            result, tz = array_to_datetime(values.astype(object), errors=errors)
+        if mask is not None:
+            iresult = result.view('i8')
+            iresult[mask] = NPY_NAT
+        return result, tz
 
     m = cast_from_unit(None, unit)
 
@@ -343,7 +363,9 @@ def array_with_unit_to_datetime(ndarray values, object unit,
         if values.dtype.kind == "i":
             # Note: this condition makes the casting="same_kind" redundant
             iresult = values.astype('i8', casting='same_kind', copy=False)
-            mask = iresult == NPY_NAT
+            # If no mask, fill mask by comparing to NPY_NAT constant
+            if mask is None:
+                mask = iresult == NPY_NAT
             iresult[mask] = 0
             fvalues = iresult.astype('f8') * m
             need_to_iterate = False
diff --git a/pandas/core/tools/datetimes.py b/pandas/core/tools/datetimes.py
@@ -38,7 +38,6 @@
 )
 from pandas.core.dtypes.missing import notna
 
-from pandas._typing import ArrayLike
 from pandas.arrays import IntegerArray
 from pandas.core import algorithms
 from pandas.core.algorithms import unique
@@ -319,22 +318,20 @@ def _convert_listlike_datetimes(
         if format is not None:
             raise ValueError("cannot specify both format and unit")
         arg = getattr(arg, "_values", arg)
+
         # GH 30050 pass an ndarray to tslib.array_with_unit_to_datetime
         # because it expects an ndarray argument
         if isinstance(arg, IntegerArray):
-            # Send only non-na values to array_with_unit_to_datetime
-            mask_na = arg.isna()
-            result_np, tz_parsed = tslib.array_with_unit_to_datetime(
-                np.compress(np.logical_not(mask_na), arg), unit, errors=errors
-            )
-            # Insert na values back in proper positions
-            ins_index = np.ravel(np.argwhere(mask_na))
-            ins_index -= range(ins_index.shape[0])
-            result = np.insert(result_np, ins_index, None)
+            # Explicitly pass NaT mask to array_with_unit_to_datetime
+            mask = arg.isna()
+            arg = arg._ndarray_values
         else:
-            result, tz_parsed = tslib.array_with_unit_to_datetime(
-                arg, unit, errors=errors
-            )
+            mask = None
+
+        result, tz_parsed = tslib.array_with_unit_to_datetime(
+            arg, mask, unit, errors=errors
+        )
+
         if errors == "ignore":
             from pandas import Index
 
diff --git a/pandas/tests/indexes/datetimes/test_tools.py b/pandas/tests/indexes/datetimes/test_tools.py
@@ -2293,10 +2293,11 @@ def test_should_cache_errors(unique_share, check_count, err_message):
         tools.should_cache(arg, unique_share, check_count)
 
 
-def test_intarray_to_datetime():
+def test_nullable_integer_to_datetime():
     # Test for #30050
     ser = pd.Series([1, 2, None, 2 ** 61, None])
     ser = ser.astype("Int64")
+    ser_copy = ser.copy()
 
     res = pd.to_datetime(ser, unit="ns")
 
@@ -2310,3 +2311,5 @@ def test_intarray_to_datetime():
         ]
     )
     tm.assert_series_equal(res, expected)
+    # Check that ser isn't mutated
+    tm.assert_series_equal(ser, ser_copy)

Original file line number	Diff line number	Diff line change
`@@ -2293,10 +2293,11 @@ def test_should_cache_errors(unique_share, check_count, err_message):`
`2293`	`2293`	`tools.should_cache(arg, unique_share, check_count)`
`2294`	`2294`
`2295`	`2295`
`2296`		`-def test_intarray_to_datetime():`
	`2296`	`+def test_nullable_integer_to_datetime():`
`2297`	`2297`	`# Test for #30050`
`2298`	`2298`	`ser = pd.Series([1, 2, None, 2 ** 61, None])`
`2299`	`2299`	`ser = ser.astype("Int64")`
	`2300`	`+ ser_copy = ser.copy()`
`2300`	`2301`
`2301`	`2302`	`res = pd.to_datetime(ser, unit="ns")`
`2302`	`2303`
`@@ -2310,3 +2311,5 @@ def test_intarray_to_datetime():`
`2310`	`2311`	`]`
`2311`	`2312`	`)`
`2312`	`2313`	`tm.assert_series_equal(res, expected)`
	`2314`	`+ # Check that ser isn't mutated`
	`2315`	`+ tm.assert_series_equal(ser, ser_copy)`