ENH(string dtype): Make str.decode return str dtype

pandas/core/strings/accessor.py

@@ -12,6 +12,8 @@
 
 import numpy as np
 
+from pandas._config import get_option
+
 from pandas._libs import lib
 from pandas._typing import (
     AlignJoin,
@@ -399,7 +401,9 @@ def cons_row(x):
             # This is a mess.
             _dtype: DtypeObj | str | None = dtype
             vdtype = getattr(result, "dtype", None)
-            if self._is_string:
+            if _dtype is not None:
+                pass
+            elif self._is_string:
                 if is_bool_dtype(vdtype):
                     _dtype = result.dtype
                 elif returns_string:
@@ -2140,9 +2144,9 @@ def decode(self, encoding, errors: str = "strict"):
             decoder = codecs.getdecoder(encoding)
             f = lambda x: decoder(x, errors)[0]
         arr = self._data.array
-        # assert isinstance(arr, (StringArray,))
         result = arr._str_map(f)
-        return self._wrap_result(result)
+        dtype = "str" if get_option("future.infer_string") else None
+        return self._wrap_result(result, dtype=dtype)
 
     @forbid_nonstring_types(["bytes"])
     def encode(self, encoding, errors: str = "strict"):

pandas/io/pytables.py

@@ -5209,7 +5209,11 @@ def _unconvert_string_array(
         dtype = f"U{itemsize}"
 
         if isinstance(data[0], bytes):
-            data = Series(data, copy=False).str.decode(encoding, errors=errors)._values
+            ser = Series(data, copy=False).str.decode(encoding, errors=errors)
+            if get_option("future.infer_string"):
+                data = ser.to_numpy()
+            else:
+                data = ser._values
         else:
             data = data.astype(dtype, copy=False).astype(object, copy=False)
 

pandas/io/sas/sas7bdat.py

@@ -22,6 +22,8 @@
 
 import numpy as np
 
+from pandas._config import get_option
+
 from pandas._libs.byteswap import (
     read_double_with_byteswap,
     read_float_with_byteswap,
@@ -699,6 +701,7 @@ def _chunk_to_dataframe(self) -> DataFrame:
         rslt = {}
 
         js, jb = 0, 0
+        infer_string = get_option("future.infer_string")
         for j in range(self.column_count):
             name = self.column_names[j]
 
@@ -715,6 +718,9 @@ def _chunk_to_dataframe(self) -> DataFrame:
                 rslt[name] = pd.Series(self._string_chunk[js, :], index=ix, copy=False)
                 if self.convert_text and (self.encoding is not None):
                     rslt[name] = self._decode_string(rslt[name].str)
+                    if infer_string:
+                        rslt[name] = rslt[name].astype("str")
+
                 js += 1
             else:
                 self.close()

pandas/tests/io/sas/test_sas7bdat.py

@@ -7,8 +7,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 from pandas.compat._constants import (
     IS64,
     WASM,
@@ -20,10 +18,6 @@
 
 from pandas.io.sas.sas7bdat import SAS7BDATReader
 
-pytestmark = pytest.mark.xfail(
-    using_string_dtype(), reason="TODO(infer_string)", strict=False
-)
-
 
 @pytest.fixture
 def dirpath(datapath):
@@ -246,11 +240,13 @@ def test_zero_variables(datapath):
         pd.read_sas(fname)
 
 
-def test_zero_rows(datapath):
+@pytest.mark.parametrize("encoding", [None, "utf8"])
+def test_zero_rows(datapath, encoding):
     # GH 18198
     fname = datapath("io", "sas", "data", "zero_rows.sas7bdat")
-    result = pd.read_sas(fname)
-    expected = pd.DataFrame([{"char_field": "a", "num_field": 1.0}]).iloc[:0]
+    result = pd.read_sas(fname, encoding=encoding)
+    str_value = b"a" if encoding is None else "a"
+    expected = pd.DataFrame([{"char_field": str_value, "num_field": 1.0}]).iloc[:0]
     tm.assert_frame_equal(result, expected)
 
 
@@ -409,7 +405,7 @@ def test_0x40_control_byte(datapath):
     fname = datapath("io", "sas", "data", "0x40controlbyte.sas7bdat")
     df = pd.read_sas(fname, encoding="ascii")
     fname = datapath("io", "sas", "data", "0x40controlbyte.csv")
-    df0 = pd.read_csv(fname, dtype="object")
+    df0 = pd.read_csv(fname, dtype="str")
     tm.assert_frame_equal(df, df0)
 
 

pandas/tests/strings/test_strings.py

@@ -566,7 +566,7 @@ def test_string_slice_out_of_bounds(any_string_dtype):
 def test_encode_decode(any_string_dtype):
     ser = Series(["a", "b", "a\xe4"], dtype=any_string_dtype).str.encode("utf-8")
     result = ser.str.decode("utf-8")
-    expected = ser.map(lambda x: x.decode("utf-8")).astype(object)
+    expected = Series(["a", "b", "a\xe4"], dtype="str")
     tm.assert_series_equal(result, expected)