Skip to content

Commit

Permalink
fix?
Browse files Browse the repository at this point in the history
  • Loading branch information
lithomas1 committed Apr 23, 2023
1 parent ba3e3be commit 3e41179
Show file tree
Hide file tree
Showing 3 changed files with 18 additions and 3 deletions.
16 changes: 16 additions & 0 deletions pandas/io/parsers/arrow_parser_wrapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,21 @@ def _get_pyarrow_options(self) -> None:
if pandas_name in self.kwds and self.kwds.get(pandas_name) is not None:
self.kwds[pyarrow_name] = self.kwds.pop(pandas_name)

# Date format handling
# If we get a string, we need to convert it into a list for pyarrow
# If we get a dict, we want to parse those separately
date_format = self.date_format
if isinstance(date_format, str):
date_format = [date_format]
else:
# In case of dict, we don't want to propagate through, so
# just set to pyarrow default of None

# Ideally, in future we disable pyarrow dtype inference (read in as string)
# to prevent misreads.
date_format = None
self.kwds["timestamp_parsers"] = date_format

self.parse_options = {
option_name: option_value
for option_name, option_value in self.kwds.items()
Expand All @@ -79,6 +94,7 @@ def _get_pyarrow_options(self) -> None:
"true_values",
"false_values",
"decimal_point",
"timestamp_parsers",
)
}
self.convert_options["strings_can_be_null"] = "" in self.kwds["null_values"]
Expand Down
4 changes: 2 additions & 2 deletions pandas/io/parsers/base_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,10 @@

from pandas import (
ArrowDtype,
DataFrame,
DatetimeIndex,
StringDtype,
concat,
)
from pandas.core import algorithms
from pandas.core.arrays import (
Expand Down Expand Up @@ -93,8 +95,6 @@
Scalar,
)

from pandas import DataFrame


class ParserBase:
class BadLineHandleMethod(Enum):
Expand Down
1 change: 0 additions & 1 deletion pandas/tests/io/parser/test_parse_dates.py
Original file line number Diff line number Diff line change
Expand Up @@ -544,7 +544,6 @@ def test_multiple_date_cols_int_cast(all_parsers):
tm.assert_frame_equal(result, expected)


@xfail_pyarrow
def test_multiple_date_col_timestamp_parse(all_parsers):
parser = all_parsers
data = """05/31/2012,15:30:00.029,1306.25,1,E,0,,1306.25
Expand Down

0 comments on commit 3e41179

Please sign in to comment.