From a07f6a739dd689f842a54ebb72a632e73300ae4d Mon Sep 17 00:00:00 2001 From: Marcus Read Date: Sat, 8 Apr 2023 23:40:10 +0100 Subject: [PATCH 1/2] Revise for pandas 2.0 Changes to accommodate changes in pandas 2.0. Mainly responding to pandas default tz changing from `pytz.UTC` to `datetime.timezone.utc`. These revisions retain all timezones as pytz in `exchange_calendars`. Also: * makes changes in light of new pandas Warnings. * resolves #294 which moved from `FutureWarning` to Error when `CustomBusinessDay.apply` was deprecated in pandas 2.0. --- exchange_calendars/calendar_helpers.py | 6 +++--- exchange_calendars/pandas_extensions/offsets.py | 2 +- exchange_calendars/utils/pandas_utils.py | 2 +- tests/test_calendar_helpers.py | 2 +- tests/test_exchange_calendar.py | 16 +++++++++++----- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/exchange_calendars/calendar_helpers.py b/exchange_calendars/calendar_helpers.py index 88d8b606..8df5a6bd 100644 --- a/exchange_calendars/calendar_helpers.py +++ b/exchange_calendars/calendar_helpers.py @@ -680,7 +680,7 @@ def trading_index(self) -> pd.DatetimeIndex: index = self._trading_index() if self.has_break: index.sort() - index = pd.DatetimeIndex(index, tz="UTC") + index = pd.DatetimeIndex(index, tz=pytz.UTC) return self.curtail_for_times(index) @contextlib.contextmanager @@ -718,7 +718,7 @@ def trading_index_intervals(self) -> pd.IntervalIndex: else: raise errors.IntervalsOverlapError() - left = pd.DatetimeIndex(left, tz="UTC") - right = pd.DatetimeIndex(right, tz="UTC") + left = pd.DatetimeIndex(left, tz=pytz.UTC) + right = pd.DatetimeIndex(right, tz=pytz.UTC) index = pd.IntervalIndex.from_arrays(left, right, self.closed) return self.curtail_for_times(index) diff --git a/exchange_calendars/pandas_extensions/offsets.py b/exchange_calendars/pandas_extensions/offsets.py index 9d2e9449..ef8626a1 100644 --- a/exchange_calendars/pandas_extensions/offsets.py +++ b/exchange_calendars/pandas_extensions/offsets.py @@ -149,7 +149,7 @@ def _apply(self, other): bday, interval = self._custom_business_day_for( other, remaining, with_interval=True ) - result = bday.apply(other) + result = bday + other while not interval.left <= result <= interval.right: previous_other = other if result < interval.left: diff --git a/exchange_calendars/utils/pandas_utils.py b/exchange_calendars/utils/pandas_utils.py index 78fcab13..80b4206b 100644 --- a/exchange_calendars/utils/pandas_utils.py +++ b/exchange_calendars/utils/pandas_utils.py @@ -117,7 +117,7 @@ def longest_run(ser: pd.Series) -> pd.Index: ... | ((ser >= 55) & (ser < 61)) ... ) >>> longest_run(bv) - Int64Index([30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40], dtype='int64') + Index([30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40], dtype='int32') >>> pd.testing.assert_index_equal(longest_run(bv), ser.index[30:41]) """ # group Trues by only adding to sum when value False. diff --git a/tests/test_calendar_helpers.py b/tests/test_calendar_helpers.py index d060b5e5..57e7f4e1 100644 --- a/tests/test_calendar_helpers.py +++ b/tests/test_calendar_helpers.py @@ -639,7 +639,7 @@ def bounds(start: pd.Series, end: pd.Series, force: bool, align: pd.Timedelta): if curtail and not (force_close and force_break_close): indices = lower_bounds.argsort() - lower_bounds.sort_values(inplace=True) + lower_bounds = lower_bounds.sort_values() upper_bounds = upper_bounds[indices] curtail_mask = upper_bounds > lower_bounds.shift(-1) if curtail_mask.any(): diff --git a/tests/test_exchange_calendar.py b/tests/test_exchange_calendar.py index 030d0fc5..88c8c717 100644 --- a/tests/test_exchange_calendar.py +++ b/tests/test_exchange_calendar.py @@ -11,6 +11,7 @@ # limitations under the License. from __future__ import annotations +import datetime import functools import itertools import pathlib @@ -181,7 +182,6 @@ def get_csv(name: str) -> pd.DataFrame: path, index_col=0, parse_dates=[0, 1, 2, 3, 4], - infer_datetime_format=True, ) # Necessary for csv saved prior to v4.0 if df.index.tz is not None: @@ -190,6 +190,8 @@ def get_csv(name: str) -> pd.DataFrame: for col in df: if df[col].dt.tz is None: df[col] = df[col].dt.tz_localize(UTC) + elif df[col].dt.tz is datetime.timezone.utc: + df[col] = df[col].dt.tz_convert(UTC) return df @@ -2146,7 +2148,7 @@ def late_opens( date_to = pd.Timestamp.max dtis: list[pd.DatetimeIndex] = [] # For each period over which a distinct open time prevails... - for date_from, time_ in s.iteritems(): + for date_from, time_ in s.items(): opens = ans.opens[date_from:date_to] sessions = opens.index td = pd.Timedelta(hours=time_.hour, minutes=time_.minute) @@ -2186,7 +2188,7 @@ def early_closes( date_to = pd.Timestamp.max dtis: list[pd.DatetimeIndex] = [] - for date_from, time_ in s.iteritems(): + for date_from, time_ in s.items(): closes = ans.closes[date_from:date_to] # index to tz-naive sessions = closes.index td = pd.Timedelta(hours=time_.hour, minutes=time_.minute) @@ -3837,14 +3839,18 @@ def unite(dtis: list[pd.DatetimeIndex]) -> pd.DatetimeIndex: else: ends = ans.closes # index for a 'left' calendar, add end so evaluated as if 'both' - index = index.append(pd.DatetimeIndex([ends[session]])) + index = index.append( + pd.DatetimeIndex([ends[session]], tz=pytz.UTC) + ) index = index[::mins] # only want every period if not index[-1] == ends[session]: # if period doesn't coincide with end, add right side of # last interval which lies beyond end. last_indice = index[-1] + period - index = index.append(pd.DatetimeIndex([last_indice])) + index = index.append( + pd.DatetimeIndex([last_indice], tz=pytz.UTC) + ) dtis.append(index) both_index = unite(dtis) From 0a7aa5cc6f75dae95414b297bc21982811e607c5 Mon Sep 17 00:00:00 2001 From: Marcus Read Date: Sat, 8 Apr 2023 23:40:48 +0100 Subject: [PATCH 2/2] Update dependencies Includes updating pandas to 2.0. --- etc/requirements.txt | 14 +++++++------ etc/requirements_dev.txt | 37 +++++++++++++++++----------------- etc/requirements_minpandas.txt | 33 +++++++++++++++--------------- 3 files changed, 43 insertions(+), 41 deletions(-) diff --git a/etc/requirements.txt b/etc/requirements.txt index db3a8c23..f8ae7529 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -1,24 +1,24 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile --output-file=etc/requirements.txt pyproject.toml # korean-lunar-calendar==0.3.1 # via exchange-calendars (pyproject.toml) -numpy==1.24.1 +numpy==1.24.2 # via # exchange-calendars (pyproject.toml) # pandas -pandas==1.5.2 +pandas==2.0.0 # via exchange-calendars (pyproject.toml) -pyluach==2.0.2 +pyluach==2.2.0 # via exchange-calendars (pyproject.toml) python-dateutil==2.8.2 # via # exchange-calendars (pyproject.toml) # pandas -pytz==2022.7 +pytz==2023.3 # via # exchange-calendars (pyproject.toml) # pandas @@ -26,3 +26,5 @@ six==1.16.0 # via python-dateutil toolz==0.12.0 # via exchange-calendars (pyproject.toml) +tzdata==2023.3 + # via pandas diff --git a/etc/requirements_dev.txt b/etc/requirements_dev.txt index b762ffdc..e782fc16 100644 --- a/etc/requirements_dev.txt +++ b/etc/requirements_dev.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile --extra=dev --output-file=etc/requirements_dev.txt pyproject.toml # @@ -8,7 +8,7 @@ attrs==22.2.0 # via # hypothesis # pytest -build==0.9.0 +build==0.10.0 # via pip-tools click==8.1.3 # via pip-tools @@ -17,7 +17,7 @@ colorama==0.4.6 # build # click # pytest -exceptiongroup==1.1.0 +exceptiongroup==1.1.1 # via # hypothesis # pytest @@ -25,27 +25,25 @@ execnet==1.9.0 # via pytest-xdist flake8==6.0.0 # via exchange-calendars (pyproject.toml) -hypothesis==6.61.0 +hypothesis==6.71.0 # via exchange-calendars (pyproject.toml) -iniconfig==1.1.1 +iniconfig==2.0.0 # via pytest korean-lunar-calendar==0.3.1 # via exchange-calendars (pyproject.toml) mccabe==0.7.0 # via flake8 -numpy==1.24.1 +numpy==1.24.2 # via # exchange-calendars (pyproject.toml) # pandas -packaging==22.0 +packaging==23.0 # via # build # pytest -pandas==1.5.2 +pandas==2.0.0 # via exchange-calendars (pyproject.toml) -pep517==0.13.0 - # via build -pip-tools==6.12.1 +pip-tools==6.13.0 # via exchange-calendars (pyproject.toml) pluggy==1.0.0 # via pytest @@ -55,22 +53,24 @@ pycodestyle==2.10.0 # via flake8 pyflakes==3.0.1 # via flake8 -pyluach==2.0.2 +pyluach==2.2.0 # via exchange-calendars (pyproject.toml) -pytest==7.2.0 +pyproject-hooks==1.0.0 + # via build +pytest==7.2.2 # via # exchange-calendars (pyproject.toml) # pytest-benchmark # pytest-xdist pytest-benchmark==4.0.0 # via exchange-calendars (pyproject.toml) -pytest-xdist==3.1.0 +pytest-xdist==3.2.1 # via exchange-calendars (pyproject.toml) python-dateutil==2.8.2 # via # exchange-calendars (pyproject.toml) # pandas -pytz==2022.7 +pytz==2023.3 # via # exchange-calendars (pyproject.toml) # pandas @@ -81,11 +81,12 @@ sortedcontainers==2.4.0 tomli==2.0.1 # via # build - # pep517 # pytest toolz==0.12.0 # via exchange-calendars (pyproject.toml) -wheel==0.38.4 +tzdata==2023.3 + # via pandas +wheel==0.40.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/etc/requirements_minpandas.txt b/etc/requirements_minpandas.txt index 65b28051..3fd39470 100644 --- a/etc/requirements_minpandas.txt +++ b/etc/requirements_minpandas.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile --extra=dev --output-file=etc/requirements_minpandas.txt pyproject.toml # @@ -12,7 +12,7 @@ attrs==22.2.0 # via # hypothesis # pytest -build==0.9.0 +build==0.10.0 # via pip-tools click==8.1.3 # via pip-tools @@ -21,7 +21,7 @@ colorama==0.4.6 # build # click # pytest -exceptiongroup==1.1.0 +exceptiongroup==1.1.1 # via # hypothesis # pytest @@ -29,27 +29,25 @@ execnet==1.9.0 # via pytest-xdist flake8==6.0.0 # via exchange-calendars (pyproject.toml) -hypothesis==6.61.0 +hypothesis==6.71.0 # via exchange-calendars (pyproject.toml) -iniconfig==1.1.1 +iniconfig==2.0.0 # via pytest korean-lunar-calendar==0.3.1 # via exchange-calendars (pyproject.toml) mccabe==0.7.0 # via flake8 -numpy==1.24.1 +numpy==1.24.2 # via # exchange-calendars (pyproject.toml) # pandas -packaging==22.0 +packaging==23.0 # via # build # pytest pandas==1.1.0 # via exchange-calendars (pyproject.toml) -pep517==0.13.0 - # via build -pip-tools==6.12.1 +pip-tools==6.13.0 # via exchange-calendars (pyproject.toml) pluggy==1.0.0 # via pytest @@ -59,22 +57,24 @@ pycodestyle==2.10.0 # via flake8 pyflakes==3.0.1 # via flake8 -pyluach==2.0.2 +pyluach==2.2.0 # via exchange-calendars (pyproject.toml) -pytest==7.2.0 +pyproject-hooks==1.0.0 + # via build +pytest==7.2.2 # via # exchange-calendars (pyproject.toml) # pytest-benchmark # pytest-xdist pytest-benchmark==4.0.0 # via exchange-calendars (pyproject.toml) -pytest-xdist==3.1.0 +pytest-xdist==3.2.1 # via exchange-calendars (pyproject.toml) python-dateutil==2.8.2 # via # exchange-calendars (pyproject.toml) # pandas -pytz==2022.7 +pytz==2023.3 # via # exchange-calendars (pyproject.toml) # pandas @@ -85,11 +85,10 @@ sortedcontainers==2.4.0 tomli==2.0.1 # via # build - # pep517 # pytest toolz==0.12.0 # via exchange-calendars (pyproject.toml) -wheel==0.38.4 +wheel==0.40.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: