Store typeshed patches in repo (#17054)

This way, we don't have to do the two step rebase and merge + update
hashes dance. We also get a cleaner commit history

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-exclude: '^(mypyc/external/)|(mypy/typeshed/)'  # Exclude all vendored code from lints
+exclude: '^(mypyc/external/)|(mypy/typeshed/)|misc/typeshed_patches'  # Exclude all vendored code from lints
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.5.0  # must match test-requirements.txt

misc/sync-typeshed.py

@@ -11,6 +11,7 @@
 
 import argparse
 import functools
+import glob
 import os
 import re
 import shutil
@@ -148,58 +149,66 @@ def main() -> None:
         if os.environ.get("GITHUB_TOKEN") is None:
             raise ValueError("GITHUB_TOKEN environment variable must be set")
 
-    branch_name = "mypybot/sync-typeshed"
-    subprocess.run(["git", "checkout", "-B", branch_name, "origin/master"], check=True)
-
-    if not args.typeshed_dir:
-        # Clone typeshed repo if no directory given.
-        with tempfile.TemporaryDirectory() as tempdir:
-            print(f"Cloning typeshed in {tempdir}...")
+    with tempfile.TemporaryDirectory() as tmpdir:
+        # Stash patches before checking out a new branch
+        typeshed_patches = os.path.join("misc", "typeshed_patches")
+        tmp_patches = os.path.join(tmpdir, "typeshed_patches")
+        shutil.copytree(typeshed_patches, tmp_patches)
+
+        branch_name = "mypybot/sync-typeshed"
+        subprocess.run(["git", "checkout", "-B", branch_name, "origin/master"], check=True)
+
+        # Copy the stashed patches back
+        shutil.rmtree(typeshed_patches, ignore_errors=True)
+        shutil.copytree(tmp_patches, typeshed_patches)
+        if subprocess.run(["git", "diff", "--quiet", "--exit-code"], check=False).returncode != 0:
+            subprocess.run(["git", "commit", "-am", "Update typeshed patches"], check=True)
+
+        if not args.typeshed_dir:
+            tmp_typeshed = os.path.join(tmpdir, "typeshed")
+            os.makedirs(tmp_typeshed)
+            # Clone typeshed repo if no directory given.
+            print(f"Cloning typeshed in {tmp_typeshed}...")
             subprocess.run(
-                ["git", "clone", "https://github.com/python/typeshed.git"], check=True, cwd=tempdir
+                ["git", "clone", "https://github.com/python/typeshed.git"],
+                check=True,
+                cwd=tmp_typeshed,
             )
-            repo = os.path.join(tempdir, "typeshed")
+            repo = os.path.join(tmp_typeshed, "typeshed")
             commit = update_typeshed(repo, args.commit)
-    else:
-        commit = update_typeshed(args.typeshed_dir, args.commit)
+        else:
+            commit = update_typeshed(args.typeshed_dir, args.commit)
 
-    assert commit
+        assert commit
 
-    # Create a commit
-    message = textwrap.dedent(
-        f"""\
-        Sync typeshed
+        # Create a commit
+        message = textwrap.dedent(
+            f"""\
+            Sync typeshed
 
-        Source commit:
-        https://github.com/python/typeshed/commit/{commit}
-        """
-    )
-    subprocess.run(["git", "add", "--all", os.path.join("mypy", "typeshed")], check=True)
-    subprocess.run(["git", "commit", "-m", message], check=True)
-    print("Created typeshed sync commit.")
-
-    commits_to_cherry_pick = [
-        "5c00e362d",  # LiteralString reverts
-        "44bc98bd5",  # sum reverts
-        "61a490091",  # ctypes reverts
-    ]
-    for commit in commits_to_cherry_pick:
-        try:
-            subprocess.run(["git", "cherry-pick", commit], check=True)
-        except subprocess.CalledProcessError:
-            if not sys.__stdin__.isatty():
-                # We're in an automated context
-                raise
-
-            # Allow the option to merge manually
-            print(
-                f"Commit {commit} failed to cherry pick."
-                " In a separate shell, please manually merge and continue cherry pick."
-            )
-            rsp = input("Did you finish the cherry pick? [y/N]: ")
-            if rsp.lower() not in {"y", "yes"}:
-                raise
-        print(f"Cherry-picked {commit}.")
+            Source commit:
+            https://github.com/python/typeshed/commit/{commit}
+            """
+        )
+        subprocess.run(["git", "add", "--all", os.path.join("mypy", "typeshed")], check=True)
+        subprocess.run(["git", "commit", "-m", message], check=True)
+        print("Created typeshed sync commit.")
+
+        patches = sorted(glob.glob(os.path.join(typeshed_patches, "*.patch")))
+        for patch in patches:
+            cmd = ["git", "am", "--3way", patch]
+            try:
+                subprocess.run(cmd, check=True)
+            except subprocess.CalledProcessError as e:
+                raise RuntimeError(
+                    f"\n\nFailed to apply patch {patch}\n"
+                    "1. Resolve the conflict, `git add --update`, then run `git am --continue`\n"
+                    "2. Run `git format-patch -1 -o misc/typeshed_patches <new_commit_sha>` "
+                    "to update the patch file.\n"
+                    "3. Re-run sync-typeshed.py"
+                ) from e
+
+            print(f"Applied patch {patch}")
 
     if args.make_pr:
         subprocess.run(["git", "push", "--force", "origin", branch_name], check=True)

misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch

@@ -0,0 +1,182 @@
+From 5c00e362d40aa26e0a22a740f05a52d05edf0f91 Mon Sep 17 00:00:00 2001
+From: Shantanu <[email protected]>
+Date: Mon, 26 Sep 2022 12:55:07 -0700
+Subject: [PATCH] Remove use of LiteralString in builtins (#13743)
+
+---
+ mypy/typeshed/stdlib/builtins.pyi | 88 -------------------------------
+ 1 file changed, 88 deletions(-)
+
+diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi
+index b4765b26c..99919c64c 100644
+--- a/mypy/typeshed/stdlib/builtins.pyi
++++ b/mypy/typeshed/stdlib/builtins.pyi
+@@ -61,7 +61,6 @@ from typing import (  # noqa: Y022
+ from typing_extensions import (  # noqa: Y023
+     Concatenate,
+     Literal,
+-    LiteralString,
+     ParamSpec,
+     Self,
+     TypeAlias,
+@@ -434,31 +433,16 @@ class str(Sequence[str]):
+     def __new__(cls, object: object = ...) -> Self: ...
+     @overload
+     def __new__(cls, object: ReadableBuffer, encoding: str = ..., errors: str = ...) -> Self: ...
+-    @overload
+-    def capitalize(self: LiteralString) -> LiteralString: ...
+-    @overload
+     def capitalize(self) -> str: ...  # type: ignore[misc]
+-    @overload
+-    def casefold(self: LiteralString) -> LiteralString: ...
+-    @overload
+     def casefold(self) -> str: ...  # type: ignore[misc]
+-    @overload
+-    def center(self: LiteralString, width: SupportsIndex, fillchar: LiteralString = " ", /) -> LiteralString: ...
+-    @overload
+     def center(self, width: SupportsIndex, fillchar: str = " ", /) -> str: ...  # type: ignore[misc]
+     def count(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ...
+     def encode(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: ...
+     def endswith(
+         self, suffix: str | tuple[str, ...], start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /
+     ) -> bool: ...
+-    @overload
+-    def expandtabs(self: LiteralString, tabsize: SupportsIndex = 8) -> LiteralString: ...
+-    @overload
+     def expandtabs(self, tabsize: SupportsIndex = 8) -> str: ...  # type: ignore[misc]
+     def find(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ...
+-    @overload
+-    def format(self: LiteralString, *args: LiteralString, **kwargs: LiteralString) -> LiteralString: ...
+-    @overload
+     def format(self, *args: object, **kwargs: object) -> str: ...
+     def format_map(self, map: _FormatMapMapping) -> str: ...
+     def index(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ...
+@@ -474,89 +458,32 @@ class str(Sequence[str]):
+     def isspace(self) -> bool: ...
+     def istitle(self) -> bool: ...
+     def isupper(self) -> bool: ...
+-    @overload
+-    def join(self: LiteralString, iterable: Iterable[LiteralString], /) -> LiteralString: ...
+-    @overload
+     def join(self, iterable: Iterable[str], /) -> str: ...  # type: ignore[misc]
+-    @overload
+-    def ljust(self: LiteralString, width: SupportsIndex, fillchar: LiteralString = " ", /) -> LiteralString: ...
+-    @overload
+     def ljust(self, width: SupportsIndex, fillchar: str = " ", /) -> str: ...  # type: ignore[misc]
+-    @overload
+-    def lower(self: LiteralString) -> LiteralString: ...
+-    @overload
+     def lower(self) -> str: ...  # type: ignore[misc]
+-    @overload
+-    def lstrip(self: LiteralString, chars: LiteralString | None = None, /) -> LiteralString: ...
+-    @overload
+     def lstrip(self, chars: str | None = None, /) -> str: ...  # type: ignore[misc]
+-    @overload
+-    def partition(self: LiteralString, sep: LiteralString, /) -> tuple[LiteralString, LiteralString, LiteralString]: ...
+-    @overload
+     def partition(self, sep: str, /) -> tuple[str, str, str]: ...  # type: ignore[misc]
+-    @overload
+-    def replace(self: LiteralString, old: LiteralString, new: LiteralString, count: SupportsIndex = -1, /) -> LiteralString: ...
+-    @overload
+     def replace(self, old: str, new: str, count: SupportsIndex = -1, /) -> str: ...  # type: ignore[misc]
+     if sys.version_info >= (3, 9):
+-        @overload
+-        def removeprefix(self: LiteralString, prefix: LiteralString, /) -> LiteralString: ...
+-        @overload
+         def removeprefix(self, prefix: str, /) -> str: ...  # type: ignore[misc]
+-        @overload
+-        def removesuffix(self: LiteralString, suffix: LiteralString, /) -> LiteralString: ...
+-        @overload
+         def removesuffix(self, suffix: str, /) -> str: ...  # type: ignore[misc]
+
+     def rfind(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ...
+     def rindex(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ...
+-    @overload
+-    def rjust(self: LiteralString, width: SupportsIndex, fillchar: LiteralString = " ", /) -> LiteralString: ...
+-    @overload
+     def rjust(self, width: SupportsIndex, fillchar: str = " ", /) -> str: ...  # type: ignore[misc]
+-    @overload
+-    def rpartition(self: LiteralString, sep: LiteralString, /) -> tuple[LiteralString, LiteralString, LiteralString]: ...
+-    @overload
+     def rpartition(self, sep: str, /) -> tuple[str, str, str]: ...  # type: ignore[misc]
+-    @overload
+-    def rsplit(self: LiteralString, sep: LiteralString | None = None, maxsplit: SupportsIndex = -1) -> list[LiteralString]: ...
+-    @overload
+     def rsplit(self, sep: str | None = None, maxsplit: SupportsIndex = -1) -> list[str]: ...  # type: ignore[misc]
+-    @overload
+-    def rstrip(self: LiteralString, chars: LiteralString | None = None, /) -> LiteralString: ...
+-    @overload
+     def rstrip(self, chars: str | None = None, /) -> str: ...  # type: ignore[misc]
+-    @overload
+-    def split(self: LiteralString, sep: LiteralString | None = None, maxsplit: SupportsIndex = -1) -> list[LiteralString]: ...
+-    @overload
+     def split(self, sep: str | None = None, maxsplit: SupportsIndex = -1) -> list[str]: ...  # type: ignore[misc]
+-    @overload
+-    def splitlines(self: LiteralString, keepends: bool = False) -> list[LiteralString]: ...
+-    @overload
+     def splitlines(self, keepends: bool = False) -> list[str]: ...  # type: ignore[misc]
+     def startswith(
+         self, prefix: str | tuple[str, ...], start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /
+     ) -> bool: ...
+-    @overload
+-    def strip(self: LiteralString, chars: LiteralString | None = None, /) -> LiteralString: ...
+-    @overload
+     def strip(self, chars: str | None = None, /) -> str: ...  # type: ignore[misc]
+-    @overload
+-    def swapcase(self: LiteralString) -> LiteralString: ...
+-    @overload
+     def swapcase(self) -> str: ...  # type: ignore[misc]
+-    @overload
+-    def title(self: LiteralString) -> LiteralString: ...
+-    @overload
+     def title(self) -> str: ...  # type: ignore[misc]
+     def translate(self, table: _TranslateTable, /) -> str: ...
+-    @overload
+-    def upper(self: LiteralString) -> LiteralString: ...
+-    @overload
+     def upper(self) -> str: ...  # type: ignore[misc]
+-    @overload
+-    def zfill(self: LiteralString, width: SupportsIndex, /) -> LiteralString: ...
+-    @overload
+     def zfill(self, width: SupportsIndex, /) -> str: ...  # type: ignore[misc]
+     @staticmethod
+     @overload
+@@ -567,9 +494,6 @@ class str(Sequence[str]):
+     @staticmethod
+     @overload
+     def maketrans(x: str, y: str, z: str, /) -> dict[int, int | None]: ...
+-    @overload
+-    def __add__(self: LiteralString, value: LiteralString, /) -> LiteralString: ...
+-    @overload
+     def __add__(self, value: str, /) -> str: ...  # type: ignore[misc]
+     # Incompatible with Sequence.__contains__
+     def __contains__(self, key: str, /) -> bool: ...  # type: ignore[override]
+@@ -578,25 +502,13 @@ class str(Sequence[str]):
+     def __getitem__(self, key: SupportsIndex | slice, /) -> str: ...
+     def __gt__(self, value: str, /) -> bool: ...
+     def __hash__(self) -> int: ...
+-    @overload
+-    def __iter__(self: LiteralString) -> Iterator[LiteralString]: ...
+-    @overload
+     def __iter__(self) -> Iterator[str]: ...  # type: ignore[misc]
+     def __le__(self, value: str, /) -> bool: ...
+     def __len__(self) -> int: ...
+     def __lt__(self, value: str, /) -> bool: ...
+-    @overload
+-    def __mod__(self: LiteralString, value: LiteralString | tuple[LiteralString, ...], /) -> LiteralString: ...
+-    @overload
+     def __mod__(self, value: Any, /) -> str: ...
+-    @overload
+-    def __mul__(self: LiteralString, value: SupportsIndex, /) -> LiteralString: ...
+-    @overload
+     def __mul__(self, value: SupportsIndex, /) -> str: ...  # type: ignore[misc]
+     def __ne__(self, value: object, /) -> bool: ...
+-    @overload
+-    def __rmul__(self: LiteralString, value: SupportsIndex, /) -> LiteralString: ...
+-    @overload
+     def __rmul__(self, value: SupportsIndex, /) -> str: ...  # type: ignore[misc]
+     def __getnewargs__(self) -> tuple[str]: ...
+
+-- 
+2.39.3 (Apple Git-146)
+

misc/typeshed_patches/0001-Revert-sum-literal-integer-change-13961.patch

@@ -0,0 +1,36 @@
+From 44bc98bd50e7170887f0740b53ed95a8eb04f00e Mon Sep 17 00:00:00 2001
+From: Shantanu <[email protected]>
+Date: Sat, 29 Oct 2022 12:47:21 -0700
+Subject: [PATCH] Revert sum literal integer change (#13961)
+
+This is allegedly causing large performance problems, see 13821
+
+typeshed/8231 had zero hits on mypy_primer, so it's not the worst thing
+to undo. Patching this in typeshed also feels weird, since there's a
+more general soundness issue. If a typevar has a bound or constraint, we
+might not want to solve it to a Literal.
+
+If we can confirm the performance regression or fix the unsoundness
+within mypy, I might pursue upstreaming this in typeshed.
+
+(Reminder: add this to the sync_typeshed script once merged)
+---
+ mypy/typeshed/stdlib/builtins.pyi | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi
+index 99919c64c..680cd5561 100644
+--- a/mypy/typeshed/stdlib/builtins.pyi
++++ b/mypy/typeshed/stdlib/builtins.pyi
+@@ -1596,7 +1596,7 @@ _SupportsSumNoDefaultT = TypeVar("_SupportsSumNoDefaultT", bound=_SupportsSumWit
+ # without creating many false-positive errors (see #7578).
+ # Instead, we special-case the most common examples of this: bool and literal integers.
+ @overload
+-def sum(iterable: Iterable[bool | _LiteralInteger], /, start: int = 0) -> int: ...  # type: ignore[overload-overlap]
++def sum(iterable: Iterable[bool], /, start: int = 0) -> int: ...  # type: ignore[overload-overlap]
+ @overload
+ def sum(iterable: Iterable[_SupportsSumNoDefaultT], /) -> _SupportsSumNoDefaultT | Literal[0]: ...
+ @overload
+-- 
+2.39.3 (Apple Git-146)
+

misc/typeshed_patches/0001-Revert-typeshed-ctypes-change.patch

@@ -0,0 +1,32 @@
+From 61a490091d7c941780919660dc4fdfa88ae6474a Mon Sep 17 00:00:00 2001
+From: AlexWaygood <[email protected]>
+Date: Mon, 1 May 2023 20:34:55 +0100
+Subject: [PATCH] Revert typeshed ctypes change Since the plugin provides
+ superior type checking:
+ https://github.com/python/mypy/pull/13987#issuecomment-1310863427 A manual
+ cherry-pick of e437cdf.
+
+---
+ mypy/typeshed/stdlib/_ctypes.pyi | 6 +-----
+ 1 file changed, 1 insertion(+), 5 deletions(-)
+
+diff --git a/mypy/typeshed/stdlib/_ctypes.pyi b/mypy/typeshed/stdlib/_ctypes.pyi
+index 60bbc51d9..cf9cb81a4 100644
+--- a/mypy/typeshed/stdlib/_ctypes.pyi
++++ b/mypy/typeshed/stdlib/_ctypes.pyi
+@@ -169,11 +169,7 @@ class Array(_CData, Generic[_CT]):
+     def _type_(self) -> type[_CT]: ...
+     @_type_.setter
+     def _type_(self, value: type[_CT]) -> None: ...
+-    # Note: only available if _CT == c_char
+-    @property
+-    def raw(self) -> bytes: ...
+-    @raw.setter
+-    def raw(self, value: ReadableBuffer) -> None: ...
++    raw: bytes  # Note: only available if _CT == c_char
+     value: Any  # Note: bytes if _CT == c_char, str if _CT == c_wchar, unavailable otherwise
+     # TODO These methods cannot be annotated correctly at the moment.
+     # All of these "Any"s stand for the array's element type, but it's not possible to use _CT
+-- 
+2.39.3 (Apple Git-146)
+