diff --git a/src/scriv/format_rst.py b/src/scriv/format_rst.py
index d52b084..baf194c 100644
--- a/src/scriv/format_rst.py
+++ b/src/scriv/format_rst.py
@@ -145,24 +145,20 @@ def format_sections(
     def convert_to_markdown(
         self, text: str
     ) -> str:  # noqa: D102 (inherited docstring)
-        rst_file = None
-        try:
-            with tempfile.NamedTemporaryFile(
-                mode="w", prefix="scriv_rst_", delete=False
-            ) as rst_file:
-                rst_file.write(text)
-                rst_file.flush()
-                ok, output = run_command(
-                    "pandoc -frst -tmarkdown_strict "
-                    + "--markdown-headings=atx --wrap=none "
-                    + "--fail-if-warnings "
-                    + rst_file.name
+        with tempfile.NamedTemporaryFile(
+            mode="w",
+            prefix="scriv_rst_",
+        ) as rst_file:
+            rst_file.write(text)
+            rst_file.flush()
+            ok, output = run_command(
+                "pandoc -frst -tmarkdown_strict "
+                + "--markdown-headings=atx --wrap=none "
+                + "--fail-if-warnings "
+                + rst_file.name
+            )
+            if not ok:
+                raise ScrivException(
+                    f"Couldn't convert ReST to Markdown: {output!r}"
                 )
-                if not ok:
-                    raise ScrivException(
-                        f"Couldn't convert ReST to Markdown: {output!r}"
-                    )
-                return output.replace("\r\n", "\n")
-        finally:
-            if rst_file is not None:
-                os.unlink(rst_file.name)
+            return output.replace("\r\n", "\n")