[http-client-python] Preserve custom fields when migrating from setup.py to pyproject.toml

.chronus/changes/copilot-fix-693fde59-8e8e-4f46-ac31-090e6f2731c7-2025-9-6-23-58-35.md

@@ -0,0 +1,8 @@
+---
+# Change versionKind to one of: internal, fix, dependencies, feature, deprecation, breaking
+changeKind: fix
+packages:
+  - "@typespec/http-client-python"
+---
+
+[http-client-python] Preserve custom fields when migrating from setup.py to pyproject.toml

packages/http-client-python/generator/pygen/codegen/serializers/__init__.py

@@ -139,14 +139,15 @@ def serialize(self) -> None:
                 if self.code_model.options["basic-setup-py"]:
                     # Write the setup file
                     self.write_file(generation_path / Path("setup.py"), general_serializer.serialize_setup_file())
-                elif not self.code_model.options["keep-setup-py"]:
-                    # remove setup.py file
-                    self.remove_file(generation_path / Path("setup.py"))
 
                 # add packaging files in root namespace (e.g. setup.py, README.md, etc.)
                 if self.code_model.options.get("package-mode"):
                     self._serialize_and_write_package_files()
 
+                if not self.code_model.options["basic-setup-py"] and not self.code_model.options["keep-setup-py"]:
+                    # remove setup.py file after reading it for migration purposes
+                    self.remove_file(generation_path / Path("setup.py"))
+
                 # write apiview-properties.json
                 if self.code_model.options.get("emit-cross-language-definition-file"):
                     self.write_file(
@@ -255,10 +256,16 @@ def _serialize_and_write_package_files(self) -> None:
                 if self.keep_version_file and file == "setup.py" and not self.code_model.options["azure-arm"]:
                     # don't regenerate setup.py file if the version file is more up to date for data-plane
                     continue
-                file_content = self.read_file(output_file) if file == "pyproject.toml" else ""
+                # For pyproject.toml, read both existing pyproject.toml and setup.py for migration
+                if file == "pyproject.toml":
+                    file_content = self.read_file(output_file)
+                    setuppy_file_content = self.read_file(root_of_sdk / "setup.py")
+                else:
+                    file_content = ""
+                    setuppy_file_content = ""
                 self.write_file(
                     output_file,
-                    serializer.serialize_package_file(template_name, file_content, **params),
+                    serializer.serialize_package_file(template_name, file_content, setuppy_file_content, **params),
                 )
 
     def _keep_patch_file(self, path_file: Path, env: Environment):

packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py

@@ -4,6 +4,7 @@
 # license information.
 # --------------------------------------------------------------------------
 import json
+import logging
 from typing import Any
 import re
 import tomli as tomllib
@@ -19,6 +20,8 @@
 from .client_serializer import ClientSerializer, ConfigSerializer
 from .base_serializer import BaseSerializer
 
+_LOGGER = logging.getLogger(__name__)
+
 VERSION_MAP = {
     "msrest": "0.7.1",
     "isodate": "0.6.1",
@@ -57,20 +60,20 @@ def _extract_min_dependency(self, s):
         m = re.search(r"[>=]=?([\d.]+(?:[a-z]+\d+)?)", s)
         return parse_version(m.group(1)) if m else parse_version("0")
 
-    def _keep_pyproject_fields(self, file_content: str) -> dict:
+    def _keep_pyproject_fields(self, file_content: str, params: dict) -> None:
         # Load the pyproject.toml file if it exists and extract fields to keep.
-        result: dict = {"KEEP_FIELDS": {}}
+        # Mutates params in place.
         try:
             loaded_pyproject_toml = tomllib.loads(file_content)
         except Exception:  # pylint: disable=broad-except
             # If parsing the pyproject.toml fails, we assume the it does not exist or is incorrectly formatted.
-            return result
+            return
 
         # Keep "azure-sdk-build" and "packaging" configuration
         if "tool" in loaded_pyproject_toml and "azure-sdk-build" in loaded_pyproject_toml["tool"]:
-            result["KEEP_FIELDS"]["tool.azure-sdk-build"] = loaded_pyproject_toml["tool"]["azure-sdk-build"]
+            params["KEEP_FIELDS"]["tool.azure-sdk-build"] = loaded_pyproject_toml["tool"]["azure-sdk-build"]
         if "packaging" in loaded_pyproject_toml:
-            result["KEEP_FIELDS"]["packaging"] = loaded_pyproject_toml["packaging"]
+            params["KEEP_FIELDS"]["packaging"] = loaded_pyproject_toml["packaging"]
 
         # Process dependencies
         if "project" in loaded_pyproject_toml:
@@ -94,22 +97,120 @@ def _keep_pyproject_fields(self, file_content: str) -> dict:
                         kept_deps.append(dep)
 
                 if kept_deps:
-                    result["KEEP_FIELDS"]["project.dependencies"] = kept_deps
+                    params["KEEP_FIELDS"]["project.dependencies"] = kept_deps
 
             # Keep optional dependencies
             if "optional-dependencies" in loaded_pyproject_toml["project"]:
-                result["KEEP_FIELDS"]["project.optional-dependencies"] = loaded_pyproject_toml["project"][
+                params["KEEP_FIELDS"]["project.optional-dependencies"] = loaded_pyproject_toml["project"][
                     "optional-dependencies"
                 ]
+
+            # Check for existing keywords and add to the set
+            if "keywords" in loaded_pyproject_toml["project"]:
+                existing_keywords = loaded_pyproject_toml["project"]["keywords"]
+                if existing_keywords:
+                    params["KEEP_FIELDS"]["project.keywords"].update(existing_keywords)
+
+            # Keep project URLs
+            if "urls" in loaded_pyproject_toml["project"]:
+                if "project.urls" not in params["KEEP_FIELDS"]:
+                    params["KEEP_FIELDS"]["project.urls"] = {}
+                params["KEEP_FIELDS"]["project.urls"].update(loaded_pyproject_toml["project"]["urls"])
 
-        return result
+    def _keep_setuppy_fields(self, setuppy_content: str, params: dict) -> None:
+        """Parse setup.py file to extract fields that should be kept when migrating to pyproject.toml.
+        Mutates params in place."""
+
+        _LOGGER.info("Keeping the following fields from setup.py when generating pyproject.toml.")
+
+        # Extract install_requires (dependencies)
+        install_requires_match = re.search(r'install_requires\s*=\s*\[(.*?)\]', setuppy_content, re.DOTALL)
+        if install_requires_match:
+            deps_str = install_requires_match.group(1)
+            # Parse the dependencies list
+            deps = []
+            for line in deps_str.split('\n'):
+                line = line.strip()
+                if line and not line.startswith('#'):
+                    # Remove quotes and trailing comma
+                    dep = line.strip(',').strip().strip('"').strip("'")
+                    if dep:
+                        # Check if this is a tracked dependency
+                        dep_name = re.split(r"[<>=\[]", dep)[0].strip()
+                        if dep_name not in VERSION_MAP:
+                            # Keep non-default dependencies
+                            deps.append(dep)
+                            _LOGGER.info(f"Keeping field dependency: {dep}")
+                        else:
+                            # For tracked dependencies, check if version is higher than default
+                            default_version = parse_version(VERSION_MAP[dep_name])
+                            dep_version = self._extract_min_dependency(dep)
+                            if dep_version > default_version:
+                                VERSION_MAP[dep_name] = str(dep_version)
+                                _LOGGER.info(f"Keeping field dependency: {dep}")
+
+            if deps:
+                if "project.dependencies" not in params["KEEP_FIELDS"]:
+                    params["KEEP_FIELDS"]["project.dependencies"] = []
+                params["KEEP_FIELDS"]["project.dependencies"].extend(deps)
+
+        # Extract project_urls
+        project_urls_match = re.search(r'project_urls\s*=\s*\{(.*?)\}', setuppy_content, re.DOTALL)
+        if project_urls_match:
+            urls_str = project_urls_match.group(1)
+            # Parse the project_urls dict
+            for line in urls_str.split('\n'):
+                line = line.strip()
+                if line and ':' in line:
+                    # Parse "key": "value" format
+                    key_val_match = re.search(r'["\']([^"\']+)["\']\s*:\s*["\']([^"\']+)["\']', line)
+                    if key_val_match:
+                        key = key_val_match.group(1)
+                        value = key_val_match.group(2)
+                        # Keep all URLs (even default Azure SDK URLs)
+                        if "project.urls" not in params["KEEP_FIELDS"]:
+                            params["KEEP_FIELDS"]["project.urls"] = {}
+                        # Add quotes around multi-word keys for TOML compatibility
+                        formatted_key = f'"{key}"' if ' ' in key else key
+                        params["KEEP_FIELDS"]["project.urls"][formatted_key] = value
+                        _LOGGER.info(f"Keeping field project.urls.{key}: {value}")
+
+        # Extract keywords
+        keywords_match = re.search(r'keywords\s*=\s*["\']([^"\']+)["\']', setuppy_content)
+        if keywords_match:
+            keywords_str = keywords_match.group(1)
+            # Parse the keywords (comma-separated)
+            keywords = [kw.strip() for kw in keywords_str.split(',')]
+            # Add keywords to the existing set (no filtering)
+            params["KEEP_FIELDS"]["project.keywords"].update(keywords)
+            _LOGGER.info(f"Keeping field project.keywords: {keywords}")
+
+        # Check PACKAGE_PPRINT_NAME and warn if different
+        pprint_match = re.search(r'PACKAGE_PPRINT_NAME\s*=\s*["\']([^"\']+)["\']', setuppy_content)
+        if pprint_match:
+            existing_pprint_name = pprint_match.group(1)
+            generated_pprint_name = self.code_model.options.get("package-pprint-name", "")
+            if existing_pprint_name != generated_pprint_name:
+                _LOGGER.warning(
+                    f"Generated package-pprint-name '{generated_pprint_name}' does not match existing "
+                    f"PACKAGE_PPRINT_NAME '{existing_pprint_name}'. Ensure the new package-pprint-name is correct, "
+                    f"otherwise change this value in the tspconfig.yaml."
+                )
 
-    def serialize_package_file(self, template_name: str, file_content: str, **kwargs: Any) -> str:
+    def serialize_package_file(self, template_name: str, file_content: str, setuppy_file_content: str = "", **kwargs: Any) -> str:
         template = self.env.get_template(template_name)
 
         # Add fields to keep from an existing pyproject.toml
         if template_name == "pyproject.toml.jinja2":
-            params = self._keep_pyproject_fields(file_content)
+            # Initialize params with default keywords
+            params: dict = {"KEEP_FIELDS": {"project.keywords": {"azure", "azure sdk"}}}
+
+            # Mutate params with fields from pyproject.toml
+            self._keep_pyproject_fields(file_content, params)
+
+            # If setup.py exists, mutate params with fields from it
+            if setuppy_file_content:
+                self._keep_setuppy_fields(setuppy_file_content, params)
         else:
             params = {}
 

packages/http-client-python/generator/pygen/codegen/templates/packaging_templates/README.md.jinja2

@@ -117,7 +117,7 @@ This package is generated by `@typespec/http-client-python` with Typespec.
 
 ### Install the package
 
-Step into folder where setup.py is then run:
+Step into folder where pyproject.toml is then run:
 
 ```bash
 pip install -e .

packages/http-client-python/generator/pygen/codegen/templates/packaging_templates/pyproject.toml.jinja2

@@ -33,8 +33,8 @@ requires-python = ">={{ MIN_PYTHON_VERSION }}"
 {% else %}
 description = "{{ options.get('package-name') }}"
 {% endif %}
-{% if code_model.is_azure_flavor %}
-keywords = ["azure", "azure sdk"]
+{% if KEEP_FIELDS and KEEP_FIELDS.get('project.keywords') %}
+keywords = [{% for kw in KEEP_FIELDS.get('project.keywords') | sort %}"{{ kw }}"{% if not loop.last %}, {% endif %}{% endfor %}]
 {% endif %}
 
 dependencies = [
@@ -74,11 +74,16 @@ version = "{{ options.get("package-version", "unknown") }}"
 ]
 {% endfor %}
 {% endif %}
-{% if code_model.is_azure_flavor %}
 
 [project.urls]
+{% if code_model.is_azure_flavor %}
 repository = "https://github.com/Azure/azure-sdk-for-python"
 {% endif %}
+{% if KEEP_FIELDS and KEEP_FIELDS.get('project.urls') %}
+{% for key, val in KEEP_FIELDS.get('project.urls').items() %}
+{{ key }} = "{{ val }}"
+{% endfor %}
+{% endif %}
 
 [tool.setuptools.dynamic]
 {% if options.get('package-mode') %}