Rewrite gen_release in Python

[jabraham17]

Rewrites gen_release, a perl script, with Python.

This PR makes two behavior changes to the script

• fixes [Bug]: Running gen_release on a Mac results in a tar with lots of warnings #26794, using the technique described in Fix unsupported xattrs when running gen_release on mac #26903
• uses --depth 1 when cloning when CHPL_GEN_RELEASE_COMMIT is not set for faster clone times

This PR makes minimal changes to the flow of logic in the script (mostly just transliterated perl code), but future work could make use of Python features to improve the script. For example, using https://docs.python.org/3/library/tarfile.html instead of spawning sub-processes to rely on the system tar.

Note to reviewer: I recommend reviewing this by having the the python code and perl code open side-by-side, as the github diff is confusing

Tested that all of the following work

• CHPL_GEN_RELEASE_SKIP_DOCS=1 CHPL_GEN_RELEASE_NO_CLONE=1 ./util/buildRelease/gen_release
• CHPL_GEN_RELEASE_SKIP_DOCS=1 CHPL_GEN_RELEASE_NO_CLONE=1 ./util/buildRelease/gen_release 2.4.0
  • check that the output can be built
• ./util/buildRelease/gen_release 2.4.0
  • check that the output can be built

[Reviewed by @DanilaFe]

[jabraham17]

As an implementation note, most of the heavy lifting for this PR was done by gen-ai. I asked it to rewrite the perl into python, and then went through line-by-line, validating the python code and rewriting as needed (for example, to prefer sub-processes with shell=False)

original generated python code

import os
import shutil
import subprocess
import tempfile
from pathlib import Path


def system_or_die(command):
    print(f"+ {command}")
    result = subprocess.run(command, shell=True)
    if result.returncode != 0:
        raise RuntimeError(f"[gen_release] Command failed with error code: {result.returncode}")

def main():
    version = ""
    if len(sys.argv) > 1:
        version = sys.argv[1]

    reldir = f"chapel-{version}" if version else "chapel"
    orig_cwd = Path.cwd().resolve()

    chplhome = os.getenv("CHPL_HOME", Path(__file__).resolve().parent.parent.parent)
    basetmpdir = os.getenv("CHPL_GEN_RELEASE_TMPDIR", tempfile.gettempdir())
    user = os.getlogin()
    tmpdir = tempfile.mkdtemp(prefix=f"chapel-release.{user}.deleteme.", dir=basetmpdir)
    archive_dir = Path(tmpdir) / reldir
    rootdir = Path(tmpdir) / "chpl_home"

    shutil.rmtree(tmpdir, ignore_errors=True)
    archive_dir.mkdir(parents=True, exist_ok=True)

    tar_executable = "gtar" if shutil.which("gtar") else "tar"

    if os.getenv("CHPL_GEN_RELEASE_NO_CLONE"):
        print("[gen_release] CHPL_GEN_RELEASE_NO_CLONE: Creating build workspace with tar...")
        system_or_die(f"cd {chplhome} && {tar_executable} -cf - . | (cd {archive_dir} && {tar_executable} -xf -)")
        resultdir = Path(chplhome) / "tar"
    else:
        git_url = os.getenv("CHPL_HOME_REPOSITORY", "https://github.com/chapel-lang/chapel")
        git_branch = os.getenv("CHPL_GEN_RELEASE_BRANCH", "main")
        repo_cache_path = os.getenv("REPO_CACHE_PATH", "/missing")

        print(f"[gen_release] Cloning the sources (repo: {git_url} branch: {git_branch})...")
        system_or_die(f"git clone --reference-if-able \"{repo_cache_path}/chapel.git\" --branch {git_branch} {git_url} {rootdir}")

        if "CHPL_GEN_RELEASE_COMMIT" in os.environ:
            commit = os.getenv("CHPL_GEN_RELEASE_COMMIT")
            print(f"[gen_release] Checking out revision {commit}...")
            system_or_die(f"cd {rootdir} && git reset --hard {commit}")

        print("[gen_release] Confirm final Git source version used.")
        system_or_die(f"cd {rootdir} && git --no-pager status && git --no-pager log -1")

        system_or_die(f"cd {rootdir} && util/config/write-git-sha frontend/lib/util --build-version --chpl-home={rootdir}")
        print("[gen_release] Creating build workspace with git archive...")
        system_or_die(f"cd {rootdir} && git archive --format=tar HEAD | (cd {archive_dir} && {tar_executable} -xf -)")
        print("[gen_release] Copying BUILD_VERSION file.")
        system_or_die(f"cp {rootdir}/compiler/main/BUILD_VERSION {archive_dir}/compiler/main/BUILD_VERSION")
        print("[gen_release] Copying git-version.cpp")
        system_or_die(f"cp {rootdir}/frontend/lib/util/git-version.cpp {archive_dir}/frontend/lib/util/git-version.cpp")

        resultdir = Path(os.getenv("CHPL_HOME", basetmpdir)) / "tar"

    files = [
        "ACKNOWLEDGEMENTS.md", "CHANGES.md", "CONTRIBUTORS.md", "COPYRIGHT", "LICENSE", "LICENSE.chapel",
        "Makefile", "CMakeLists.txt", "Dockerfile", "README.rst", "README.files", "compiler/codegen/reservedSymbolNames",
        "configure", "highlight/README.md", "util/README", "util/buildRelease/chpl-make-cpu_count", "util/buildRelease/install.sh",
        "util/chpl-completion.bash", "util/printchplenv", "util/setchplenv.bash", "util/setchplenv.csh", "util/setchplenv.fish",
        "util/setchplenv.sh", "util/start_test", "util/chpltags", "frontend/include/chpl/config/config.h.cmake"
    ]

    code_dirs = ["compiler", "frontend"]
    complete_dirs = [
        "compiler/etc", "doc", "examples", "highlight/emacs", "highlight/source-highlight", "highlight/vim", "make",
        "man/man1", "modules", "runtime", "third-party", "util/build_configs", "util/chplenv", "util/config", "util/cmake",
        "util/quickstart", "util/test", "tools/chpldoc", "tools/chplvis", "tools/c2chapel", "tools/mason", "tools/protoc-gen-chpl",
        "tools/unstableWarningAnonymizer/", "tools/chapel-py", "tools/chplcheck", "tools/chpl-language-server"
    ]

    os.chdir(archive_dir)

    print("[gen_release] Creating the spec tests...")
    system_or_die("make spectests")

    print("[gen_release] Building the docs...")
    os.environ['CHPL_HOME'] = str(archive_dir)
    os.environ['CHPL_COMM'] = "none"
    os.environ['CHPL_LLVM'] = "none"
    print(f"[gen_release] CHPL_HOME is set to: {os.environ['CHPL_HOME']}")

    if "CHPL_GEN_RELEASE_SKIP_DOCS" not in os.environ:
        print("[gen_release] Building the html docs...")
        system_or_die("make chapel-py-venv")
        system_or_die("make docs")

        print("[gen_release] Pruning the docs directory...")
        system_or_die("cd doc && make clean-symlinks")
        system_or_die("cd doc && make prunedocs")
        system_or_die("cd doc && rm -f Makefile*")
        system_or_die("cd doc && rm -rf util")
        system_or_die("cd doc && rm -f rst/conf.py")
        system_or_die("cd doc && rm -f rst/index.rst")
        system_or_die("cd doc && rm -f rst/*/index.rst")
        system_or_die("cd doc && rm -rf rst/developer")
        system_or_die("cd doc && rm -rf rst/meta")

        print("[gen_release] Building the man pages...")
        system_or_die("make man")
        system_or_die("make man-chpldoc")
    else:
        system_or_die("mkdir -p man/man1")

    system_or_die("make clobber")

    print("[gen_release] Creating the examples directory...")
    system_or_die("rm examples")
    system_or_die("cp -r test/release/examples .")
    system_or_die("cd util && cp start_test ../examples/")

    print("[gen_release] Removing runtime directories that are not ready for release...")
    system_or_die("cd runtime/src/launch && rm -r dummy")
    system_or_die("cd runtime/src/launch && rm -r mpirun")

    print("[gen_release] Removing frontend test directory which is not intended for release...")
    system_or_die("cd frontend && rm -r test")

    print("[gen_release] Removing third-party directories that are not intended for release...")
    system_or_die("cd third-party && rm *.devel*")
    system_or_die("cd third-party/chpl-venv && rm *.devel*")
    system_or_die("cd third-party/chpl-venv && rm chplspell-requirements.txt")

    print("[gen_release] Removing Git metadata files not intended for release...")
    system_or_die('find . -type f \\( -name .gitignore -o -name .gitattributes \\) -exec rm -f {} \\; -print')

    os.chdir(archive_dir)

    print("[gen_release] Chmodding the hierarchy")
    system_or_die("chmod -R ugo+rX .")
    system_or_die("chmod -R go-w .")

    tarfiles = []
    for file in files:
        if not Path(file).exists():
            print(f"[gen_release] {file} does not exist")
            sys.exit(9)
        tarfiles.append(f"{reldir}/{file}")

    for dir in code_dirs:
        for fullpath in Path(dir).rglob('*'):
            if fullpath.suffix in {'.h', '.cpp', '.c', '.ypp', '.lex'} or fullpath.name in {'Makefile', 'README', 'BUILD_VERSION', 'CMakeLists.txt'}:
                tarfiles.append(f"{reldir}/{fullpath}")

    for dir in complete_dirs:
        if not Path(dir).exists():
            print(f"[gen_release] {dir} does not exist")
        tarfiles.append(f"{reldir}/{dir}")

    if not Path(resultdir).exists():
        print(f"Creating {resultdir}")
        resultdir.mkdir(parents=True, exist_ok=True)

    tarball_name = resultdir / f"{reldir}.tar.gz"
    cmd = f"{tar_executable} -cz -f {tarball_name} --no-xattrs --disable-copyfile {' '.join(tarfiles)}"
    os.chdir("..")
    print(f"[gen_release] {cmd}")
    system_or_die(cmd)

    print(f"[gen_release] Left result in {tarball_name}")
    os.chdir(orig_cwd)

if __name__ == "__main__":
    import sys
    main()

[DanilaFe]

a perf script, with Python.

I assume you mean perl?

[DanilaFe]

considering this a gen_release file (without .py) I highly doubt it can be imported, so. I'm not sure you need to stuff everything into main.

[jabraham17]

probably not, but my personal preference is to have a proper main. It makes large scripts more readable