Begin transitioning to uv

.github/workflows/ci.yml

@@ -25,11 +25,15 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
-      - name: Install tox
-        run: pip install tox
-      - name: Run flake8
-        run: tox -e flake8
-      - name: Run unit tests
-        run: tox -e py3
+      - name: Install pipx
+        run: pip install pipx
+      - name: Install uv
+        run: pipx install uv
+      - name: Run pipx ensurepath
+        run: pipx ensurepath
+      - name: Lint
+        run: uv tool run ruff check
+      - name: Run tests
+        run: uv run --all-extras pytest teuthology scripts
       - name: Run docs build
-        run: tox -e docs
+        run: uv run tox -e docs

bootstrap

@@ -1,159 +1,88 @@
 #!/bin/bash
 set -e
 if [ $# -eq 0 ]; then
-  install=false
+    install=false
 else
-  if [ "$1" = "install" ]; then
-      install=true
-  else
-      echo "Invalid command, supported commands are: 'install'"
-      exit 1
-  fi
-fi
-
-if [[ "$PYTHON" =~ "python2" ]]; then
-    echo "python2 is not supported." >&2
-    exit 1
-fi
-
-# Use the newest version we find
-if [ -z "$PYTHON" ]; then
-  for i in 12 11 10; do
-    command -v "python3.$i" && PYTHON="python3.$i" &>/dev/null && break
-  done
-fi
-if [ -z "$PYTHON" ]; then
-  # This would be bizarre, but I suppose possible
-  PYTHON=${PYTHON:-"python3"}
+    if [ "$1" = "install" ]; then
+        install=true
+    else
+        echo "Invalid command, supported commands are: 'install'"
+        exit 1
+    fi
 fi
-echo "Using python: $PYTHON"
 
 case "$(uname -s)" in
 Linux)
-    if command -v lsb_release; then
-        OS=$(lsb_release --id --short)
-    else
-        . /etc/os-release
-        OS=$(echo $NAME | tr -d ' ')
-    fi
-    # rpm/dnf is the default, to reduce repetition in the case statement
-    has_pkg="rpm -q --whatprovides"
-    install_pkg="sudo dnf install -y"
+    . /etc/os-release
+    OS="${NAME// /}"
     case "$OS" in
     Ubuntu|Debian|LinuxMint)
-        deps=(qemu-utils python3-dev libssl-dev python3-pip python3-wheel python3-venv libev-dev libvirt-dev libffi-dev libyaml-dev build-essential jq curl)
-        has_pkg="dpkg -s"
+        deps=(qemu-utils python3-dev libssl-dev libev-dev libvirt-dev libffi-dev libyaml-dev pipx build-essential jq curl)
+        has_pkg="dpkg -C"
         install_pkg="sudo apt install -y"
         ;;
-    RedHatEnterpriseWorkstation|RedHatEnterpriseServer|RedHatEnterprise|CentOS)
-        deps=(python39-pip python39-devel mariadb-devel libev-devel libvirt-devel libffi-devel)
-        ;;
-    CentOSStream)
-        PYTHON=python3.12
-        deps=($PYTHON-pip $PYTHON-devel)
-        ;;
-    AlmaLinux|RockyLinux)
-        PYTHON=python3.12
-        deps=($PYTHON-pip $PYTHON-devel libev-devel libvirt-devel libffi-devel)
-        ;;
-    Fedora|FedoraLinux)
-        deps=($PYTHON-pip $PYTHON-devel libev-devel libvirt-devel libffi-devel)
+    RedHatEnterpriseWorkstation|RedHatEnterpriseServer|RedHatEnterprise|CentOS|CentOSStream|AlmaLinux|RockyLinux|Fedora|FedoraLinux)
+        # Use the newest version we find
+        if [ -z "$PYTHON" ]; then
+            for i in 13 12 11 10; do
+                command -v "python3.$i" > /dev/null && PYTHON="python3.$i" && break
+            done
+        fi
+        if [ -z "$PYTHON" ]; then
+            # This would be bizarre, but I suppose possible
+            PYTHON=${PYTHON:-"python3"}
+        fi
+        # rpm/dnf is the default, to reduce repetition in the case statement
+        has_pkg="rpm -q --whatprovides"
+        install_pkg="sudo dnf install -y"
+        deps=(libev-devel libffi-devel libvirt-devel "$PYTHON-devel" pipx)
         ;;
     "openSUSE project"|"SUSE LINUX"|"openSUSE"|"openSUSELeap"|"openSUSETumbleweed")
-        PYTHON=python3.12
-        deps=(python312-pip python312-devel python312 libev-devel libvirt-devel libffi-devel)
+        PYTHON=${PYTHON:-"python3"}
+        deps=("$PYTHON" "$PYTHON-devel" "$PYTHON-pipx" libev-devel libffi-devel libvirt-devel)
+        has_pkg="rpm -q --whatprovides"
         install_pkg="sudo zypper install"
         ;;
     esac
     ;;
 
 Darwin)
-    deps="python libvirt libev libffi"
+    deps=(python libvirt libev libffi pipx uv)
     has_pkg="brew list"
     install_pkg="brew install"
     ;;
 esac
-for package in ${deps[@]}; do
-    if ! $has_pkg $package &>/dev/null; then
+for package in "${deps[@]}"; do
+    if ! $has_pkg "$package" &>/dev/null; then
         # add a space after old values
         missing="${missing:+$missing }$package"
-        echo missing=${missing}
     fi
 done
 if [ -n "$missing" ]; then
     echo "$0: missing required packages:" 1>&2
     echo "$missing"
     if [ "$install" = true ]; then
         echo "Installing missing packages..."
-        $install_pkg $missing
+        $install_pkg "$missing"
     else
         echo "Please install missing packages or run './bootstrap install'"
         echo "$install_pkg $missing"
         exit 1
     fi
-    fi
-
-PYTHON_BIN=$(command -v $PYTHON)
-if [ -z $PYTHON_BIN -o ! -e $PYTHON_BIN -o ! -x $PYTHON_BIN ]; then
-  echo "Cannot find $PYTHON!"
-  exit 1
-fi
-PYTHON_VER_OUT=$($PYTHON_BIN --version)
-
-VENV=${VENV:-"./virtualenv"}
-# If the venv was set to use system site-packages, fix that
-if [ -f "$VENV/pyvenv.cfg" ]; then
-    sed -i'' -e 's/\(include-system-site-packages\s*=\s*\)true/\1false/g' $VENV/pyvenv.cfg
 fi
 
 # Attempt to force a UTF-8 locale without being specific to English
 export LANG=${LANG:-C.UTF-8}
-(echo $LANG | grep -qi utf-8) || export LC_ALL=$LANG.UTF-8
-
-if [ -z "$NO_CLOBBER" ] && \
-  [ ! -e "$VENV/bin/pip" -o ! -e "$VENV/bin/$PYTHON" ] || \
-  [ "${PYTHON_VER_OUT}" != "$($VENV/bin/$PYTHON --version)" ] \
-  ; then
-   echo "Deleting existing virtual environment"
-    rm -rf virtualenv
-fi
-
-if [ -z "$NO_CLOBBER" ] || [ ! -e $VENV ]; then
-    echo "Creating new venv at $VENV"
-    $PYTHON_BIN -m venv $VENV
-fi
-
-PY_MAJOR=$($VENV/bin/python -c "import sys; print(sys.version_info[0])")
-PY_MINOR=$($VENV/bin/python -c "import sys; print(sys.version_info[1])")
-
-# Python version check
-if [[ "$PY_MAJOR" -ne 3 || "$PY_MINOR" -lt 10 ]]; then
-    echo "Python version should be 3.10 or higher, found $PY_MAJOR.$PY_MINOR"
-    exit 1
-fi
+(echo "$LANG" | grep -qi utf-8) || export LC_ALL=$LANG.UTF-8
 
-$VENV/bin/pip install packaging
-
-if [ -f "$VENV/bin/ansible" ]; then
-    uninstall_ansible=$($VENV/bin/python3 -c "import ansible; from packaging.version import parse; print(parse(ansible.__version__) < parse('2.10.0'))")
-    if [ "$uninstall_ansible" = "True" ]; then
-        $VENV/bin/pip uninstall -y ansible
-    fi
-fi
-
-# First, upgrade pip
-$VENV/bin/pip install --upgrade pip
-
-# See https://github.com/pypa/pip/issues/8559
-$VENV/bin/pip install -r requirements.txt --use-pep517
-
-# By default, install teuthology in editable mode
-$VENV/bin/pip install ${PIP_INSTALL_FLAGS:---editable '.[test]'}
-
-# Check to make sure requirements are met
-$VENV/bin/pip check
+[ -z "$NO_CLOBBER" ] && rm -rf virtualenv
+command -v uv > /dev/null || pipx install uv
+command -v uv > /dev/null || pipx ensurepath
+PATH=$PATH:$HOME/.local/bin
+# Create the venv if it does not exist, and install teuthology and dependencies
+uv sync --frozen --all-extras
+# To avoid crashing older dispatchers
+ln -sf .venv virtualenv
 
 # Install ansible collections
-$VENV/bin/ansible-galaxy install -r requirements.yml
-
-echo "Bootstrap completed successfully!!!"
+uv run ansible-galaxy install -r requirements.yml

containers/teuthology-dev/Dockerfile

@@ -1,5 +1,6 @@
 FROM ubuntu:noble
 ENV DEBIAN_FRONTEND=noninteractive
+ENV SETUPTOOLS_SCM_PRETEND_VERSION_FOR_TEUTHOLOGY=0.0.0
 ENV LANG=C.UTF-8
 RUN apt-get update && \
     apt-get install -y \
@@ -8,8 +9,7 @@
     python3-dev \
     libssl-dev \
     ipmitool \
-    python3-pip \
-    python3-venv \
+    pipx \
     vim \
     jq \
     curl \
@@ -22,13 +22,13 @@
     apt-get clean all && \
     locale-gen $LC_ALL
 WORKDIR /teuthology
-COPY requirements.txt requirements.yml ansible.cfg bootstrap /teuthology/
+COPY pyproject.toml requirements.txt requirements.yml uv.lock ansible.cfg bootstrap /teuthology/
 RUN \
     cd /teuthology && \
     mkdir ../archive_dir && \
     mkdir log && \
     chmod +x /teuthology/bootstrap && \
-    PIP_INSTALL_FLAGS="-r requirements.txt" ./bootstrap
+    ./bootstrap
 COPY . /teuthology
 RUN \
     (git config -f ./.git/config --unset 'http.https://github.com/.extraheader' || true ) && \
@@ -42,4 +42,4 @@
     chmod 600 $HOME/.ssh/id_rsa && \
     echo "StrictHostKeyChecking=no" > $HOME/.ssh/config && \
     echo "UserKnownHostsFile=/dev/null" >> $HOME/.ssh/config
 ENTRYPOINT /teuthology.sh

containers/teuthology-dev/teuthology.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/bash
 set -e
-source /teuthology/virtualenv/bin/activate
+PATH=$PATH:/teuthology/.venv/bin
 set -x
 cat /run/secrets/id_rsa > $HOME/.ssh/id_rsa
 if [ -n "$TEUTHOLOGY_TESTNODES" ]; then