U/robj/py312 upgrade

.github/workflows/ci.yaml

@@ -16,13 +16,12 @@ jobs:
       fail-fast: false
       matrix:
         make_target:
-          - itest_bionic
           - itest_jammy
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.12
       - run: |
           sudo apt-get -y install libzookeeper-mt-dev
           pip install tox

dockerfiles/itest/docker-compose-jammy.yml

@@ -1,4 +1,3 @@
-version: '2.4'
 services:
   itest:
     build: ../../dockerfiles/itest/itest_jammy
@@ -8,26 +7,42 @@ services:
     volumes:
      - ../..:/work
      - "${SSH_AUTH_SOCK}:${SSH_AUTH_SOCK}"
-    links:
-     - servicethree
-     - serviceone
-     - scribe
-     - zookeeper
+    depends_on:
+      - servicethree
+      - serviceone
+      - scribe
+      - zookeeper
 
   servicethree:
     # Create dummy service listening on port 1024 that returns /status
     image: python:3-alpine
     command:  /bin/sh -c "echo OK > status && python3 -m http.server 1024"
+    networks:
+      default:
+        aliases:
+          - servicethree_1
 
   serviceone:
     # Create dummy service listening on port 1025
     image: alpine/socat:latest
     command: TCP4-LISTEN:1025,fork /dev/null
+    networks:
+      default:
+        aliases:
+          - serviceone_1
 
   scribe:
     # Create dummy scribe listening on port 1464
     image: alpine/socat:latest
     command: TCP4-LISTEN:1464,fork /dev/null
+    networks:
+      default:
+        aliases:
+          - scribe_1
 
   zookeeper:
     image: zookeeper:3.5
+    networks:
+      default:
+        aliases:
+          - zookeeper_1

dockerfiles/itest/itest/Dockerfile.jammy

@@ -1,30 +1,40 @@
 FROM ubuntu:jammy
 
+ARG DEBIAN_FRONTEND=noninteractive
+ENV DEBIAN_FRONTEND=${DEBIAN_FRONTEND}
+ENV TZ=Etc/UTC
+
 ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple
 ENV PIP_INDEX_URL=$PIP_INDEX_URL
 
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends gnupg2 software-properties-common && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gnupg2 software-properties-common && \
     add-apt-repository ppa:deadsnakes/ppa
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install \
+    build-essential \
     python3-kazoo \
     python3-pytest \
     python3-setuptools \
     socat \
     ruby ruby-dev ruby-bundler zlib1g-dev \
     libcurl4 \
-    python3.7 \
-    python3.7-dev \
-    python3.7-distutils \
-    python3-pip \
+    libffi-dev \
+    python3.11 \
+    python3.11-dev \
+    python3.11-venv \
+    python3.12 \
+    python3.12-dev \
+    python3.12-venv \
     python3-zope.interface \
     git \
     libyaml-dev \
     libc6-dev \
     libstdc++6 \
     wget
 
+RUN python3.11 -m ensurepip --upgrade
+
 # Install nerve
 ADD nerve_Gemfile Gemfile
 RUN mkdir -p /opt/nerve

dockerfiles/itest/itest/itest.py

@@ -41,7 +41,7 @@
 ]
 
 
-@pytest.yield_fixture(scope='module')
+@pytest.fixture(scope='module')
 def setup():
     # Forward healthchecks to the services
     socat_procs = []
@@ -140,7 +140,7 @@ def test_nerve_service_config(setup):
         ],
         "host": MY_IP_ADDRESS,
         "port": 1024,
-        "weight": CPUS,
+        "weight": 10,
         "zk_hosts": [ZOOKEEPER_CONNECT_STRING],
         "zk_path": "/smartstack/global/service_three.main",
         'labels': {

dockerfiles/itest/itest/run_itest.sh

@@ -3,11 +3,14 @@ set -vx
 set -e
 
 echo "installing hacheck"
-GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@github.yelpcorp.com:packages/hacheck && cd /hacheck && python3.7 -m pip install . && cp /usr/local/bin/ha* /usr/bin/
+GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone --depth=1 git@github.yelpcorp.com:packages/hacheck /hacheck
+cd /hacheck
+python3.11 -m pip install .
+cp /usr/local/bin/ha* /usr/bin/
 
 echo "installing paasta-tools (dependency of nerve-tools.)"
 . /etc/lsb-release
-PAASTA_VERSION=0.145.0
+PAASTA_VERSION=1.47.0
 PAASTA_DEB_NAME=paasta-tools_${PAASTA_VERSION}.${DISTRIB_CODENAME}1_amd64.deb
 wget "https://github.com/Yelp/paasta/releases/download/v${PAASTA_VERSION}/${PAASTA_DEB_NAME}"
 
@@ -25,4 +28,4 @@ echo "Testing that pyyaml uses optimized cyaml parsers if present"
 /opt/venvs/nerve-tools/bin/python -c 'import yaml; assert yaml.__with_libyaml__'
 
 echo "Full integration test"
-py.test-3 -vvv /itest.py
+python3 -m pytest -vvv /itest.py

dockerfiles/jammy/Dockerfile

@@ -1,14 +1,18 @@
 FROM ubuntu:jammy
 
+ARG DEBIAN_FRONTEND=noninteractive
+ENV DEBIAN_FRONTEND=${DEBIAN_FRONTEND}
+ENV TZ=Etc/UTC
+
 ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple
 ENV PIP_INDEX_URL=$PIP_INDEX_URL
 
-# Need Python 3.7 for nerve-tools
+# Need Python 3.12 for nerve-tools
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends gnupg2 software-properties-common && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gnupg2 software-properties-common && \
     add-apt-repository ppa:deadsnakes/ppa
 
-RUN apt-get update && apt-get -y install \
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install \
     build-essential \
     cargo \
     curl \
@@ -22,17 +26,17 @@ RUN apt-get update && apt-get -y install \
     libyaml-dev \
     libzookeeper-mt-dev \
     protobuf-compiler \
-    python3.7-dev \
-    python3.7-distutils \
+    python3.12 \
+    python3.12-dev \
+    python3.12-venv \
     rustc \
     wget
 
-# Preferred method to install pip when not bundled with python - https://pip.pypa.io/en/stable/installing/
-RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-RUN python3.7 get-pip.py
-RUN pip install -U tox pip virtualenv setuptools
+RUN python3.12 -m ensurepip --upgrade
+ADD src/requirements-bootstrap.txt /tmp/requirements-bootstrap.txt
+RUN python3.12 -m pip install -U -r /tmp/requirements-bootstrap.txt tox virtualenv
 
-ADD location_types.json /nail/etc/services/
-ADD location_mapping.json /nail/etc/services/
+ADD dockerfiles/jammy/location_types.json /nail/etc/services/
+ADD dockerfiles/jammy/location_mapping.json /nail/etc/services/
 
 WORKDIR /work

dockerfiles/jammy/docker-compose.yml

@@ -1,8 +1,9 @@
-version: '2.4'
 services:
   jammy:
-    build: ../../dockerfiles/jammy
-    command: bash -c "cp -R /code/* /work && cd /work/src && tox -e py37-linux && dpkg-buildpackage -d -uc -us && mv ../*.deb /dist"
+    build:
+      context: ../..
+      dockerfile: ./dockerfiles/jammy/Dockerfile
+    command: bash -c "cp -R /code/* /work && cd /work/src && tox -e py312-linux && dpkg-buildpackage -d -uc -us && mv ../*.deb /dist"
     volumes:
       - ../..:/code:ro
       - ../../dist:/dist

src/debian/compat

@@ -1 +1 @@
-9
+10

src/debian/control

@@ -1,9 +1,13 @@
 Source: nerve-tools
+Section: admin
+Priority: optional
 Maintainer: John Billings <billings@yelp.com>
 Build-Depends:
     dh-virtualenv,
 
-Depends: python3.7, paasta-tools (>= 0.145.0~), ${shlibs:Depends}
 Package: nerve-tools
+Section: admin
+Priority: optional
 Architecture: any
+Depends: python3.12, paasta-tools (>= 1.47.0~), ${shlibs:Depends}
 Description: Nerve-related tools for use on Yelp machines.

src/debian/rules

@@ -24,9 +24,6 @@ override_dh_auto_test:
 # TravisCI can't handle the large logs that verbose output creates and it will just terminate the build :(
 override_dh_virtualenv:
 	dh_virtualenv -i $(PIP_INDEX_URL) \
-	--python=/usr/bin/python3.7 \
+	--python=/usr/bin/python3.12 \
 	--preinstall no-manylinux1
 
-# For some reason dh virtualenv can't find libcrypto bundled but this seems to work. Probably there is a real fix somewhere?
-override_dh_shlibdeps:
-	dh_shlibdeps --dpkg-shlibdeps-params=-ldebian/nerve-tools/opt/venvs/nerve-tools/lib/python3.7/site-packages/cryptography.libs

src/mypy.ini

@@ -1,5 +1,5 @@
 [mypy]
-python_version = 3.6
+python_version = 3.12
 ignore_missing_imports = True
 check_untyped_defs = True
 disallow_untyped_defs = True

src/nerve_tools/config.py

@@ -1,9 +1,9 @@
-from mypy_extensions import TypedDict
-from typing import Mapping
-from typing import Iterable
 from typing import Dict
-from typing import Tuple
+from typing import Iterable
+from typing import Mapping
 from typing import Optional
+from typing import Tuple
+from typing import TypedDict
 
 
 class CheckDict(TypedDict, total=False):

src/nerve_tools/updown_service.py

@@ -12,7 +12,7 @@
 import requests
 from requests.exceptions import RequestException
 
-from paasta_tools.marathon_tools import load_service_namespace_config
+from paasta_tools.long_running_service_tools import load_service_namespace_config
 from service_configuration_lib import read_service_configuration
 
 

src/requirements-bootstrap.txt

@@ -1,3 +1,3 @@
-pip==21.0.1
-setuptools==54.1.2
-wheel==0.36.2
+pip==25.2
+setuptools==80.10.1
+wheel==0.46.2

src/requirements-dev.txt

@@ -1,4 +1,4 @@
-coverage==4.5.4
-flake8==3.7.9
-mock==3.0.5
-pytest==5.2.1
+coverage==6.5.0
+flake8==7.3.0
+mypy==1.17.1
+pytest==7.4.3

src/requirements.txt

@@ -1,12 +1,6 @@
-argparse==1.2.1
-cryptography==35.0.0
-dulwich==0.17.3
 environment_tools==1.1.3
 kazoo==2.8.0
-ldap3==2.7
-mypy-extensions==0.4.3
-paasta-tools==0.105.0
+paasta-tools==1.47.0
 PyYAML==6.0.1
-requests==2.23.0
-service-configuration-lib==2.5.4
-setuptools==42.0.2
+requests==2.32.5
+service-configuration-lib==3.3.8

src/setup.py

@@ -1,15 +1,21 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-from pkg_resources import yield_lines
-from setuptools import setup, find_packages
+from pathlib import Path
 
+from setuptools import find_packages
+from setuptools import setup
 
-def get_install_requires():
-    with open('requirements.txt', 'r') as f:
-        minimal_reqs = list(yield_lines(f.read()))
 
-    return minimal_reqs
+HERE = Path(__file__).parent
+
+
+def get_install_requires():
+    return [
+        line.strip()
+        for line in (HERE / 'requirements.txt').read_text().splitlines()
+        if line.strip() and not line.startswith('#')
+    ]
 
 
 setup(
@@ -20,7 +26,6 @@ def get_install_requires():
     author_email='compute-infra@yelp.com',
     description='Nerve-related tools for use on Yelp machines',
     packages=find_packages(exclude=['tests']),
-    setup_requires=['setuptools'],
     include_package_data=True,
     install_requires=get_install_requires(),
     entry_points={

src/tests/clean_nerve_test.py

@@ -1,6 +1,6 @@
 from nerve_tools import clean_nerve
 
-import mock
+from unittest import mock
 
 
 def test_parse_args():

src/tests/configure_nerve_test.py

@@ -1,10 +1,10 @@
 import copy
-import mock
-from mock import call
-from mock import patch
-from mock import mock_open
-from mock import MagicMock
-from mock import Mock
+from unittest import mock
+from unittest.mock import call
+from unittest.mock import MagicMock
+from unittest.mock import mock_open
+from unittest.mock import Mock
+from unittest.mock import patch
 import pytest
 import sys
 import multiprocessing

src/tests/envoy_test.py

@@ -1,4 +1,4 @@
-from mock import patch
+from unittest.mock import patch
 
 from nerve_tools.envoy import get_envoy_ingress_listeners
 

src/tests/updown_service_test.py

@@ -1,4 +1,4 @@
-import mock
+from unittest import mock
 import pytest
 from requests.exceptions import RequestException
 import yaml