diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 99f1cbe..b92cd6f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,22 +8,22 @@ on: release: jobs: - tox: + tests: env: PIP_INDEX_URL: https://pypi.python.org/simple - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: - make_target: - - itest_bionic - - itest_jammy + tox_env: + - tests + - mypy steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: 3.7 + python-version: 3.12 - run: | sudo apt-get -y install libzookeeper-mt-dev pip install tox - - run: make ${{ matrix.make_target }} + - run: cd src && tox -e ${{ matrix.tox_env }} diff --git a/dockerfiles/itest/docker-compose-jammy.yml b/dockerfiles/itest/docker-compose-jammy.yml index ed809c1..5e17459 100644 --- a/dockerfiles/itest/docker-compose-jammy.yml +++ b/dockerfiles/itest/docker-compose-jammy.yml @@ -1,4 +1,3 @@ -version: '2.4' services: itest: build: ../../dockerfiles/itest/itest_jammy @@ -7,27 +6,43 @@ services: hostname: itesthost.itestdomain volumes: - ../..:/work - - "${SSH_AUTH_SOCK}:${SSH_AUTH_SOCK}" - links: - - servicethree - - serviceone - - scribe - - zookeeper + - "${SSH_AUTH_SOCK:-/dev/null}:${SSH_AUTH_SOCK:-/dev/null}" + depends_on: + - servicethree + - serviceone + - scribe + - zookeeper servicethree: # Create dummy service listening on port 1024 that returns /status image: python:3-alpine command: /bin/sh -c "echo OK > status && python3 -m http.server 1024" + networks: + default: + aliases: + - servicethree_1 serviceone: # Create dummy service listening on port 1025 image: alpine/socat:latest command: TCP4-LISTEN:1025,fork /dev/null + networks: + default: + aliases: + - serviceone_1 scribe: # Create dummy scribe listening on port 1464 image: alpine/socat:latest command: TCP4-LISTEN:1464,fork /dev/null + networks: + default: + aliases: + - scribe_1 zookeeper: image: zookeeper:3.5 + networks: + default: + aliases: + - zookeeper_1 diff --git a/dockerfiles/itest/itest/Dockerfile.jammy b/dockerfiles/itest/itest/Dockerfile.jammy index 381944a..2778d14 100644 --- a/dockerfiles/itest/itest/Dockerfile.jammy +++ b/dockerfiles/itest/itest/Dockerfile.jammy @@ -1,5 +1,8 @@ FROM ubuntu:jammy +ENV DEBIAN_FRONTEND=noninteractive +ENV TZ=Etc/UTC + ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple ENV PIP_INDEX_URL=$PIP_INDEX_URL @@ -7,17 +10,21 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends gnupg2 software-properties-common && \ add-apt-repository ppa:deadsnakes/ppa -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install \ +RUN apt-get update && apt-get -y install \ + build-essential \ python3-kazoo \ python3-pytest \ python3-setuptools \ socat \ ruby ruby-dev ruby-bundler zlib1g-dev \ libcurl4 \ - python3.7 \ - python3.7-dev \ - python3.7-distutils \ - python3-pip \ + libffi-dev \ + python3.11 \ + python3.11-dev \ + python3.11-venv \ + python3.12 \ + python3.12-dev \ + python3.12-venv \ python3-zope.interface \ git \ libyaml-dev \ @@ -25,6 +32,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install \ libstdc++6 \ wget +RUN python3.11 -m ensurepip --upgrade + # Install nerve ADD nerve_Gemfile Gemfile RUN mkdir -p /opt/nerve diff --git a/dockerfiles/itest/itest/itest.py b/dockerfiles/itest/itest/itest.py index 0acd6f8..5d30b49 100644 --- a/dockerfiles/itest/itest/itest.py +++ b/dockerfiles/itest/itest/itest.py @@ -41,7 +41,7 @@ ] -@pytest.yield_fixture(scope='module') +@pytest.fixture(scope='module') def setup(): # Forward healthchecks to the services socat_procs = [] @@ -140,7 +140,7 @@ def test_nerve_service_config(setup): ], "host": MY_IP_ADDRESS, "port": 1024, - "weight": CPUS, + "weight": 10, "zk_hosts": [ZOOKEEPER_CONNECT_STRING], "zk_path": "/smartstack/global/service_three.main", 'labels': { diff --git a/dockerfiles/itest/itest/run_itest.sh b/dockerfiles/itest/itest/run_itest.sh index 22b352e..bea8a40 100644 --- a/dockerfiles/itest/itest/run_itest.sh +++ b/dockerfiles/itest/itest/run_itest.sh @@ -3,11 +3,14 @@ set -vx set -e echo "installing hacheck" -GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@github.yelpcorp.com:packages/hacheck && cd /hacheck && python3.7 -m pip install . && cp /usr/local/bin/ha* /usr/bin/ +GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone --depth=1 git@github.yelpcorp.com:packages/hacheck /hacheck +cd /hacheck +python3.11 -m pip install . +cp /usr/local/bin/ha* /usr/bin/ echo "installing paasta-tools (dependency of nerve-tools.)" . /etc/lsb-release -PAASTA_VERSION=0.145.0 +PAASTA_VERSION=1.47.0 PAASTA_DEB_NAME=paasta-tools_${PAASTA_VERSION}.${DISTRIB_CODENAME}1_amd64.deb wget "https://github.com/Yelp/paasta/releases/download/v${PAASTA_VERSION}/${PAASTA_DEB_NAME}" @@ -25,4 +28,4 @@ echo "Testing that pyyaml uses optimized cyaml parsers if present" /opt/venvs/nerve-tools/bin/python -c 'import yaml; assert yaml.__with_libyaml__' echo "Full integration test" -py.test-3 -vvv /itest.py +python3 -m pytest -vvv /itest.py diff --git a/dockerfiles/jammy/Dockerfile b/dockerfiles/jammy/Dockerfile index d81633d..a436da2 100644 --- a/dockerfiles/jammy/Dockerfile +++ b/dockerfiles/jammy/Dockerfile @@ -1,9 +1,12 @@ FROM ubuntu:jammy +ENV DEBIAN_FRONTEND=noninteractive +ENV TZ=Etc/UTC + ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple ENV PIP_INDEX_URL=$PIP_INDEX_URL -# Need Python 3.7 for nerve-tools +# Need Python 3.12 for nerve-tools RUN apt-get update && \ apt-get install -y --no-install-recommends gnupg2 software-properties-common && \ add-apt-repository ppa:deadsnakes/ppa @@ -22,17 +25,17 @@ RUN apt-get update && apt-get -y install \ libyaml-dev \ libzookeeper-mt-dev \ protobuf-compiler \ - python3.7-dev \ - python3.7-distutils \ + python3.12 \ + python3.12-dev \ + python3.12-venv \ rustc \ wget -# Preferred method to install pip when not bundled with python - https://pip.pypa.io/en/stable/installing/ -RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py -RUN python3.7 get-pip.py -RUN pip install -U tox pip virtualenv setuptools +RUN python3.12 -m ensurepip --upgrade +ADD src/requirements-bootstrap.txt /tmp/requirements-bootstrap.txt +RUN python3.12 -m pip install -U -r /tmp/requirements-bootstrap.txt tox virtualenv -ADD location_types.json /nail/etc/services/ -ADD location_mapping.json /nail/etc/services/ +ADD dockerfiles/jammy/location_types.json /nail/etc/services/ +ADD dockerfiles/jammy/location_mapping.json /nail/etc/services/ WORKDIR /work diff --git a/dockerfiles/jammy/docker-compose.yml b/dockerfiles/jammy/docker-compose.yml index 3d47682..352e9e0 100644 --- a/dockerfiles/jammy/docker-compose.yml +++ b/dockerfiles/jammy/docker-compose.yml @@ -1,8 +1,9 @@ -version: '2.4' services: jammy: - build: ../../dockerfiles/jammy - command: bash -c "cp -R /code/* /work && cd /work/src && tox -e py37-linux && dpkg-buildpackage -d -uc -us && mv ../*.deb /dist" + build: + context: ../.. + dockerfile: ./dockerfiles/jammy/Dockerfile + command: bash -c "cp -R /code/* /work && cd /work/src && tox -e py312-linux && dpkg-buildpackage -d -uc -us && mv ../*.deb /dist" volumes: - ../..:/code:ro - ../../dist:/dist diff --git a/src/debian/compat b/src/debian/compat index ec63514..f599e28 100644 --- a/src/debian/compat +++ b/src/debian/compat @@ -1 +1 @@ -9 +10 diff --git a/src/debian/control b/src/debian/control old mode 100644 new mode 100755 index e44ce64..7c0221a --- a/src/debian/control +++ b/src/debian/control @@ -1,9 +1,13 @@ Source: nerve-tools +Section: admin +Priority: optional Maintainer: John Billings Build-Depends: dh-virtualenv, -Depends: python3.7, paasta-tools (>= 0.145.0~), ${shlibs:Depends} Package: nerve-tools +Section: admin +Priority: optional Architecture: any +Depends: python3.12, paasta-tools (>= 1.47.0~), ${shlibs:Depends} Description: Nerve-related tools for use on Yelp machines. diff --git a/src/debian/rules b/src/debian/rules index d3306c6..519525c 100755 --- a/src/debian/rules +++ b/src/debian/rules @@ -24,9 +24,5 @@ override_dh_auto_test: # TravisCI can't handle the large logs that verbose output creates and it will just terminate the build :( override_dh_virtualenv: dh_virtualenv -i $(PIP_INDEX_URL) \ - --python=/usr/bin/python3.7 \ - --preinstall no-manylinux1 + --python=/usr/bin/python3.12 -# For some reason dh virtualenv can't find libcrypto bundled but this seems to work. Probably there is a real fix somewhere? -override_dh_shlibdeps: - dh_shlibdeps --dpkg-shlibdeps-params=-ldebian/nerve-tools/opt/venvs/nerve-tools/lib/python3.7/site-packages/cryptography.libs diff --git a/src/mypy.ini b/src/mypy.ini old mode 100644 new mode 100755 index 1d0c40f..6ff81f4 --- a/src/mypy.ini +++ b/src/mypy.ini @@ -1,5 +1,5 @@ [mypy] -python_version = 3.6 +python_version = 3.12 ignore_missing_imports = True check_untyped_defs = True disallow_untyped_defs = True diff --git a/src/nerve_tools/config.py b/src/nerve_tools/config.py old mode 100644 new mode 100755 index 2bca6b4..8760095 --- a/src/nerve_tools/config.py +++ b/src/nerve_tools/config.py @@ -1,9 +1,9 @@ -from mypy_extensions import TypedDict -from typing import Mapping -from typing import Iterable from typing import Dict -from typing import Tuple +from typing import Iterable +from typing import Mapping from typing import Optional +from typing import Tuple +from typing import TypedDict class CheckDict(TypedDict, total=False): diff --git a/src/nerve_tools/updown_service.py b/src/nerve_tools/updown_service.py index 4c2eeff..4f329dc 100644 --- a/src/nerve_tools/updown_service.py +++ b/src/nerve_tools/updown_service.py @@ -12,7 +12,7 @@ import requests from requests.exceptions import RequestException -from paasta_tools.marathon_tools import load_service_namespace_config +from paasta_tools.long_running_service_tools import load_service_namespace_config from service_configuration_lib import read_service_configuration diff --git a/src/requirements-bootstrap.txt b/src/requirements-bootstrap.txt old mode 100644 new mode 100755 index 4dfb33b..21376f2 --- a/src/requirements-bootstrap.txt +++ b/src/requirements-bootstrap.txt @@ -1,3 +1,3 @@ -pip==21.0.1 -setuptools==54.1.2 -wheel==0.36.2 \ No newline at end of file +pip==25.2 +setuptools==80.10.1 +wheel==0.46.2 diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt old mode 100644 new mode 100755 index 73219f0..a651471 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -1,4 +1,6 @@ -coverage==4.5.4 -flake8==3.7.9 -mock==3.0.5 -pytest==5.2.1 \ No newline at end of file +coverage==6.5.0 +flake8==7.3.0 +mypy==1.17.1 +pytest==7.4.3 +types-PyYAML==6.0.12.20250915 +types-requests==2.32.4.20250913 diff --git a/src/requirements.txt b/src/requirements.txt old mode 100644 new mode 100755 index 3f13310..2a35bca --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,12 +1,6 @@ -argparse==1.2.1 -cryptography==35.0.0 -dulwich==0.17.3 environment_tools==1.1.3 kazoo==2.8.0 -ldap3==2.7 -mypy-extensions==0.4.3 -paasta-tools==0.105.0 +paasta-tools==1.47.0 PyYAML==6.0.1 -requests==2.23.0 -service-configuration-lib==2.5.4 -setuptools==42.0.2 +requests==2.32.5 +service-configuration-lib==3.3.8 diff --git a/src/setup.py b/src/setup.py old mode 100644 new mode 100755 index 8e17a96..581fd70 --- a/src/setup.py +++ b/src/setup.py @@ -1,15 +1,21 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from pkg_resources import yield_lines -from setuptools import setup, find_packages +from pathlib import Path +from setuptools import find_packages +from setuptools import setup -def get_install_requires(): - with open('requirements.txt', 'r') as f: - minimal_reqs = list(yield_lines(f.read())) - return minimal_reqs +HERE = Path(__file__).parent + + +def get_install_requires(): + return [ + line.strip() + for line in (HERE / 'requirements.txt').read_text().splitlines() + if line.strip() and not line.startswith('#') + ] setup( @@ -20,7 +26,6 @@ def get_install_requires(): author_email='compute-infra@yelp.com', description='Nerve-related tools for use on Yelp machines', packages=find_packages(exclude=['tests']), - setup_requires=['setuptools'], include_package_data=True, install_requires=get_install_requires(), entry_points={ diff --git a/src/tests/clean_nerve_test.py b/src/tests/clean_nerve_test.py old mode 100644 new mode 100755 index ec36584..3494317 --- a/src/tests/clean_nerve_test.py +++ b/src/tests/clean_nerve_test.py @@ -1,6 +1,6 @@ from nerve_tools import clean_nerve -import mock +from unittest import mock def test_parse_args(): diff --git a/src/tests/configure_nerve_test.py b/src/tests/configure_nerve_test.py old mode 100644 new mode 100755 index b31235a..281b8a2 --- a/src/tests/configure_nerve_test.py +++ b/src/tests/configure_nerve_test.py @@ -1,10 +1,10 @@ import copy -import mock -from mock import call -from mock import patch -from mock import mock_open -from mock import MagicMock -from mock import Mock +from unittest import mock +from unittest.mock import call +from unittest.mock import MagicMock +from unittest.mock import mock_open +from unittest.mock import Mock +from unittest.mock import patch import pytest import sys import multiprocessing @@ -75,6 +75,9 @@ def get_named_zookeeper_topology(cluster_type, cluster_location, zk_topology_dir }[(cluster_type, cluster_location)] +LOCATION_TYPES = ['ecosystem', 'superregion', 'habitat', 'region'] + + @pytest.fixture def expected_sub_config(): expected_config = { @@ -215,6 +218,9 @@ def test_generate_subconfiguration(expected_sub_config): ), patch( 'nerve_tools.configure_nerve.get_named_zookeeper_topology', side_effect=get_named_zookeeper_topology + ), patch( + 'environment_tools.type_utils.available_location_types', + return_value=LOCATION_TYPES ), patch( 'nerve_tools.configure_nerve.get_labels_by_service_and_port', side_effect=get_labels_by_service_and_port @@ -261,6 +267,9 @@ def test_generate_subconfiguration_k8s(expected_sub_config): ), patch( 'nerve_tools.configure_nerve.get_named_zookeeper_topology', side_effect=get_named_zookeeper_topology + ), patch( + 'environment_tools.type_utils.available_location_types', + return_value=LOCATION_TYPES ), patch( 'nerve_tools.configure_nerve.get_labels_by_service_and_port', side_effect=get_labels_by_service_and_port @@ -319,6 +328,9 @@ def test_generate_subconfiguration_with_envoy_ingress_listeners( ), patch( 'nerve_tools.configure_nerve.get_named_zookeeper_topology', side_effect=get_named_zookeeper_topology + ), patch( + 'environment_tools.type_utils.available_location_types', + return_value=LOCATION_TYPES ), patch( 'nerve_tools.configure_nerve.get_labels_by_service_and_port', side_effect=get_labels_by_service_and_port diff --git a/src/tests/envoy_test.py b/src/tests/envoy_test.py old mode 100644 new mode 100755 index 7517b57..e6804a3 --- a/src/tests/envoy_test.py +++ b/src/tests/envoy_test.py @@ -1,4 +1,4 @@ -from mock import patch +from unittest.mock import patch from nerve_tools.envoy import get_envoy_ingress_listeners diff --git a/src/tests/updown_service_test.py b/src/tests/updown_service_test.py old mode 100644 new mode 100755 index 40bd6a2..3f19138 --- a/src/tests/updown_service_test.py +++ b/src/tests/updown_service_test.py @@ -1,4 +1,4 @@ -import mock +from unittest import mock import pytest from requests.exceptions import RequestException import yaml diff --git a/src/tox.ini b/src/tox.ini old mode 100644 new mode 100755 index 57c8fda..c71b872 --- a/src/tox.ini +++ b/src/tox.ini @@ -1,20 +1,20 @@ [tox] -envlist=py37-linux +envlist=py312-linux [testenv] -basepython=python3.7 +basepython=python3.12 deps = -r{toxinidir}/requirements.txt [testenv:tests] -envdir = .tox/py37-linux/ +envdir = .tox/py312-linux/ recreate = false deps = {[testenv]deps} -r{toxinidir}/requirements-dev.txt --editable={toxinidir} commands = - coverage run -m py.test {posargs:tests} + coverage run -m pytest {posargs:tests} coverage report -m --include nerve_tools/* --show-missing --skip-covered --fail-under 75 flake8 nerve_tools tests @@ -22,11 +22,10 @@ commands = max-line-length = 120 [testenv:mypy] -envdir = .tox/py37-linux/ +envdir = .tox/py312-linux/ recreate = false deps = {[testenv:tests]deps} - mypy==0.790 setenv = MYPYPATH = {toxinidir} mypy_paths = diff --git a/tox.ini b/tox.ini index 66fb3c3..36fed27 100644 --- a/tox.ini +++ b/tox.ini @@ -5,36 +5,36 @@ skipsdist = true # The Makefile and .travis.yml override the indexserver to the public one when # running outside of Yelp. indexserver = https://pypi.yelpcorp.com/simple -basepython = python3.7 -deps = - docker-compose==1.29.2 - cryptography==35.0.0 +basepython = python3.12 passenv = DOCKER_TLS_VERIFY,DOCKER_HOST,DOCKER_CERT_PATH setenv = TZ = UTC PIP_INDEX_URL = {env:PIP_INDEX_URL:https://pypi.yelpcorp.com/simple} [testenv:package_jammy] -allowlist_externals = cp +allowlist_externals = + cp + docker setenv = COMPOSE_FILE = dockerfiles/jammy/docker-compose.yml SSH_AUTH_SOCK={env:SSH_AUTH_SOCK} commands = cp dockerfiles/itest/itest/yelpsoa-configs/location_mapping.json dockerfiles/itest/itest/yelpsoa-configs/location_types.json dockerfiles/jammy/ - docker-compose down - docker-compose --verbose build --no-cache --build-arg PIP_INDEX_URL={env:PIP_INDEX_URL:https://pypi.yelpcorp.com/simple} - docker-compose run jammy - docker-compose stop - docker-compose rm --force + docker compose down + docker compose build --no-cache --build-arg PIP_INDEX_URL={env:PIP_INDEX_URL:https://pypi.yelpcorp.com/simple} + docker compose run jammy + docker compose stop + docker compose rm --force [testenv:itest_jammy] +allowlist_externals = docker setenv = COMPOSE_PROJECT_NAME=nerve_tools_jammy COMPOSE_FILE = dockerfiles/itest/docker-compose-jammy.yml - SSH_AUTH_SOCK={env:SSH_AUTH_SOCK} + SSH_AUTH_SOCK={env:SSH_AUTH_SOCK:} commands = - docker-compose down - docker-compose --verbose build --no-cache --build-arg PIP_INDEX_URL={env:PIP_INDEX_URL:https://pypi.yelpcorp.com/simple} - docker-compose run itest - docker-compose stop - docker-compose rm --force + docker compose down + docker compose build --no-cache --build-arg PIP_INDEX_URL={env:PIP_INDEX_URL:https://pypi.yelpcorp.com/simple} + docker compose run itest + docker compose stop + docker compose rm --force