From fc42c4512b50180c201f5f562de2fe30727689f6 Mon Sep 17 00:00:00 2001 From: Joachim Metz Date: Sun, 14 Apr 2024 09:51:30 +0200 Subject: [PATCH] Changes for CI tests (#1233) --- .github/workflows/test_docker.yml | 39 +++++++------------ .github/workflows/test_tox.yml | 16 ++++---- .yamllint.yaml | 8 ++++ data/presets.ini | 4 +- data/templates/github_actions/test_docker.yml | 4 +- data/templates/github_actions/test_docs.yml | 2 +- data/templates/github_actions/test_tox.yml | 10 +++-- l2tdevtools/build_helpers/dpkg.py | 2 +- pyproject.toml | 7 ++++ tests/helpers/project.py | 2 +- tests/review_helpers/cli.py | 2 +- tests/review_helpers/git.py | 2 +- tests/review_helpers/github.py | 2 +- tests/review_helpers/pylint.py | 2 +- tests/review_helpers/review.py | 2 +- tox.ini | 16 +++++++- 16 files changed, 71 insertions(+), 49 deletions(-) create mode 100644 .yamllint.yaml diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index b365c03e..9cda41da 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -7,18 +7,18 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - version: ['38'] + version: ['39', '40'] container: image: registry.fedoraproject.org/fedora:${{ matrix.version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up container run: | dnf install -y dnf-plugins-core langpacks-en - name: Install dependencies run: | dnf copr -y enable @gift/dev - dnf install -y @development-tools python3 python3-devel python3-setuptools + dnf install -y @development-tools python3 python3-build python3-devel python3-setuptools python3-wheel - name: Run tests env: LANG: C.utf8 @@ -27,25 +27,21 @@ jobs: - name: Run end-to-end tests run: | if test -f tests/end-to-end.py; then PYTHONPATH=. python3 ./tests/end-to-end.py --debug -c config/end-to-end.ini; fi - - name: Build source distribution + - name: Build source distribution (sdist) run: | - python3 ./setup.py sdist - - name: Build binary distribution + python3 -m build --no-isolation --sdist + - name: Build binary distribution (wheel) run: | - python3 ./setup.py bdist - - name: Run build and install test - run: | - python3 ./setup.py build - python3 ./setup.py install + python3 -m build --no-isolation --wheel test_ubuntu: runs-on: ubuntu-latest strategy: matrix: - version: ['22.04'] + version: ['24.04'] container: image: ubuntu:${{ matrix.version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up container env: DEBIAN_FRONTEND: noninteractive @@ -58,7 +54,7 @@ jobs: run: | add-apt-repository -y ppa:gift/dev apt-get update -q - apt-get install -y build-essential python3 python3-dev python3-distutils python3-pip python3-setuptools python3-wheel + apt-get install -y build-essential python3 python3-build python3-dev python3-pip python3-setuptools python3-wheel - name: Run tests env: LANG: en_US.UTF-8 @@ -69,16 +65,9 @@ jobs: LANG: en_US.UTF-8 run: | if test -f tests/end-to-end.py; then PYTHONPATH=. python3 ./tests/end-to-end.py --debug -c config/end-to-end.ini; fi - - name: Update setuptools - run: | - python3 -m pip install -U setuptools - - name: Build source distribution - run: | - python3 ./setup.py sdist - - name: Build binary distribution + - name: Build source distribution (sdist) run: | - python3 ./setup.py bdist - - name: Run build and install test + python3 -m build --no-isolation --sdist + - name: Build binary distribution (wheel) run: | - python3 ./setup.py build - python3 ./setup.py install + python3 -m build --no-isolation --wheel diff --git a/.github/workflows/test_tox.yml b/.github/workflows/test_tox.yml index ab928711..7b32b6d5 100644 --- a/.github/workflows/test_tox.yml +++ b/.github/workflows/test_tox.yml @@ -27,7 +27,7 @@ jobs: container: image: ubuntu:22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up container env: DEBIAN_FRONTEND: noninteractive @@ -44,7 +44,7 @@ jobs: add-apt-repository -y ppa:deadsnakes/ppa add-apt-repository -y ppa:gift/dev apt-get update -q - apt-get install -y build-essential git python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv python3-distutils python3-pip python3-setuptools + apt-get install -y build-essential git python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv python3-distutils python3-lib2to3 python3-pip python3-setuptools - name: Install tox run: | python3 -m pip install tox @@ -63,7 +63,7 @@ jobs: container: image: ubuntu:22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up container env: DEBIAN_FRONTEND: noninteractive @@ -80,7 +80,7 @@ jobs: add-apt-repository -y ppa:deadsnakes/ppa add-apt-repository -y ppa:gift/dev apt-get update -q - apt-get install -y build-essential git python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv python3-distutils python3-pip python3-setuptools + apt-get install -y build-essential git python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv python3-distutils python3-lib2to3 python3-pip python3-setuptools - name: Install tox run: | python3 -m pip install tox @@ -90,7 +90,9 @@ jobs: run: | tox -e${{ matrix.toxenv }} - name: Upload coverage report to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} lint: runs-on: ubuntu-latest strategy: @@ -101,7 +103,7 @@ jobs: container: image: ubuntu:22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up container env: DEBIAN_FRONTEND: noninteractive @@ -118,7 +120,7 @@ jobs: add-apt-repository -y ppa:deadsnakes/ppa add-apt-repository -y ppa:gift/dev apt-get update -q - apt-get install -y build-essential git python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv python3-distutils python3-pip python3-setuptools + apt-get install -y build-essential git python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv python3-distutils python3-lib2to3 python3-pip python3-setuptools - name: Install tox run: | python3 -m pip install tox diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 00000000..676c49ff --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,8 @@ +extends: default + +rules: + line-length: disable + indentation: + spaces: consistent + indent-sequences: false + check-multi-line-strings: true diff --git a/data/presets.ini b/data/presets.ini index efad08e7..7832c935 100644 --- a/data/presets.ini +++ b/data/presets.ini @@ -7,7 +7,7 @@ presets: dfvfs,dfwinreg projects: artifacts,dfimagetools [dfvfs] -projects: cffi,dfdatetime,dfvfs,dtfabric,idna,libcaes,libbde,libewf,libfcrypto,libfsapfs,libfsext,libfsfat,libfshfs,libfsntfs,libfsxfs,libfvde,libfwnt,libgzipf,libluksde,libmodi,libphdi,libqcow,libsigscan,libsmdev,libsmraw,libvhdi,libvmdk,libvsapm,libvsbsdl,libvsgpt,libvshadow,libvslvm,mock,pbr,pytsk3,PyYAML,xattr +projects: cffi,dfdatetime,dfvfs,dtfabric,libcaes,libbde,libewf,libfcrypto,libfsapfs,libfsext,libfsfat,libfshfs,libfsntfs,libfsxfs,libfvde,libfwnt,libgzipf,libluksde,libmodi,libphdi,libqcow,libsigscan,libsmdev,libsmraw,libvhdi,libvmdk,libvsapm,libvsbsdl,libvsgpt,libvshadow,libvslvm,mock,pbr,pytsk3,PyYAML,xattr [dfwinreg] projects: dfdatetime,dfwinreg,dtfabric,libcreg,libregf,mock,pbr,PyYAML @@ -29,7 +29,7 @@ projects: libbde,libcaes,libcdata,libcdatetime,libcdirectory,libcerror,libcfile, [plaso] presets: dfimagetools,redis-py -projects: acstore,bencode,certifi,cffi,chardet,dateutil,defusedxml,fakeredis,Flor,libcaes,libesedb,libevt,libevtx,libexe,libfwps,libfwsi,liblnk,libmsiecf,libolecf,libscca,mock,opensearch-py,pbr,pefile,plaso,psutil,pyparsing,python-lz4,python-zstd,pytz,pyzmq,requests,urllib3,XlsxWriter,yara-python +projects: acstore,bencode,certifi,cffi,chardet,dateutil,defusedxml,fakeredis,Flor,idna,libcaes,libesedb,libevt,libevtx,libexe,libfwps,libfwsi,liblnk,libmsiecf,libolecf,libscca,mock,opensearch-py,pbr,pefile,plaso,psutil,pyparsing,python-lz4,python-zstd,pytz,pyzmq,requests,urllib3,XlsxWriter,yara-python [redis-py] projects: deprecated,importlib-metadata,redis-py diff --git a/data/templates/github_actions/test_docker.yml b/data/templates/github_actions/test_docker.yml index 24cbc15b..cf1036e3 100644 --- a/data/templates/github_actions/test_docker.yml +++ b/data/templates/github_actions/test_docker.yml @@ -11,7 +11,7 @@ jobs: container: image: registry.fedoraproject.org/fedora:$${{ matrix.version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up container run: | dnf install -y dnf-plugins-core langpacks-en @@ -41,7 +41,7 @@ jobs: container: image: ubuntu:$${{ matrix.version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up container env: DEBIAN_FRONTEND: noninteractive diff --git a/data/templates/github_actions/test_docs.yml b/data/templates/github_actions/test_docs.yml index 1d19fbf3..c627fde5 100644 --- a/data/templates/github_actions/test_docs.yml +++ b/data/templates/github_actions/test_docs.yml @@ -19,7 +19,7 @@ jobs: container: image: ubuntu:22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up container env: DEBIAN_FRONTEND: noninteractive diff --git a/data/templates/github_actions/test_tox.yml b/data/templates/github_actions/test_tox.yml index 3c2ae9aa..4f3d27a1 100644 --- a/data/templates/github_actions/test_tox.yml +++ b/data/templates/github_actions/test_tox.yml @@ -27,7 +27,7 @@ jobs: container: image: ubuntu:22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up container env: DEBIAN_FRONTEND: noninteractive @@ -63,7 +63,7 @@ jobs: container: image: ubuntu:22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up container env: DEBIAN_FRONTEND: noninteractive @@ -90,7 +90,9 @@ jobs: run: | tox -e$${{ matrix.toxenv }} - name: Upload coverage report to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 + with: + token: $${{ secrets.CODECOV_TOKEN }} lint: runs-on: ubuntu-latest strategy: @@ -101,7 +103,7 @@ jobs: container: image: ubuntu:22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up container env: DEBIAN_FRONTEND: noninteractive diff --git a/l2tdevtools/build_helpers/dpkg.py b/l2tdevtools/build_helpers/dpkg.py index a834b457..f27369cd 100644 --- a/l2tdevtools/build_helpers/dpkg.py +++ b/l2tdevtools/build_helpers/dpkg.py @@ -763,7 +763,7 @@ class PybuildDPKGBuildHelperBase(DPKGBuildHelper): """Shared functionality for dh-pybuild build system dpkg build helpers.""" def _DetermineBuildConfiguration(self, source_directory): - """Determines the build configuration of a project that has setup.py + """Determines the build configuration of a project that has setup.py. Args: source_directory (str): path of the source directory. diff --git a/pyproject.toml b/pyproject.toml index 9787c3bd..ee74093b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,10 @@ [build-system] requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" + +[tool.docformatter] +black = false +non-cap = ["dfDateTime", "dfImageTools", "dfVFS", "dfWinReg", "dtFabric", "iMessage", "iOS", "iPod", "mDNS"] +non-strict = false +wrap-summaries = 80 +wrap-descriptions = 80 diff --git a/tests/helpers/project.py b/tests/helpers/project.py index e02ae6ec..96095184 100644 --- a/tests/helpers/project.py +++ b/tests/helpers/project.py @@ -10,7 +10,7 @@ class ProjectHelperTest(test_lib.BaseTestCase): - """Tests the project helper""" + """Tests the project helper.""" # pylint: disable=protected-access diff --git a/tests/review_helpers/cli.py b/tests/review_helpers/cli.py index 7fc6cf9c..bc6bb787 100644 --- a/tests/review_helpers/cli.py +++ b/tests/review_helpers/cli.py @@ -10,7 +10,7 @@ class CLIHelperTest(test_lib.BaseTestCase): - """Tests the command line helper""" + """Tests the command line helper.""" def testRunCommand(self): """Tests that the helper can be initialized.""" diff --git a/tests/review_helpers/git.py b/tests/review_helpers/git.py index 6f874300..e63793db 100644 --- a/tests/review_helpers/git.py +++ b/tests/review_helpers/git.py @@ -10,7 +10,7 @@ class GitHelperTest(test_lib.BaseTestCase): - """Tests the git helper""" + """Tests the git helper.""" def testInitialize(self): """Tests that the helper can be initialized.""" diff --git a/tests/review_helpers/github.py b/tests/review_helpers/github.py index b3d60ece..a687c6cb 100644 --- a/tests/review_helpers/github.py +++ b/tests/review_helpers/github.py @@ -12,7 +12,7 @@ class GitHubHelperTest(shared_test_lib.BaseTestCase): - """Tests the command line helper""" + """Tests the command line helper.""" # pylint: disable=protected-access diff --git a/tests/review_helpers/pylint.py b/tests/review_helpers/pylint.py index 353d5b05..ac4065df 100644 --- a/tests/review_helpers/pylint.py +++ b/tests/review_helpers/pylint.py @@ -10,7 +10,7 @@ class PylintHelperTest(test_lib.BaseTestCase): - """Tests the pylint helper""" + """Tests the pylint helper.""" # pylint: disable=protected-access diff --git a/tests/review_helpers/review.py b/tests/review_helpers/review.py index d2d8a955..2b72c82b 100644 --- a/tests/review_helpers/review.py +++ b/tests/review_helpers/review.py @@ -10,7 +10,7 @@ class ReviewHelperTest(test_lib.BaseTestCase): - """Tests the review helper""" + """Tests the review helper.""" def testInitialize(self): """Tests that the helper can be initialized.""" diff --git a/tox.ini b/tox.ini index 8ba68410..361374a1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py3{8,9,10,11,12},coverage,lint,wheel +envlist = py3{8,9,10,11,12},coverage,docformatter,lint,wheel [testenv] allowlist_externals = ./run_tests.py @@ -25,6 +25,13 @@ commands = coverage: coverage xml wheel: python -m build --no-isolation --wheel +[testenv:docformatter] +usedevelop = True +deps = + docformatter +commands = + docformatter --in-place --recursive l2tdevtools tests + [testenv:lint] skipsdist = True pip_pre = True @@ -37,7 +44,14 @@ setenv = deps = -rrequirements.txt -rtest_requirements.txt + docformatter pylint >= 3.0.0, < 3.1.0 + setuptools + yamllint >= 1.26.0 commands = + docformatter --version pylint --version + yamllint -v + docformatter --check --diff --recursive l2tdevtools setup.py tests tools pylint --rcfile=.pylintrc l2tdevtools setup.py tests tools + yamllint -c .yamllint.yaml data