diff --git a/TODO b/TODO index c03ebc7..78c5c3c 100644 --- a/TODO +++ b/TODO @@ -1,3 +1,5 @@ +# NEXT: Add coverage and mypy + # Update old files The following files have just been copied without checking if their content is ok: @@ -11,4 +13,10 @@ The reason for this is to ensure the continuity of the git history. # Tool to convert yaml config to toml - This is probably just a sub-command of pyaptly -- We probably still want to switch to a modern yaml parser \ No newline at end of file +- We probably still want to switch to a modern yaml parser + +# Rename testsets mentioning google + +# Replace all subprocess commands with a modern one (usually run()) + +# All logging should be done via logging (no stdout/stderr) \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 9f23997..6faf71e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,9 +1,10 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry and should not be changed by hand. [[package]] name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -22,6 +23,7 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "black" version = "23.12.0" description = "The uncompromising code formatter." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -66,6 +68,7 @@ uvloop = ["uvloop (>=0.15.2)"] name = "click" version = "8.1.7" description = "Composable command line interface toolkit" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -80,6 +83,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -91,6 +95,7 @@ files = [ name = "docstring-to-markdown" version = "0.13" description = "On the fly conversion of Python docstrings to markdown" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -102,6 +107,7 @@ files = [ name = "fancycompleter" version = "0.9.1" description = "colorful TAB completion for Python prompt" +category = "dev" optional = false python-versions = "*" files = [ @@ -117,6 +123,7 @@ pyrepl = ">=0.8.2" name = "flake8" version = "6.1.0" description = "the modular source code checker: pep8 pyflakes and co" +category = "dev" optional = false python-versions = ">=3.8.1" files = [ @@ -133,6 +140,7 @@ pyflakes = ">=3.1.0,<3.2.0" name = "flake8-bugbear" version = "23.12.2" description = "A plugin for flake8 finding likely bugs and design problems in your program. Contains warnings that don't belong in pyflakes and pycodestyle." +category = "dev" optional = false python-versions = ">=3.8.1" files = [ @@ -151,6 +159,7 @@ dev = ["coverage", "hypothesis", "hypothesmith (>=0.2)", "pre-commit", "pytest", name = "flake8-debugger" version = "4.1.2" description = "ipdb/pdb statement checker plugin for flake8" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -166,6 +175,7 @@ pycodestyle = "*" name = "flake8-docstrings" version = "1.7.0" description = "Extension for flake8 which uses pydocstyle to check docstrings" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -181,6 +191,7 @@ pydocstyle = ">=2.1" name = "flake8-isort" version = "6.1.1" description = "flake8 plugin that integrates isort" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -199,6 +210,7 @@ test = ["pytest"] name = "flake8-string-format" version = "0.3.0" description = "string format checker, plugin for flake8" +category = "dev" optional = false python-versions = "*" files = [ @@ -213,6 +225,7 @@ flake8 = "*" name = "flake8-tuple" version = "0.4.1" description = "Check code for 1 element tuple." +category = "dev" optional = false python-versions = "*" files = [ @@ -228,6 +241,7 @@ six = "*" name = "freezegun" version = "1.3.1" description = "Let your Python tests travel through time" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -242,6 +256,7 @@ python-dateutil = ">=2.7" name = "hypothesis" version = "6.92.0" description = "A library for property-based testing" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -273,6 +288,7 @@ zoneinfo = ["backports.zoneinfo (>=0.2.1)", "tzdata (>=2023.3)"] name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -284,6 +300,7 @@ files = [ name = "isort" version = "5.13.2" description = "A Python utility / library to sort Python imports." +category = "dev" optional = false python-versions = ">=3.8.0" files = [ @@ -298,6 +315,7 @@ colors = ["colorama (>=0.4.6)"] name = "jedi" version = "0.19.1" description = "An autocompletion tool for Python that can be used for text editors." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -317,6 +335,7 @@ testing = ["Django", "attrs", "colorama", "docopt", "pytest (<7.0.0)"] name = "mccabe" version = "0.7.0" description = "McCabe checker, plugin for flake8" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -328,6 +347,7 @@ files = [ name = "mock" version = "5.1.0" description = "Rolling backport of unittest.mock for all Pythons" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -344,6 +364,7 @@ test = ["pytest", "pytest-cov"] name = "mypy" version = "1.7.1" description = "Optional static typing for Python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -390,6 +411,7 @@ reports = ["lxml"] name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." +category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -401,6 +423,7 @@ files = [ name = "packaging" version = "23.2" description = "Core utilities for Python packages" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -412,6 +435,7 @@ files = [ name = "parso" version = "0.8.3" description = "A Python Parser" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -427,6 +451,7 @@ testing = ["docopt", "pytest (<6.0.0)"] name = "pathspec" version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -438,6 +463,7 @@ files = [ name = "pdbpp" version = "0.10.3" description = "pdb++, a drop-in replacement for pdb" +category = "dev" optional = false python-versions = "*" files = [ @@ -458,6 +484,7 @@ testing = ["funcsigs", "pytest"] name = "platformdirs" version = "4.1.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -473,6 +500,7 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co name = "pluggy" version = "1.3.0" description = "plugin and hook calling mechanisms for python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -488,6 +516,7 @@ testing = ["pytest", "pytest-benchmark"] name = "pretty-dump" version = "3.0" description = "Diff and dump anything" +category = "main" optional = false python-versions = "*" files = [] @@ -506,6 +535,7 @@ resolved_reference = "a5bd2bdfc68d46df01695079886b3818477f3137" name = "pycodestyle" version = "2.11.1" description = "Python style guide checker" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -517,6 +547,7 @@ files = [ name = "pydocstyle" version = "6.3.0" description = "Python docstring style checker" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -534,6 +565,7 @@ toml = ["tomli (>=1.2.3)"] name = "pyflakes" version = "3.1.0" description = "passive checker of Python programs" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -545,6 +577,7 @@ files = [ name = "pygments" version = "2.17.2" description = "Pygments is a syntax highlighting package written in Python." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -560,6 +593,7 @@ windows-terminal = ["colorama (>=0.4.6)"] name = "pyreadline" version = "2.1" description = "A python implmementation of GNU readline." +category = "dev" optional = false python-versions = "*" files = [ @@ -570,6 +604,7 @@ files = [ name = "pyrepl" version = "0.9.0" description = "A library for building flexible command line interfaces" +category = "dev" optional = false python-versions = "*" files = [ @@ -580,6 +615,7 @@ files = [ name = "pytest" version = "7.4.3" description = "pytest: simple powerful testing with Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -600,6 +636,7 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" +category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -614,6 +651,7 @@ six = ">=1.5" name = "python-lsp-black" version = "1.3.0" description = "Black plugin for the Python LSP Server" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -632,6 +670,7 @@ dev = ["flake8", "isort (>=5.0)", "mypy", "pre-commit", "pytest", "types-pkg-res name = "python-lsp-isort" version = "0.1" description = "isort plugin for the Python LSP Server" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -649,6 +688,7 @@ dev = ["pytest"] name = "python-lsp-jsonrpc" version = "1.1.2" description = "JSON RPC 2.0 server library" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -666,6 +706,7 @@ test = ["coverage", "pycodestyle", "pyflakes", "pylint", "pytest", "pytest-cov"] name = "python-lsp-server" version = "1.9.0" description = "Python Language Server for the Language Server Protocol" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -698,6 +739,7 @@ yapf = ["whatthepatch (>=1.0.2,<2.0.0)", "yapf (>=0.33.0)"] name = "pytz" version = "2023.3.post1" description = "World timezone definitions, modern and historical" +category = "main" optional = false python-versions = "*" files = [ @@ -709,6 +751,7 @@ files = [ name = "pyyaml" version = "6.0.1" description = "YAML parser and emitter for Python" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -768,6 +811,7 @@ files = [ name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -779,6 +823,7 @@ files = [ name = "snowballstemmer" version = "2.2.0" description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." +category = "dev" optional = false python-versions = "*" files = [ @@ -790,6 +835,7 @@ files = [ name = "sortedcontainers" version = "2.4.0" description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" +category = "dev" optional = false python-versions = "*" files = [ @@ -801,6 +847,7 @@ files = [ name = "testfixtures" version = "7.2.2" description = "A collection of helpers and mock objects for unit tests and doc tests." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -817,6 +864,7 @@ test = ["django", "mypy", "pytest (>=3.6)", "pytest-cov", "pytest-django", "sybi name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" +category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -828,6 +876,7 @@ files = [ name = "typing-extensions" version = "4.9.0" description = "Backported and Experimental Type Hints for Python 3.8+" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -839,6 +888,7 @@ files = [ name = "ujson" version = "5.9.0" description = "Ultra fast JSON encoder and decoder for Python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -913,6 +963,7 @@ files = [ name = "wmctrl" version = "0.5" description = "A tool to programmatically control windows inside X" +category = "dev" optional = false python-versions = ">=2.7" files = [ @@ -929,4 +980,4 @@ test = ["pytest"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "4040715d1ef4567da6077f9c2cffb3f08c032983acd2f88d4149aff91a52c92f" +content-hash = "4d7ec7ba8ba4181faed29aeb205967f505df634e9f1289b7b815b240402aba1f" diff --git a/pyaptly/aptly_test.py b/pyaptly/aptly_test.py index 6a3614f..6557740 100644 --- a/pyaptly/aptly_test.py +++ b/pyaptly/aptly_test.py @@ -120,6 +120,7 @@ def do_mirror_update(config): assert aptly_state["number of packages"] == "2" +@pytest.mark.skip def test_mirror_update(): """Test if updating mirrors works.""" with test.clean_and_config( diff --git a/pyaptly/cli.py b/pyaptly/cli.py new file mode 100644 index 0000000..1dd34f9 --- /dev/null +++ b/pyaptly/cli.py @@ -0,0 +1,65 @@ +from pathlib import Path + +import click + +# I decided it is a good pattern to do lazy imports in the cli module. I had to +# do this in a few other CLIs for startup performance. + + +@click.group() +@click.option( + "-d/-nd", + "--debug/--no-debug", + type=bool, + default=False, + help="Add default values to fields if missing", +) +def cli(debug): + from pyaptly import util + + util._DEBUG = debug + + +@cli.command(help="run legacy command parser") +def legacy(): + from pyaptly import main + + main() + + +@cli.command(help="convert yaml- to toml-comfig") +@click.argument( + "yaml_path", + type=click.Path( + file_okay=True, + dir_okay=False, + exists=True, + readable=True, + path_type=Path, + ), +) +@click.argument( + "toml_path", + type=click.Path( + file_okay=True, + dir_okay=False, + exists=None, + writable=True, + path_type=Path, + ), +) +@click.option( + "-a/-na", + "--add-defaults/--no-add-defaults", + type=bool, + default=False, + help="Add default values to fields if missing", +) +def yaml_to_toml(yaml_path, toml_path, add_defaults): + from pyaptly import config_file + + config_file.yaml_to_toml( + yaml_path, + toml_path, + add_defaults=add_defaults, + ) diff --git a/pyaptly/config_file.py b/pyaptly/config_file.py new file mode 100644 index 0000000..fce38f6 --- /dev/null +++ b/pyaptly/config_file.py @@ -0,0 +1,29 @@ +from pathlib import Path + +import toml +import yaml + + +def yaml_to_toml(yaml_path: Path, toml_path: Path, *, add_defaults: bool = False): + with yaml_path.open("r", encoding="UTF-8") as yf: + with toml_path.open("w", encoding="UTF-8") as tf: + config = yaml.safe_load(yf) + if add_defaults: + add_dehfault_to_config(config) + toml.dump(config, tf) + + +def add_dehfault_to_config(config): + if "mirror" in config: + for mirror in config["mirror"].values(): + if "components" not in mirror: + mirror["components"] = "main" + if "distribution" not in mirror: + mirror["distribution"] = "main" + if "publish" in config: + for publish in config["publish"].values(): + for item in publish: + if "components" not in item: + item["components"] = "main" + if "distribution" not in item: + item["distribution"] = "main" diff --git a/pyaptly/conftest.py b/pyaptly/conftest.py index 5b0a374..a73f4b6 100644 --- a/pyaptly/conftest.py +++ b/pyaptly/conftest.py @@ -1,4 +1,5 @@ import json +import logging import os import tempfile from pathlib import Path @@ -12,7 +13,23 @@ @pytest.fixture() -def environment(): +def debug_mode(): + from pyaptly import util + + level = logging.root.getEffectiveLevel() + + try: + util._PYTEST_DEBUG = True + logging.root.setLevel(logging.DEBUG) + + yield + finally: + util._PYTEST_DEBUG = False + logging.root.setLevel(level) + + +@pytest.fixture() +def environment(debug_mode): tempdir_obj = tempfile.TemporaryDirectory() tempdir = Path(tempdir_obj.name).absolute() diff --git a/pyaptly/tests/mirror-basic.toml b/pyaptly/tests/mirror-basic.toml new file mode 100644 index 0000000..31c2b05 --- /dev/null +++ b/pyaptly/tests/mirror-basic.toml @@ -0,0 +1,12 @@ +[mirror.fakerepo01] +max-tries = 2 +archive = "http://localhost:3123/fakerepo01" +gpg-keys = [ "2841988729C7F3FF",] +components = "main" +distribution = "main" + +[mirror.fakerepo02] +archive = "http://localhost:3123/fakerepo02" +gpg-keys = [ "2841988729C7F3FF",] +components = "main" +distribution = "main" diff --git a/pyaptly/tests/mirror-extra.toml b/pyaptly/tests/mirror-extra.toml new file mode 100644 index 0000000..72e1044 --- /dev/null +++ b/pyaptly/tests/mirror-extra.toml @@ -0,0 +1,6 @@ +[mirror.fakerepo03] +archive = "http://localhost:3123/fakerepo03" +gpg-keys = [ "EC54D33E5B5EBE98",] +gpg-urls = [ "http://localhost:3123/keys/test02.key",] +components = "main" +distribution = "main" diff --git a/pyaptly/tests/publish.toml b/pyaptly/tests/publish.toml deleted file mode 100644 index 465859e..0000000 --- a/pyaptly/tests/publish.toml +++ /dev/null @@ -1,6 +0,0 @@ -[mirror.fakerepo03] -archive = "http://localhost:3123/fakerepo03" -gpg-keys = ["EC54D33E5B5EBE98"] -gpg-urls = ["http://localhost:3123/keys/test02.key"] -components = "main" -distribution = "main" \ No newline at end of file diff --git a/pyaptly/tests/test_mirror.py b/pyaptly/tests/test_mirror.py index 5c192b6..0f1bedd 100644 --- a/pyaptly/tests/test_mirror.py +++ b/pyaptly/tests/test_mirror.py @@ -3,9 +3,10 @@ import pytest import pyaptly +from pyaptly import util -@pytest.mark.parametrize("config", ["publish.toml"], indirect=True) +@pytest.mark.parametrize("config", ["mirror-extra.toml"], indirect=True) def test_mirror_create(environment, config, caplog): """Test if creating mirrors works.""" config_file, config_dict = config @@ -26,3 +27,31 @@ def test_mirror_create(environment, config, caplog): state = pyaptly.SystemStateReader() state.read() assert state.mirrors == expect + + +@pytest.mark.parametrize("config", ["mirror-basic.toml"], indirect=True) +def test_mirror_update(environment, config): + config_file, config_dict = config + do_mirror_update(config_file) + + +def do_mirror_update(config_file): + """Test if updating mirrors works.""" + args = ["-c", config_file, "mirror", "create"] + state = pyaptly.SystemStateReader() + state.read() + assert "fakerepo01" not in state.mirrors + pyaptly.main(args) + state.read() + assert "fakerepo01" in state.mirrors + args[3] = "update" + pyaptly.main(args) + args = [ + "aptly", + "mirror", + "show", + "fakerepo01", + ] + result = util.run(args, stdout=util.PIPE, check=True) + aptly_state = util.parse_aptly_show_command(result.stdout) + assert aptly_state["number of packages"] == "2" diff --git a/pyaptly/util.py b/pyaptly/util.py new file mode 100644 index 0000000..5109277 --- /dev/null +++ b/pyaptly/util.py @@ -0,0 +1,99 @@ +import logging +import subprocess +from subprocess import DEVNULL, PIPE + +_DEBUG = False +_PYTEST_DEBUG = False + +RESULT_LOG = """ +Command call +args: {args} +returncode: {returncode} +stdout: '{stdout}' +stderr: '{stderr}' +""".strip() +_indent = " " * 13 + +logger = logging.getLogger(__name__) + + +def is_debug_mode(): + """Check if we are in debug mode.""" + return _DEBUG or _PYTEST_DEBUG + + +def run(cmd_args, *, decode=True, **kwargs): + """Instrumented subprocess.run() for easier debugging. + + By default this run command will add `encoding="UTF-8"` to kwargs. Disable + with `decode=False`. + """ + debug = is_debug_mode() + added_stdout = False + added_stderr = False + if debug: + if "stdout" not in kwargs: + kwargs["stdout"] = PIPE + added_stdout = True + if "stderr" not in kwargs: + kwargs["stderr"] = PIPE + added_stderr = True + result = None + if decode: + kwargs["encoding"] = "UTF-8" + try: + result = subprocess.run(cmd_args, **kwargs) + finally: + if debug and result: + log_run_result(result) + # Do not change returned result by debug mode + if added_stdout: + delattr(result, "stdout") + if added_stderr: + delattr(result, "stderr") + return result + + +def indent_out(output): + output = output.strip() + if not output: + return "" + indented = False + if hasattr(output, "decode"): + try: + output = output.decode(encoding="UTF-8") + lines = output.splitlines() + result = [lines[0]] + for line in lines[1:]: + result.append(f"{_indent}{line}") + indented = True + except UnicodeDecodeError: + pass + + if not indented: + lines = output.splitlines() + result = [str(lines[0])] + for line in lines[1:]: + result.append(f"{_indent}{str(line)}") + return "\n".join(result) + + +def log_run_result(result): + msg = RESULT_LOG.format( + args=result.args, + returncode=result.returncode, + stdout=indent_out(result.stdout), + stderr=indent_out(result.stderr), + ) + logger.debug(msg) + + +def parse_aptly_show_command(show): + """Parse an aptly show command.""" + result = {} + for line in show.split("\n"): + if ":" in line: + key, value = line.split(":", 1) + key = key.lower() + result[key] = value.strip() + return result diff --git a/pyproject.toml b/pyproject.toml index 49379fa..2a9ed7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,12 +6,16 @@ authors = ["Jean-Louis Fuchs "] license = "AGPL-3.0-or-later" readme = "README.md" +[tool.poetry.scripts] +pyaptly = 'pyaptly.cli:cli' + [tool.poetry.dependencies] python = "^3.11" pretty-dump = {git = "https://github.com/adfinis/freeze"} pytz = "^2023.3.post1" pyyaml = "^6.0.1" toml = "^0.10.2" +click = "^8.1.7" [tool.poetry.group.dev.dependencies] freezegun = "^1.2.2"