Merge pull request #112 from adfinis/jlf/fix-improve-toml-converter

fix: improve toml converter

.github/workflows/main.yml

@@ -60,7 +60,7 @@ jobs:
 
     - name: Run tests
       run: |
-        make test
+        HYPOTHESIS_PROFILE=ci make test
 
   package:
     runs-on: ubuntu-latest

.github/workflows/pull-requests.yml

@@ -30,4 +30,4 @@ jobs:
 
     - name: Run tests
       run: |
-        make test
+        HYPOTHESIS_PROFILE=ci make test

Makefile

@@ -46,7 +46,7 @@ mypy: poetry-install
 
 .PHONY: pytest
 pytest: poetry-install ## run pytest
-	@docker compose exec testing poetry run pytest -vv --cov
+	@docker compose exec testing poetry run sh -c "HYPOTHESIS_PROFILE=$(HYPOTHESIS_PROFILE) pytest -vv --cov"
 
 .PHONY: format
 format: poetry-install ## format code with ruff

pyaptly/__init__.py

@@ -9,10 +9,10 @@
 def init_hypothesis():
     """Initialize hypothesis profile if hypothesis is available."""
     try:  # pragma: no cover
-        if "HYPOTHESIS_PROFILE" in os.environ:
+        if "HYPOTHESIS_PROFILE" in os.environ and os.environ["HYPOTHESIS_PROFILE"]:
             from hypothesis import settings
 
-            settings.register_profile("ci", settings(max_examples=10000))
+            settings.register_profile("ci", settings(max_examples=500))
             settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "default"))
     except (ImportError, AttributeError):  # pragma: no cover
         pass

pyaptly/cli.py

@@ -167,7 +167,7 @@ def publish(**kwargs):
     publish.publish(cfg, args=fake_args)
 
 
-@cli.command(help="convert yaml- to toml-comfig")
+@cli.command()
 @click.option("--debug/--no-debug", "-d/-nd", default=False, type=bool)
 @click.argument(
     "yaml_path",
@@ -205,3 +205,43 @@ def yaml_to_toml(yaml_path: Path, toml_path: Path, add_defaults: bool, debug: bo
         toml_path,
         add_defaults=add_defaults,
     )
+
+
+@cli.command()
+@click.option("--debug/--no-debug", "-d/-nd", default=False, type=bool)
+@click.argument(
+    "in_path",
+    type=click.Path(
+        file_okay=True,
+        dir_okay=False,
+        exists=True,
+        readable=True,
+        path_type=Path,
+    ),
+)
+@click.argument(
+    "toml_path",
+    type=click.Path(
+        file_okay=True,
+        dir_okay=False,
+        exists=False,
+        writable=True,
+        path_type=Path,
+    ),
+)
+@click.option(
+    "-a/-na",
+    "--add-defaults/--no-add-defaults",
+    type=bool,
+    default=False,
+    help="Add default values to fields if missing",
+)
+def toml_to_toml(in_path: Path, toml_path: Path, add_defaults: bool, debug: bool):
+    """Convert pyaptly config files from toml to toml."""
+    from . import config_file
+
+    config_file.toml_to_toml(
+        in_path,
+        toml_path,
+        add_defaults=add_defaults,
+    )

pyaptly/config_file.py

@@ -1,10 +1,14 @@
 """Handling pyaptly config-files."""
 
+# TODO: remove this as soon as most people have converted their config.
+
 from pathlib import Path
 
-import tomli_w
+import tomli
 import yaml
 
+from pyaptly import tomli_w
+
 
 def yaml_to_toml(yaml_path: Path, toml_path: Path, *, add_defaults: bool = False):
     """Convert pyaptly config files from yaml to toml.
@@ -19,6 +23,19 @@ def yaml_to_toml(yaml_path: Path, toml_path: Path, *, add_defaults: bool = False
             tomli_w.dump(config, tf)
 
 
+def toml_to_toml(in_path: Path, toml_path: Path, *, add_defaults: bool = False):
+    """Convert pyaptly config files from toml to toml.
+
+    Setting `add_defaults=True` will set common default during conversion.
+    """
+    with in_path.open("rb") as nf:
+        with toml_path.open("wb") as tf:
+            config = tomli.load(nf)
+            if add_defaults:
+                add_default_to_config(config)
+            tomli_w.dump(config, tf)
+
+
 def add_default_to_config(config):
     """Set common default in config if the fields are missing."""
     if "mirror" in config:

pyaptly/tests/debug.toml

@@ -1,31 +1,64 @@
 [mirror.trusty]
-components = [ "main", "multiverse", "restricted", "universe",]
-architectures = [ "amd64", "i386",]
+components = [
+    "main",
+    "multiverse",
+    "restricted",
+    "universe",
+]
+architectures = [
+    "amd64",
+    "i386",
+]
 distribution = "trusty"
 sources = true
 udeb = true
 archive = "http://ch.archive.ubuntu.com/ubuntu/"
-gpg-keys = [ "40976EAF437D05B5", "3B4FE6ACC0B21F32" ]
+gpg-keys = [
+    "40976EAF437D05B5",
+    "3B4FE6ACC0B21F32",
+]
 keyserver = "keyserver.ubuntu.com"
 
 [mirror.trusty-updates]
-components = [ "main", "multiverse", "restricted", "universe",]
-architectures = [ "amd64", "i386",]
+components = [
+    "main",
+    "multiverse",
+    "restricted",
+    "universe",
+]
+architectures = [
+    "amd64",
+    "i386",
+]
 distribution = "trusty-updates"
 sources = true
 udeb = true
 archive = "http://ch.archive.ubuntu.com/ubuntu/"
-gpg-keys = [ "40976EAF437D05B5", "3B4FE6ACC0B21F32" ]
+gpg-keys = [
+    "40976EAF437D05B5",
+    "3B4FE6ACC0B21F32",
+]
 keyserver = "keyserver.ubuntu.com"
 
 [mirror.trusty-backports]
-components = [ "main", "multiverse", "restricted", "universe",]
-architectures = [ "amd64", "i386",]
+components = [
+    "main",
+    "multiverse",
+    "restricted",
+    "universe",
+]
+architectures = [
+    "amd64",
+    "i386",
+]
 distribution = "trusty-backports"
 sources = true
 udeb = true
 archive = "http://ch.archive.ubuntu.com/ubuntu/"
-gpg-keys = [ "40976EAF437D05B5", "3B4FE6ACC0B21F32" ]
+gpg-keys = [
+    "40976EAF437D05B5",
+    "3B4FE6ACC0B21F32",
+]
 keyserver = "keyserver.ubuntu.com"
 
 [snapshot.trusty-latest]

pyaptly/tests/mirror-basic.toml

@@ -1,12 +1,16 @@
 [mirror.fakerepo01]
 max-tries = 2
 archive = "http://localhost:3123/fakerepo01"
-gpg-keys = [ "2841988729C7F3FF",]
+gpg-keys = [
+    "2841988729C7F3FF",
+]
 components = "main"
 distribution = "main"
 
 [mirror.fakerepo02]
 archive = "http://localhost:3123/fakerepo02"
-gpg-keys = [ "2841988729C7F3FF",]
+gpg-keys = [
+    "2841988729C7F3FF",
+]
 components = "main"
 distribution = "main"

pyaptly/tests/mirror-extra.toml

@@ -1,6 +1,10 @@
 [mirror.fakerepo03]
 archive = "http://localhost:3123/fakerepo03"
-gpg-keys = [ "EC54D33E5B5EBE98",]
-gpg-urls = [ "http://localhost:3123/keys/test02.key",]
+gpg-keys = [
+    "EC54D33E5B5EBE98",
+]
+gpg-urls = [
+    "http://localhost:3123/keys/test02.key",
+]
 components = "main"
 distribution = "main"

pyaptly/tests/publish-current.toml

@@ -1,13 +1,56 @@
+[[publish."fakerepo01/current"]]
+distribution = "stable"
+architectures = [
+    "amd64",
+]
+components = [
+    "main",
+]
+snapshots = [
+    "fakerepo01-current",
+]
+automatic-update = true
+
+[[publish."fakerepo02/current"]]
+distribution = "stable"
+architectures = [
+    "amd64",
+]
+components = [
+    "main",
+]
+snapshots = [
+    "fakerepo02-current",
+]
+automatic-update = true
+
+[[publish."fake/current"]]
+distribution = "stable"
+architectures = [
+    "amd64",
+]
+components = [
+    "main",
+]
+snapshots = [
+    "fake-current",
+]
+automatic-update = true
+
 [mirror.fakerepo01]
 max-tries = 2
 archive = "http://localhost:3123/fakerepo01"
-gpg-keys = [ "2841988729C7F3FF",]
+gpg-keys = [
+    "2841988729C7F3FF",
+]
 components = "main"
 distribution = "main"
 
 [mirror.fakerepo02]
 archive = "http://localhost:3123/fakerepo02"
-gpg-keys = [ "2841988729C7F3FF",]
+gpg-keys = [
+    "2841988729C7F3FF",
+]
 components = "main"
 distribution = "main"
 
@@ -18,27 +61,7 @@ mirror = "fakerepo01"
 mirror = "fakerepo02"
 
 [snapshot.fake-current]
-merge = [ "fakerepo01-current", "fakerepo02-current",]
-
-[publish]
-[[publish."fakerepo01/current"]]
-distribution = "stable"
-architectures = [ "amd64",]
-components = [ "main",]
-snapshots = [ "fakerepo01-current",]
-automatic-update = true
-
-[[publish."fakerepo02/current"]]
-distribution = "stable"
-architectures = [ "amd64",]
-components = [ "main",]
-snapshots = [ "fakerepo02-current",]
-automatic-update = true
-
-[[publish."fake/current"]]
-distribution = "stable"
-architectures = [ "amd64",]
-components = [ "main",]
-snapshots = [ "fake-current",]
-automatic-update = true
-
+merge = [
+    "fakerepo01-current",
+    "fakerepo02-current",
+]

pyaptly/tests/publish-previous.toml

@@ -1,34 +1,31 @@
 [mirror.fakerepo01]
 max-tries = 2
 archive = "http://localhost:3123/fakerepo01"
-gpg-keys = [ "2841988729C7F3FF",]
+gpg-keys = [
+    "2841988729C7F3FF",
+]
 components = "main"
 distribution = "main"
 
 [mirror.fakerepo02]
 archive = "http://localhost:3123/fakerepo02"
-gpg-keys = [ "2841988729C7F3FF",]
+gpg-keys = [
+    "2841988729C7F3FF",
+]
 components = "main"
 distribution = "main"
 
 [snapshot."fakerepo01-%T"]
 mirror = "fakerepo01"
+timestamp = { time = "00:00" }
 
 [snapshot."fakerepo02-%T"]
 mirror = "fakerepo02"
-
-[snapshot."fakerepo01-%T".timestamp]
-time = "00:00"
-
-[snapshot."fakerepo02-%T".timestamp]
-time = "00:00"
-repeat-weekly = "sat"
+timestamp = { time = "00:00", repeat-weekly = "sat" }
 
 [snapshot."superfake-%T"]
 merge = [
     { name = "fakerepo01-%T", timestamp = "previous" },
     { name = "fakerepo02-%T", timestamp = 0 },
 ]
-
-[snapshot."superfake-%T".timestamp]
-time = "00:00"
+timestamp = { time = "00:00" }