playbooks for CI checks, units and functional tests

.coveragerc

@@ -0,0 +1,5 @@
+[run]
+data_file = /tmp/.coverage
+omit = 
+	galaxy_ng/_vendor/*
+	galaxy_ng/tests/*

.github/workflows/ci_full.yml

@@ -0,0 +1,101 @@
+---
+name: galaxy_ng/ci
+on: {pull_request: {branches: ['**']}, push: {branches: ['**']}}
+
+jobs:
+
+  check_commit:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.after }}  # for PR avoids checking out merge commit
+          fetch-depth: 0  # include all history
+
+      - name: Run script to validate commits for both pull request and a push
+        env:
+          PY_COLORS: '1'
+          ANSIBLE_FORCE_COLOR: '1'
+          GITHUB_PULL_REQUEST: ${{ github.event.number }}
+          GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }}
+          GITHUB_BRANCH: ${{ github.head_ref }}
+          GITHUB_REPO_SLUG: ${{ github.repository }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }}
+          GITHUB_USER: ${{ github.event.pull_request.user.login }}
+          GITHUB_PR_COMMITS_URL: ${{ github.event.pull_request.commits_url }}
+          START_COMMIT: ${{ github.event.before }}
+          END_COMMIT: ${{ github.event.after }}
+        run: |
+          python .ci/scripts/validate_commit_message_custom.py
+
+  lint_po:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.after }}  # for PR avoids checking out merge commit
+          fetch-depth: 0  # include all history
+
+      - name: intsall and run lint-po
+        run: |
+          pip install lint-po
+          lint-po ./galaxy_ng/locale/*/LC_MESSAGES/*.po
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.after }}  # for PR avoids checking out merge commit
+          fetch-depth: 0  # include all history
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Install requirements
+        run: pip3 install -r lint_requirements.txt
+
+      - name: Run extra lint checks
+        run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh"
+
+      - name: Check manifest
+        run: check-manifest
+
+      - name: Check for pulpcore imports outside of pulpcore.plugin
+        run: sh .ci/scripts/check_pulpcore_imports.sh
+
+      - name: Check for gettext problems
+        run: sh .ci/scripts/check_gettext.sh
+
+  test:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - uses: actions/checkout@v4
+
+      - name: install ansible
+        run: pip3 install ansible
+
+      - name: run the build container playbook
+        run: cd dev/playbooks; ansible-playbook -i 'localhost,' --forks=1 -v build_container.yaml
+
+      - name: run the start container playbook
+        run: cd dev/playbooks; ansible-playbook -i 'localhost,' --forks=1 -v start_container.yaml
+
+      - name: run the unit test playbook
+        run: cd dev/playbooks; ansible-playbook -i 'localhost,' --forks=1 -v run_unit_tests.yaml
+
+      - name: run the functional test playbook
+        run: cd dev/playbooks; ansible-playbook -i 'localhost,' --forks=1 -v run_functional_tests.yaml

MANIFEST.in

@@ -22,3 +22,4 @@ recursive-include galaxy_ng *.py
 include galaxy_ng/tests/integration/utils/gpg/collection_sign.sh
 include galaxy_ng/tests/integration/utils/gpg/qe-sign-priv.gpg
 include galaxy-operator/bin/readyz.py
+exclude .coveragerc

dev/playbooks/RUNALL.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+docker-killall
+docker-rmall
+
+ANSIBLE="ansible-playbook -i 'localhost,' --forks=1 -vvvv"
+
+$ANSIBLE build_container.yaml
+$ANSIBLE start_container.yaml
+$ANSIBLE run_unit_tests.yaml

dev/playbooks/build_container.yaml

@@ -0,0 +1,34 @@
+# Ansible playbook to create the pulp service containers image
+---
+- hosts: localhost
+  connection: local
+  gather_facts: false
+  vars_files:
+    - vars/main.yaml
+
+  tasks:
+
+    - name: "Build the base image"
+      galaxy_ng.tools.local_run:
+        command: "cd docker; ./BUILD.sh"
+
+    - name: "Clean out the cache directory"
+      shell: "sudo rm -rf cache"
+
+    - name: "Make the cache directory"
+      file:
+        name: cache
+        state: directory
+
+    - name: "Generate Containerfile from template"
+      template:
+        src: Containerfile.j2
+        dest: cache/Containerfile
+
+    - name: "Build pulp image"
+      galaxy_ng.tools.local_run:
+        command: "docker build --network host --no-cache={{ not cache | default(true) | bool }} -t {{ image.name }}:{{ image.tag }} -f {{ playbook_dir }}//cache/Containerfile ../../.."
+
+    - name: "Clean image cache"
+      docker_prune:
+        images : true

dev/playbooks/collections/ansible_collections/galaxy_ng/tools/plugins/action/local_run.py

@@ -0,0 +1,23 @@
+from ansible.plugins.action import ActionBase
+import subprocess
+
+
+class ActionModule(ActionBase):
+    TRANSFERS_FILES = False
+    _VALID_ARGS = frozenset(('command',))
+
+    def run(self, tmp=None, task_vars=None):
+        super(ActionModule, self).run(tmp, task_vars)
+
+        # Retrieve the command from the task's arguments
+        command = self._task.args.get('command', None)
+
+        if command is None:
+            return {"failed": True, "msg": "The 'command' argument is required"}
+
+        try:
+            # Run the command without capturing stdout or stderr
+            subprocess.run(command, shell=True, check=True)
+            return {"changed": True, "msg": "Command executed successfully"}
+        except subprocess.CalledProcessError as e:
+            return {"failed": True, "msg": "Command execution failed", "error": str(e)}

dev/playbooks/collections/ansible_collections/galaxy_ng/tools/plugins/filter/repr.py

@@ -0,0 +1,31 @@
+from __future__ import absolute_import, division, print_function
+from packaging.version import parse as parse_version
+
+__metaclass__ = type
+
+
+ANSIBLE_METADATA = {
+    "metadata_version": "1.1",
+    "status": ["preview"],
+    "supported_by": "community",
+}
+
+
+def _repr_filter(value):
+    return repr(value)
+
+
+def _canonical_semver_filter(value):
+    return str(parse_version(value))
+
+
+# ---- Ansible filters ----
+class FilterModule(object):
+    """Repr filter."""
+
+    def filters(self):
+        """Filter associations."""
+        return {
+            "repr": _repr_filter,
+            "canonical_semver": _canonical_semver_filter,
+        }

dev/playbooks/collections/ansible_collections/galaxy_ng/tools/roles/pulp_client/tasks/main.yml

@@ -0,0 +1,33 @@
+# https://github.com/pulp/oci_env/blob/main/base/local_scripts/generate_client.sh
+# https://github.com/pulp/oci_env/blob/main/base/container_scripts/install_client.sh
+
+- name: make sure there's a checkout of pulp-openapi-generator
+  shell: test -d ../../../pulp-openapi-generator || git clone https://github.com/pulp/pulp-openapi-generator ../../../pulp-openapi-generator
+  connection: local
+
+- name: get the pulp container IP
+  shell: docker inspect pulp | jq '.[0].NetworkSettings.Networks.pulp_ci_bridge.IPAddress' | tr -d '"'
+  connection: local
+  register: pulp_ip
+
+- name: run the generate script
+  galaxy_ng.tools.local_run:
+    command: >
+       cd ../../../pulp-openapi-generator;
+       export USE_LOCAL_API_JSON=true;
+       curl -L -k -u admin:password -o api.json 'https://{{ pulp_ip.stdout }}/api/galaxy/pulp/api/v3/docs/api.json?bindings&plugin={{ item }}';
+       bash -x generate.sh {{ item }} python
+  connection: local
+  loop:
+      - galaxy_ng
+      - pulp_ansible
+      - pulp_container
+      - pulpcore
+
+- name: install the generated client inside the pulp container
+  shell: cd /src/pulp-openapi-generator/{{ item }}-client/; pip3 install -e .
+  loop:
+      - galaxy_ng
+      - pulp_ansible
+      - pulp_container
+      - pulpcore

dev/playbooks/collections/ansible_collections/galaxy_ng/tools/roles/pulp_smash/tasks/main.yml

@@ -0,0 +1,79 @@
+- name: "Make /opt/oci_env/base/container_scripts"
+  file:
+    name: /opt/oci_env/base/container_scripts
+    state: directory
+
+- name: "Get the dynaconf script"
+  command:
+    cmd: "curl -o /opt/oci_env/base/container_scripts/get_dynaconf_var.sh https://raw.githubusercontent.com/pulp/oci_env/main/base/container_scripts/get_dynaconf_var.sh"
+
+#- name: install the pulpcli
+#  command:
+#    cmd: "pip3 install pulp-cli"
+
+#- name: "run pulp config"
+#  shell: |
+#      #/bin/bash
+#      PULP_API_ROOT="/api/galaxy/pulp/"
+#      pulp config create --base-url https://pulp --api-root "$PULP_API_ROOT"
+
+- name: "Install pulp-smash"
+  command:
+    #cmd: "pip3 show pulp-smash || pip3 install git+https://github.com/pulp/pulp-smash.git"
+    cmd: "pip3 install git+https://github.com/pulp/pulp-smash.git"
+
+- name: "Get the pulp-smash setup script"
+  command:
+    cmd: "curl -o /tmp/configure_pulp_smash.sh https://raw.githubusercontent.com/pulp/oci_env/main/base/container_scripts/configure_pulp_smash.sh"
+
+- name: "Run the pulp smash config"
+  command:
+    cmd: "bash /tmp/configure_pulp_smash.sh"
+  environment:
+    API_HOST: "pulp"
+    API_PORT: "443"
+    API_PROTOCOL: "https"
+    DJANGO_SUPERUSER_USERNAME: "admin"
+    DJANGO_SUPERUSER_PASSWORD: "password"
+
+- name: "Make sure the smash config was made correctly"
+  command: "jq . /opt/settings/pulp_smash/settings.json"
+
+- name: "Set perms on the pulp_smash directory"
+  file:
+      path: ~/.config/pulp_smash/
+      state: directory
+      mode: "0777"
+
+- name: "Copy settings to config dir"
+  shell: "cp -f /opt/settings/pulp_smash/settings.json ~/.config/pulp_smash/setting.json"
+
+- name: "Set perms on the pulp_smash settings file"
+  file:
+      path: ~/.config/pulp_smash/setting.json
+      state: file
+      mode: "0666"
+
+- name: "Setup the ca-certs directory"
+  file:
+      path: /usr/local/share/ca-certificates/
+      state: directory
+
+- name: "Copy the cert to the ca folder"
+  shell: cp -f /etc/pulp/certs/pulp_webserver.crt /usr/local/share/ca-certificates/.
+
+#- name: "Add the CA cert"
+#  script:
+#    interpreter: /bin/bash
+#    content: |
+#      #!/bin/bash
+#      # Hack: adding pulp CA to certifi.where()
+#      CERTIFI=$(python3 -c 'import certifi; print(certifi.where())')
+#      cat /usr/local/share/ca-certificates/pulp_webserver.crt | sudo tee -a "$CERTIFI" > /dev/null
+#      if [[ "$TEST" = "azure" ]]; then
+#        cat /usr/local/share/ca-certificates/azcert.crt | sudo tee -a "$CERTIFI" > /dev/null
+#      fi
+#      # Hack: adding pulp CA to default CA file
+#      CERT=$(python3 -c 'import ssl; print(ssl.get_default_verify_paths().openssl_cafile)')
+#      cat "$CERTIFI" | sudo tee -a "$CERT" > /dev/null
+#      # update-ca-certificates

dev/playbooks/docker/BUILD.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [[ ! -d pulp-oci-images ]]; then
+    git clone https://github.com/pulp/pulp-oci-images
+fi
+cd pulp-oci-images
+git reset --hard
+cd ..
+
+cp -f switch_python pulp-oci-images/images/assets/.
+chmod +x pulp-oci-images/images/assets/switch_python
+
+cd pulp-oci-images
+git apply ../py311.patch
+
+docker build --file images/Containerfile.core.base --tag pulp/base:latest .
+docker build --file images/pulp_ci_centos/Containerfile --tag pulp/pulp-ci-centos9:latest .
+

dev/playbooks/docker/py311.patch

@@ -0,0 +1,15 @@
+diff --git a/images/Containerfile.core.base b/images/Containerfile.core.base
+index c05669c..7d84cb8 100644
+--- a/images/Containerfile.core.base
++++ b/images/Containerfile.core.base
+@@ -130,6 +130,10 @@ COPY images/assets/add_signing_service.sh /usr/bin/add_signing_service.sh
+ COPY images/assets/pulp-api /usr/bin/pulp-api
+ COPY images/assets/pulp-content /usr/bin/pulp-content
+ COPY images/assets/pulp-worker /usr/bin/pulp-worker
++COPY images/assets/switch_python /usr/bin/switch_python
++
++ARG PYTHON_VERSION=3.11
++RUN switch_python "$PYTHON_VERSION"
+
+ # Need to precreate when running pulp as the pulp user
+ RUN touch /var/log/galaxy_api_access.log && \

dev/playbooks/docker/switch_python

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+
+NEWV=$1
+if [[ "$NEWV" == "3.8" ]]; then
+    echo "using default python 3.8"
+    exit 0
+fi
+
+echo "switching python to $NEWV"
+
+rm -f /usr/local/bin/pip3
+dnf install -y python${NEWV} python${NEWV}-pip python${NEWV}-devel
+update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${NEWV} 1
+update-alternatives --set python3 /usr/bin/python${NEWV}
+update-alternatives --install /usr/bin/pip3 pip3 /usr/bin/pip-${NEWV} 1
+update-alternatives --set pip3 /usr/bin/pip-${NEWV}