From abd852e1db45069d8eed2a03c3fc23190cbb9fec Mon Sep 17 00:00:00 2001 From: James Tanner Date: Wed, 21 Feb 2024 12:34:33 -0500 Subject: [PATCH] Initial set of playbooks. No-Issue Signed-off-by: James Tanner --- dev/playbooks/build_container.yaml | 30 ++++++++ .../galaxy_ng/tools/galaxy.yml | 0 .../tools/plugins/action/local_run.py | 22 ++++++ .../galaxy_ng/tools/plugins/filter/repr.py | 31 ++++++++ dev/playbooks/files/run_units.sh | 15 ++++ dev/playbooks/files/smash-config.json | 34 +++++++++ dev/playbooks/run_unit_tests.yaml | 60 +++++++++++++++ dev/playbooks/start_container.yaml | 76 +++++++++++++++++++ dev/playbooks/templates/Containerfile.j2 | 54 +++++++++++++ dev/playbooks/templates/settings.py.j2 | 21 +++++ dev/playbooks/vars/main.yaml | 29 +++++++ 11 files changed, 372 insertions(+) create mode 100644 dev/playbooks/build_container.yaml create mode 100644 dev/playbooks/collections/ansible_collections/galaxy_ng/tools/galaxy.yml create mode 100644 dev/playbooks/collections/ansible_collections/galaxy_ng/tools/plugins/action/local_run.py create mode 100644 dev/playbooks/collections/ansible_collections/galaxy_ng/tools/plugins/filter/repr.py create mode 100644 dev/playbooks/files/run_units.sh create mode 100644 dev/playbooks/files/smash-config.json create mode 100644 dev/playbooks/run_unit_tests.yaml create mode 100644 dev/playbooks/start_container.yaml create mode 100644 dev/playbooks/templates/Containerfile.j2 create mode 100644 dev/playbooks/templates/settings.py.j2 create mode 100644 dev/playbooks/vars/main.yaml diff --git a/dev/playbooks/build_container.yaml b/dev/playbooks/build_container.yaml new file mode 100644 index 0000000000..6f5ca27706 --- /dev/null +++ b/dev/playbooks/build_container.yaml @@ -0,0 +1,30 @@ +# Ansible playbook to create the pulp service containers image +--- +- hosts: localhost + connection: local + gather_facts: false + vars_files: + - vars/main.yaml + + tasks: + + - name: "Clean out the cache directory" + shell: "sudo rm -rf cache" + + - name: "Make the cache directory" + file: + name: cache + state: directory + + - name: "Generate Containerfile from template" + template: + src: Containerfile.j2 + dest: cache/Containerfile + + - name: "Build pulp image" + galaxy_ng.tools.local_run: + command: "docker build --network host --no-cache={{ not cache | default(true) | bool }} -t {{ image.name }}:{{ image.tag }} -f {{ playbook_dir }}//cache/Containerfile ../../.." + + - name: "Clean image cache" + docker_prune: + images : true diff --git a/dev/playbooks/collections/ansible_collections/galaxy_ng/tools/galaxy.yml b/dev/playbooks/collections/ansible_collections/galaxy_ng/tools/galaxy.yml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dev/playbooks/collections/ansible_collections/galaxy_ng/tools/plugins/action/local_run.py b/dev/playbooks/collections/ansible_collections/galaxy_ng/tools/plugins/action/local_run.py new file mode 100644 index 0000000000..9f85227bb6 --- /dev/null +++ b/dev/playbooks/collections/ansible_collections/galaxy_ng/tools/plugins/action/local_run.py @@ -0,0 +1,22 @@ +from ansible.plugins.action import ActionBase +import subprocess + +class ActionModule(ActionBase): + TRANSFERS_FILES = False + _VALID_ARGS = frozenset(('command',)) + + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + # Retrieve the command from the task's arguments + command = self._task.args.get('command', None) + + if command is None: + return {"failed": True, "msg": "The 'command' argument is required"} + + try: + # Run the command without capturing stdout or stderr + subprocess.run(command, shell=True, check=True) + return {"changed": True, "msg": "Command executed successfully"} + except subprocess.CalledProcessError as e: + return {"failed": True, "msg": "Command execution failed", "error": str(e)} diff --git a/dev/playbooks/collections/ansible_collections/galaxy_ng/tools/plugins/filter/repr.py b/dev/playbooks/collections/ansible_collections/galaxy_ng/tools/plugins/filter/repr.py new file mode 100644 index 0000000000..8455c3442f --- /dev/null +++ b/dev/playbooks/collections/ansible_collections/galaxy_ng/tools/plugins/filter/repr.py @@ -0,0 +1,31 @@ +from __future__ import absolute_import, division, print_function +from packaging.version import parse as parse_version + +__metaclass__ = type + + +ANSIBLE_METADATA = { + "metadata_version": "1.1", + "status": ["preview"], + "supported_by": "community", +} + + +def _repr_filter(value): + return repr(value) + + +def _canonical_semver_filter(value): + return str(parse_version(value)) + + +# ---- Ansible filters ---- +class FilterModule(object): + """Repr filter.""" + + def filters(self): + """Filter associations.""" + return { + "repr": _repr_filter, + "canonical_semver": _canonical_semver_filter, + } diff --git a/dev/playbooks/files/run_units.sh b/dev/playbooks/files/run_units.sh new file mode 100644 index 0000000000..4d87a12396 --- /dev/null +++ b/dev/playbooks/files/run_units.sh @@ -0,0 +1,15 @@ +#!/usr/bin/bash + +export XDG_CONFIG_HOME=/opt/settings +export PULP_API_ROOT="$(bash "/opt/oci_env/base/container_scripts/get_dynaconf_var.sh" API_ROOT)" +export PULP_DATABASES__default__USER=postgres +export PYTEST=/usr/local/bin/pytest + +env | sort + +rm -rf /tmp/galaxy_ng +cp -Rp /galaxy_ng /tmp/. +cd /tmp/galaxy_ng +sudo chown -R pulp:pulp /tmp/galaxy_ng + +sudo -u pulp -E $PYTEST -v -r sx --color=yes --pyargs "galaxy_ng.tests.unit" "${@:2}" --capture=no -k test_me_get diff --git a/dev/playbooks/files/smash-config.json b/dev/playbooks/files/smash-config.json new file mode 100644 index 0000000000..941122debe --- /dev/null +++ b/dev/playbooks/files/smash-config.json @@ -0,0 +1,34 @@ +{ + "pulp": { + "auth": [ + "admin", + "password" + ], + "selinux enabled": false, + "version": "3", + "aiohttp_fixtures_origin": "127.0.0.1" + }, + "hosts": [ + { + "hostname": "pulp", + "roles": { + "api": { + "port": 443, + "scheme": "https", + "service": "nginx" + }, + "content": { + "port": 443, + "scheme": "https", + "service": "pulp_content_app" + }, + "pulp resource manager": {}, + "pulp workers": {}, + "redis": {}, + "shell": { + "transport": "local" + } + } + } + ] +} diff --git a/dev/playbooks/run_unit_tests.yaml b/dev/playbooks/run_unit_tests.yaml new file mode 100644 index 0000000000..e837b0c144 --- /dev/null +++ b/dev/playbooks/run_unit_tests.yaml @@ -0,0 +1,60 @@ +# Ansible playbook to start the pulp service container and its supporting services +--- +- hosts: localhost + connection: local + gather_facts: false + tasks: + - name: add the pulp container as an inventory host + add_host: + name: pulp + +- hosts: pulp + connection: docker + gather_facts: false + tasks: + + - name: "Install unit test requirements" + command: + cmd: "pip3 install -r /galaxy_ng/unittest_requirements.txt" + + - name: "Make /opt/oci_env/base/container_scripts" + file: + name: /opt/oci_env/base/container_scripts + state: directory + + - name: "Get the dynaconf script" + command: + cmd: "curl -o /opt/oci_env/base/container_scripts/get_dynaconf_var.sh https://raw.githubusercontent.com/pulp/oci_env/main/base/container_scripts/get_dynaconf_var.sh" + + - name: "Install pulp-smash" + command: + #cmd: "pip3 show pulp-smash || pip3 install git+https://github.com/pulp/pulp-smash.git" + cmd: "pip3 install git+https://github.com/pulp/pulp-smash.git" + + - name: "Get the pulp-smash setup script" + command: + cmd: "curl -o /tmp/configure_pulp_smash.sh https://raw.githubusercontent.com/pulp/oci_env/main/base/container_scripts/configure_pulp_smash.sh" + + - name: "Run the pulp smash config" + command: + cmd: "bash /tmp/configure_pulp_smash.sh" + environment: + API_HOST: "pulp" + API_PORT: "443" + API_PROTOCOL: "https" + DJANGO_SUPERUSER_USERNAME: "admin" + DJANGO_SUPERUSER_PASSWORD: "password" + + - name: "Make sure the smash config was made correctly" + command: "jq . /opt/settings/pulp_smash/settings.json" + + - name: "copy the helper script" + copy: + src: run_units.sh + dest: /tmp/run_units.sh + mode: "0777" + + - name: "Run the unit tests" + galaxy_ng.tools.local_run: + command: docker exec -it pulp /bin/bash -c '/tmp/run_units.sh' + diff --git a/dev/playbooks/start_container.yaml b/dev/playbooks/start_container.yaml new file mode 100644 index 0000000000..d0ce2b7304 --- /dev/null +++ b/dev/playbooks/start_container.yaml @@ -0,0 +1,76 @@ +# Ansible playbook to start the pulp service container and its supporting services +--- +- hosts: localhost + connection: local + gather_facts: false + vars_files: + - vars/main.yaml + tasks: + - name: "Create Settings Directories" + file: + path: "cache/{{ item }}" + state: directory + mode: "0755" + loop: + - settings + #- ssh + #- ~/.config/pulp_smash + + - name: "Generate Pulp Settings" + template: + src: settings.py.j2 + dest: cache/settings/settings.py + + # required for hostname resolution inside the container(s) + - name: "Setup docker networking" + docker_network: + name: pulp_ci_bridge + + - name: "Start Service Containers" + docker_container: + name: "{{ item.name }}" + image: "{{ item.image }}" + auto_remove: true + recreate: true + privileged: true + networks: + - name: pulp_ci_bridge + aliases: "{{ item.name }}" + volumes: "{{ item.volumes | default(omit) }}" + env: "{{ item.env | default(omit) }}" + command: "{{ item.command | default(omit) }}" + state: started + loop: "{{ services | default([]) }}" + + - name: add the pulp container as an inventory host + add_host: + name: pulp + +- hosts: pulp + connection: docker + gather_facts: false + tasks: + + # curl -v -k -L -u admin:password https://pulp/api/galaxy/v3/collections/ + - block: + - name: "Wait for Pulp" + uri: + url: "https://pulp/api/galaxy/pulp/api/v3/status/" + follow_redirects: all + validate_certs: no + register: result + until: result.status == 200 + retries: 12 + delay: 5 + rescue: + - name: "Output pulp container log" + command: "docker logs pulp" + failed_when: true + + - name: "Set pulp admin password" + command: + cmd: "pulpcore-manager reset-admin-password --password password" + + - name: "Install django-extensions" + command: + cmd: "pip3 install django-extensions" diff --git a/dev/playbooks/templates/Containerfile.j2 b/dev/playbooks/templates/Containerfile.j2 new file mode 100644 index 0000000000..0eb5d5ccb0 --- /dev/null +++ b/dev/playbooks/templates/Containerfile.j2 @@ -0,0 +1,54 @@ +FROM {{ ci_base | default("ghcr.io/pulp/pulp-ci-centos:" + pulp_container_tag) }} + +# Add source directories to container +{% for item in plugins %} +{% if item.source.startswith("./") or item.ci_requirements | default(false) %} +ADD ./{{ item.name }} ./{{ item.name }} +{% endif %} +{% endfor %} + +# Install python packages + +RUN pip3 install +{%- for item in plugins -%} +{%- if item.name == "pulp-certguard" -%} +{{ " " }}python-dateutil rhsm +{%- endif -%} +{{ " " }}{{ item.source }} +{%- if item.name == "pulpcore" -%} +{%- if s3_test | default(false) -%} +[s3] +{%- elif azure_test | default(false) -%} +[azure] +{%- elif gcp_test | default(false) -%} +[google] +{%- endif -%} +{%- endif -%} +{%- if item.ci_requirements | default(false) -%} +{{ " " }}-r ./{{ item.name }}/ci_requirements.txt +{%- endif -%} +{%- endfor %} + +{% if pulp_env is defined and pulp_env %} +{% for key, value in pulp_env.items() %} +ENV {{ key | upper }}={{ value }} +{% endfor %} +{% endif %} + +{% if pulp_scenario_env is defined and pulp_scenario_env %} +{% for key, value in pulp_scenario_env.items() %} +ENV {{ key | upper }}={{ value }} +{% endfor %} +{% endif %} + +USER pulp:pulp +RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \ + /usr/local/bin/pulpcore-manager collectstatic --clear --noinput --link +USER root:root + +{% for item in plugins %} +RUN export plugin_path="$(pip3 show {{ item.name }} | sed -n -e 's/Location: //p')/{{ item.name }}" && \ + ln $plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true +{% endfor %} + +ENTRYPOINT ["/init"] diff --git a/dev/playbooks/templates/settings.py.j2 b/dev/playbooks/templates/settings.py.j2 new file mode 100644 index 0000000000..3a62d03920 --- /dev/null +++ b/dev/playbooks/templates/settings.py.j2 @@ -0,0 +1,21 @@ +CONTENT_ORIGIN = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}" +ANSIBLE_API_HOSTNAME = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}" +ANSIBLE_CONTENT_HOSTNAME = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}/pulp/content" +PRIVATE_KEY_PATH = "/etc/pulp/certs/token_private_key.pem" +PUBLIC_KEY_PATH = "/etc/pulp/certs/token_public_key.pem" +TOKEN_SERVER = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}/token/" +TOKEN_SIGNATURE_ALGORITHM = "ES256" +CACHE_ENABLED = True +REDIS_HOST = "localhost" +REDIS_PORT = 6379 +ANALYTICS = False + +{% if api_root is defined %} +API_ROOT = {{ api_root | galaxy_ng.tools.repr }} +{% endif %} + +{% if pulp_settings %} +{% for key, value in pulp_settings.items() %} +{{ key | upper }} = {{ value | galaxy_ng.tools.repr }} +{% endfor %} +{% endif %} diff --git a/dev/playbooks/vars/main.yaml b/dev/playbooks/vars/main.yaml new file mode 100644 index 0000000000..8119e74fed --- /dev/null +++ b/dev/playbooks/vars/main.yaml @@ -0,0 +1,29 @@ +ci_base: pulp/pulp-ci-centos9 +image: + name: pulp + tag: "ci_build" +plugins: + - name: galaxy_ng + source: "./galaxy_ng" +pulp_container_tag: "latest" +pulp_scheme: "https" + +api_root: "/api/galaxy/pulp/" + +services: + - name: pulp + image: "pulp:ci_build" + volumes: + - ./cache/settings:/etc/pulp + - ../../.:/galaxy_ng + env: + PULP_WORKERS: "4" + PULP_HTTPS: "true" + +s3_test: False +gcp_test: False +azure_test: False + +pulp_settings: + RH_ENTITLEMENT_REQUIRED: False +