From b122f3cdb0ed1aacc6d637e62c593a12f0d1cb59 Mon Sep 17 00:00:00 2001 From: James Tanner Date: Tue, 12 Dec 2023 20:29:46 -0500 Subject: [PATCH] Allow overrides on role imports. No-Issue Signed-off-by: James Tanner --- galaxy_ng/app/api/v1/serializers.py | 4 + galaxy_ng/app/api/v1/tasks.py | 83 +++++++++++++- galaxy_ng/app/api/v1/viewsets/roles.py | 12 +- galaxy_ng/app/utils/galaxy.py | 12 +- .../community/test_role_import_overrides.py | 105 ++++++++++++++++++ galaxy_ng/tests/integration/utils/legacy.py | 76 +++++++++++++ 6 files changed, 277 insertions(+), 15 deletions(-) create mode 100644 galaxy_ng/tests/integration/community/test_role_import_overrides.py diff --git a/galaxy_ng/app/api/v1/serializers.py b/galaxy_ng/app/api/v1/serializers.py index 7e0424b7d9..a7340fb1c6 100644 --- a/galaxy_ng/app/api/v1/serializers.py +++ b/galaxy_ng/app/api/v1/serializers.py @@ -569,7 +569,9 @@ class LegacyImportSerializer(serializers.Serializer): github_user = serializers.CharField() github_repo = serializers.CharField() + alternate_namespace_name = serializers.CharField(required=False) alternate_role_name = serializers.CharField(required=False) + alternate_clone_url = serializers.CharField(required=False) github_reference = serializers.CharField(required=False) class Meta: @@ -577,7 +579,9 @@ class Meta: fields = [ 'github_user', 'github_repo', + 'alternate_namespace_name', 'alternate_role_name', + 'alternate_clone_url', 'github_reference', ] diff --git a/galaxy_ng/app/api/v1/tasks.py b/galaxy_ng/app/api/v1/tasks.py index 09bd0b955e..c5bbd6233b 100644 --- a/galaxy_ng/app/api/v1/tasks.py +++ b/galaxy_ng/app/api/v1/tasks.py @@ -19,6 +19,7 @@ from galaxy_ng.app.utils.galaxy import upstream_role_iterator from galaxy_ng.app.utils.legacy import process_namespace from galaxy_ng.app.utils.namespaces import generate_v3_namespace_from_attributes +from galaxy_ng.app.utils.rbac import get_v3_namespace_owners from galaxy_ng.app.api.v1.models import LegacyNamespace from galaxy_ng.app.api.v1.models import LegacyRole @@ -271,8 +272,10 @@ def legacy_role_import( github_user=None, github_repo=None, github_reference=None, + alternate_namespace_name=None, alternate_role_name=None, - superuser_can_create_namespaces=False + alternate_clone_url=None, + superuser_can_create_namespaces=False, ): """ Import a legacy role by user, repo and or reference. @@ -285,9 +288,17 @@ def legacy_role_import( The github repository name that the role lives in. :param github_reference: A commit, branch or tag name to import. + :param alternate_namespace_name: + NO-OP ... future for UI :param alternate_role_name: - Override the enumerated role name when the repo does - not conform to the ansible-role- convention. + NO-OP ... future for UI + :param alternate_clone_url: + Override the enumerated clone url for the repo. + Only used for testing right now. + :param superuser_can_create_namespaces: + If the function is called by a superuser, it still + won't be allowed to create namespaces on the fly + without this being set to True This function attempts to clone the github repository to a temporary directory and uses galaxy-importer functions to @@ -324,6 +335,8 @@ def legacy_role_import( logger.info(f'github_user: {github_user}') logger.info(f'github_repo: {github_repo}') logger.info(f'github_reference: {github_reference}') + logger.info(f'alternate_clone_url: {alternate_clone_url}') + logger.info(f'alternate_namespace_name: {alternate_namespace_name}') logger.info(f'alternate_role_name: {alternate_role_name}') logger.info('') @@ -350,6 +363,9 @@ def legacy_role_import( ) logger.info('') + if alternate_clone_url: + clone_url = alternate_clone_url + # the user should have a legacy and v3 namespace if they logged in ... namespace = LegacyNamespace.objects.filter(name=real_namespace_name).first() if not namespace: @@ -419,10 +435,65 @@ def legacy_role_import( logger.info('') logger.info('===== PROCESSING LOADER RESULTS ====') + + # Allow the meta/main.yml to define the destination namespace ... + new_namespace_name = namespace.name + if alternate_namespace_name: + new_namespace_name = alternate_namespace_name + logger.info(f'overriding namespace name via parameter: {alternate_namespace_name}') + elif result['metadata']['galaxy_info'].get('namespace'): + new_namespace_name = result['metadata']['galaxy_info'].get('namespace') + logger.info(f'overriding namespace name via metadata: {new_namespace_name}') + + if namespace.name != new_namespace_name: + + # does it exist and is the user an owner? + found = LegacyNamespace.objects.filter(name=new_namespace_name).first() + if found: + provider = found.namespace + if provider: + owners = get_v3_namespace_owners(provider) + else: + owners = [] + + if not provider and not request_user.is_superuser: + logger.error( + f'legacy namespace {found} has no provider namespace to define owners' + ) + raise Exception('permission denied') + + if request_user not in owners and not request_user.is_superuser: + logger.error( + f'{request_user.username} is not an owner' + f' of provider namespace {provider.name}' + ) + raise Exception('permission denied') + + else: + # we need to create the namespace but only if allowed ... + if not request_user.is_superuser or not superuser_can_create_namespaces: + logger.error( + f'legacy namespace {new_namespace_name} does not exist' + ) + raise Exception('permission denied') + + logger.info('creating legacy namespace {new_namespace_name}') + namespace, _ = LegacyNamespace.objects.get_or_create(name=new_namespace_name) + + namespace, _ = LegacyNamespace.objects.get_or_create(name=new_namespace_name) + real_role = None + # munge the role name via an order of precedence - role_name = result["metadata"]["galaxy_info"].get("role_name") or \ - alternate_role_name or github_repo.replace("ansible-role-", "") + if alternate_role_name: + role_name = alternate_role_name + elif result["metadata"]["galaxy_info"].get("role_name"): + role_name = result["metadata"]["galaxy_info"]["role_name"] + else: + role_name = github_repo.replace("ansible-role-", "") + logger.info(f'enumerated role name {role_name}') + if real_role and real_role.name != role_name: + real_role = None galaxy_info = result["metadata"]["galaxy_info"] new_full_metadata = { @@ -487,7 +558,7 @@ def legacy_role_import( logger.info('') logger.info('Import completed') - return True + return this_role def legacy_sync_from_upstream( diff --git a/galaxy_ng/app/api/v1/viewsets/roles.py b/galaxy_ng/app/api/v1/viewsets/roles.py index 1b1e3def03..65d66d9aa4 100644 --- a/galaxy_ng/app/api/v1/viewsets/roles.py +++ b/galaxy_ng/app/api/v1/viewsets/roles.py @@ -263,18 +263,13 @@ def create(self, request): to the pulp tasking system and translating the task UUID to an integer to retain v1 compatibility. """ - serializer_class = LegacyImportSerializer - serializer = serializer_class(data=request.data) + serializer = LegacyImportSerializer(data=request.data) serializer.is_valid(raise_exception=True) kwargs = dict(serializer.validated_data) # tell the defered task who started this job kwargs['request_username'] = request.user.username - # synthetically create the name for the response - role_name = kwargs.get('alternate_role_name') or \ - kwargs['github_repo'].replace('ansible-role-', '') - task_id, pulp_id = self.legacy_dispatch(legacy_role_import, kwargs=kwargs) return Response({ @@ -284,9 +279,12 @@ def create(self, request): 'github_user': kwargs['github_user'], 'github_repo': kwargs['github_repo'], 'github_reference': kwargs.get('github_reference'), + 'alternate_namespace_name': kwargs.get('alternate_namespace_name'), + 'alternate_role_name': kwargs.get('alternate_role_name'), + 'alternate_clone_url': kwargs.get('alternate_clone_url'), 'summary_fields': { 'role': { - 'name': role_name + 'name': None } } }] diff --git a/galaxy_ng/app/utils/galaxy.py b/galaxy_ng/app/utils/galaxy.py index 989330b71e..2c4f6e912a 100644 --- a/galaxy_ng/app/utils/galaxy.py +++ b/galaxy_ng/app/utils/galaxy.py @@ -443,12 +443,13 @@ def upstream_role_iterator( pagenum = 0 role_count = 0 while next_url: - logger.info(f'fetch {pagenum} {next_url} role-count:{role_count}') + logger.info(f'fetch {pagenum} {next_url} role-count:{role_count} ...') page = safe_fetch(next_url) # Some upstream pages return ISEs for whatever reason. if page.status_code >= 500: + logger.error(f'{next_url} returned 500ISE. incrementing the page manually') if 'page=' in next_url: next_url = next_url.replace(f'page={pagenum}', f'page={pagenum+1}') else: @@ -520,9 +521,16 @@ def upstream_role_iterator( if ds.get('next'): next_url = ds['next'] elif ds.get('next_link'): - next_url = _baseurl + ds['next_link'] + next_url = ds['next_link'] else: # break if no next page break + api_prefix = '/api/v1' + if not next_url.startswith(_baseurl): + if not next_url.startswith(api_prefix): + next_url = _baseurl + api_prefix + next_url + else: + next_url = _baseurl + next_url + pagenum += 1 diff --git a/galaxy_ng/tests/integration/community/test_role_import_overrides.py b/galaxy_ng/tests/integration/community/test_role_import_overrides.py new file mode 100644 index 0000000000..dc68e3b678 --- /dev/null +++ b/galaxy_ng/tests/integration/community/test_role_import_overrides.py @@ -0,0 +1,105 @@ +"""test_community.py - Tests related to the community featureset. +""" + +import pytest + +from ..utils import ( + get_client, +) +from ..utils.legacy import ( + cleanup_social_user, + LegacyRoleGitRepoBuilder, + wait_for_v1_task +) + +pytestmark = pytest.mark.qa # noqa: F821 + + +@pytest.mark.parametrize( + 'spec', + [ + { + 'namespace': 'foo', + 'name': 'bar', + 'github_user': 'foo', + 'github_repo': 'bar', + 'meta_namespace': None, + 'meta_name': None, + 'alternate_namespace_name': 'foo', + 'alternate_role_name': 'bar', + }, + { + 'namespace': 'jim32', + 'name': 'bob32', + 'github_user': 'jim', + 'github_repo': 'bob', + 'meta_namespace': 'jim32', + 'meta_name': 'bob32', + 'alternate_namespace_name': None, + 'alternate_role_name': None, + } + + ] +) +@pytest.mark.deployment_community +def test_role_import_overrides(ansible_config, spec): + """" Validate setting namespace in meta/main.yml does the right thing """ + + admin_config = ansible_config("admin") + admin_client = get_client( + config=admin_config, + request_token=False, + require_auth=True + ) + + # all required namespaces ... + ns_names = [ + spec['namespace'], + spec['github_user'], + spec['alternate_namespace_name'], + spec['meta_namespace'] + ] + ns_names = sorted(set([x for x in ns_names if x])) + + # cleanup + for ns_name in ns_names: + cleanup_social_user(ns_name, ansible_config) + try: + admin_client(f'/api/v3/namespaces/{ns_name}/', method='DELETE') + except Exception: + pass + + # make the namespace(s) + for ns_name in ns_names: + v1 = admin_client('/api/v1/namespaces/', method='POST', args={'name': ns_name}) + v3 = admin_client('/api/v3/namespaces/', method='POST', args={'name': ns_name}) + admin_client( + f'/api/v1/namespaces/{v1["id"]}/providers/', method='POST', args={'id': v3['id']} + ) + + # make a local git repo + builder_kwargs = { + 'namespace': spec['namespace'], + 'name': spec['name'], + 'meta_namespace': spec['meta_namespace'], + 'meta_name': spec['meta_name'], + } + lr = LegacyRoleGitRepoBuilder(**builder_kwargs) + + # run the import + payload = {'alternate_clone_url': lr.role_dir} + for key in ['github_user', 'github_repo', 'alternate_namespace_name', 'alternate_role_name']: + if spec.get(key): + payload[key] = spec[key] + resp = admin_client('/api/v1/imports/', method='POST', args=payload) + task_id = resp['results'][0]['id'] + result = wait_for_v1_task(task_id=task_id, api_client=admin_client) + assert result['results'][0]['state'] == 'SUCCESS' + + # find the role and check it's attributes ... + roles_search = admin_client(f'/api/v1/roles/?namespace={spec["namespace"]}&name={spec["name"]}') + assert roles_search['count'] == 1 + assert roles_search['results'][0]['summary_fields']['namespace']['name'] == spec['namespace'] + assert roles_search['results'][0]['name'] == spec['name'] + assert roles_search['results'][0]['github_user'] == spec['github_user'] + assert roles_search['results'][0]['github_repo'] == spec['github_repo'] diff --git a/galaxy_ng/tests/integration/utils/legacy.py b/galaxy_ng/tests/integration/utils/legacy.py index dd223a259b..a211c5e97f 100644 --- a/galaxy_ng/tests/integration/utils/legacy.py +++ b/galaxy_ng/tests/integration/utils/legacy.py @@ -1,6 +1,10 @@ import random import string +import os +import subprocess +import tempfile import time +import yaml from galaxykit.users import delete_user as delete_user_gk from .client_ansible_lib import get_client @@ -53,6 +57,17 @@ def clean_all_roles(ansible_config): if resp['next'] is None: break next_url = resp['next'] + ix = next_url.index('/api') + next_url = next_url[ix:] + + # cleanup_social_user would delete these -IF- all + # are associated to a user but we can't rely on that + for role_data in pre_existing: + role_url = f'/api/v1/roles/{role_data["id"]}/' + try: + admin_client(role_url, method='DELETE') + except Exception: + pass usernames = [x['github_user'] for x in pre_existing] usernames = sorted(set(usernames)) @@ -218,3 +233,64 @@ def generate_unused_legacy_namespace(api_client=None): existing = get_all_legacy_namespaces(api_client=api_client) existing = dict((x['name'], x) for x in existing) return generate_legacy_namespace(exclude=list(existing.keys())) + + +class LegacyRoleGitRepoBuilder: + + def __init__( + self, + namespace=None, + name=None, + meta_namespace=None, + meta_name=None + ): + self.namespace = namespace + self.name = name + self.meta_namespace = meta_namespace + self.meta_name = meta_name + + self.workdir = tempfile.mkdtemp(prefix='gitrepo_') + self.role_dir = None + + self.role_init() + self.role_edit() + self.git_init() + self.git_commit() + + self.fix_perms() + + def fix_perms(self): + subprocess.run(f'chown -R pulp:pulp {self.workdir}', shell=True) + + def role_init(self): + cmd = f'ansible-galaxy role init {self.namespace}.{self.name}' + self.role_dir = os.path.join(self.workdir, self.namespace + '.' + self.name) + pid = subprocess.run(cmd, shell=True, cwd=self.workdir) + assert pid.returncode == 0 + assert os.path.exists(self.role_dir) + + def role_edit(self): + if self.meta_namespace or self.meta_name: + + meta_file = os.path.join(self.role_dir, 'meta', 'main.yml') + with open(meta_file, 'r') as f: + meta = yaml.safe_load(f.read()) + + if self.meta_namespace: + meta['galaxy_info']['namespace'] = self.meta_namespace + if self.meta_name: + meta['galaxy_info']['role_name'] = self.meta_name + + with open(meta_file, 'w') as f: + f.write(yaml.dump(meta)) + + def git_init(self): + subprocess.run('git init', shell=True, cwd=self.role_dir) + + def git_commit(self): + + subprocess.run('git config --global user.email "root@localhost"', shell=True) + subprocess.run('git config --global user.name "root at localhost"', shell=True) + + subprocess.run('git add *', shell=True, cwd=self.role_dir) + subprocess.run('git commit -m "first checkin"', shell=True, cwd=self.role_dir)