support legacy role import overrides for namespace and name.

galaxy_ng/app/api/v1/serializers.py

@@ -569,15 +569,19 @@ class LegacyImportSerializer(serializers.Serializer):
 
     github_user = serializers.CharField()
     github_repo = serializers.CharField()
+    alternate_namespace_name = serializers.CharField(required=False)
     alternate_role_name = serializers.CharField(required=False)
+    alternate_clone_url = serializers.CharField(required=False)
     github_reference = serializers.CharField(required=False)
 
     class Meta:
         model = None
         fields = [
             'github_user',
             'github_repo',
+            'alternate_namespace_name',
             'alternate_role_name',
+            'alternate_clone_url',
             'github_reference',
         ]
 

galaxy_ng/app/api/v1/tasks.py

@@ -19,6 +19,7 @@
 from galaxy_ng.app.utils.galaxy import upstream_role_iterator
 from galaxy_ng.app.utils.legacy import process_namespace
 from galaxy_ng.app.utils.namespaces import generate_v3_namespace_from_attributes
+from galaxy_ng.app.utils.rbac import get_v3_namespace_owners
 
 from galaxy_ng.app.api.v1.models import LegacyNamespace
 from galaxy_ng.app.api.v1.models import LegacyRole
@@ -271,8 +272,10 @@ def legacy_role_import(
     github_user=None,
     github_repo=None,
     github_reference=None,
+    alternate_namespace_name=None,
     alternate_role_name=None,
-    superuser_can_create_namespaces=False
+    alternate_clone_url=None,
+    superuser_can_create_namespaces=False,
 ):
     """
     Import a legacy role by user, repo and or reference.
@@ -285,9 +288,17 @@ def legacy_role_import(
         The github repository name that the role lives in.
     :param github_reference:
         A commit, branch or tag name to import.
+    :param alternate_namespace_name:
+        NO-OP ... future for UI
     :param alternate_role_name:
-        Override the enumerated role name when the repo does
-        not conform to the ansible-role-<name> convention.
+        NO-OP ... future for UI
+    :param alternate_clone_url:
+        Override the enumerated clone url for the repo.
+        Only used for testing right now.
+    :param superuser_can_create_namespaces:
+        If the function is called by a superuser, it still
+        won't be allowed to create namespaces on the fly
+        without this being set to True
 
     This function attempts to clone the github repository to a
     temporary directory and uses galaxy-importer functions to
@@ -324,6 +335,8 @@ def legacy_role_import(
     logger.info(f'github_user: {github_user}')
     logger.info(f'github_repo: {github_repo}')
     logger.info(f'github_reference: {github_reference}')
+    logger.info(f'alternate_clone_url: {alternate_clone_url}')
+    logger.info(f'alternate_namespace_name: {alternate_namespace_name}')
     logger.info(f'alternate_role_name: {alternate_role_name}')
     logger.info('')
 
@@ -350,6 +363,9 @@ def legacy_role_import(
         )
     logger.info('')
 
+    if alternate_clone_url:
+        clone_url = alternate_clone_url
+
     # the user should have a legacy and v3 namespace if they logged in ...
     namespace = LegacyNamespace.objects.filter(name=real_namespace_name).first()
     if not namespace:
@@ -419,10 +435,65 @@ def legacy_role_import(
         logger.info('')
 
         logger.info('===== PROCESSING LOADER RESULTS ====')
+
+        # Allow the meta/main.yml to define the destination namespace ...
+        new_namespace_name = namespace.name
+        if alternate_namespace_name:
+            new_namespace_name = alternate_namespace_name
+            logger.info(f'overriding namespace name via parameter: {alternate_namespace_name}')
+        elif result['metadata']['galaxy_info'].get('namespace'):
+            new_namespace_name = result['metadata']['galaxy_info'].get('namespace')
+            logger.info(f'overriding namespace name via metadata: {new_namespace_name}')
+
+        if namespace.name != new_namespace_name:
+
+            # does it exist and is the user an owner?
+            found = LegacyNamespace.objects.filter(name=new_namespace_name).first()
+            if found:
+                provider = found.namespace
+                if provider:
+                    owners = get_v3_namespace_owners(provider)
+                else:
+                    owners = []
+
+                if not provider and not request_user.is_superuser:
+                    logger.error(
+                        f'legacy namespace {found} has no provider namespace to define owners'
+                    )
+                    raise Exception('permission denied')
+
+                if request_user not in owners and not request_user.is_superuser:
+                    logger.error(
+                        f'{request_user.username} is not an owner'
+                        f' of provider namespace {provider.name}'
+                    )
+                    raise Exception('permission denied')
+
+            else:
+                # we need to create the namespace but only if allowed ...
+                if not request_user.is_superuser or not superuser_can_create_namespaces:
+                    logger.error(
+                        f'legacy namespace {new_namespace_name} does not exist'
+                    )
+                    raise Exception('permission denied')
+
+                logger.info('creating legacy namespace {new_namespace_name}')
+                namespace, _ = LegacyNamespace.objects.get_or_create(name=new_namespace_name)
+
+            namespace, _ = LegacyNamespace.objects.get_or_create(name=new_namespace_name)
+            real_role = None
+
         # munge the role name via an order of precedence
-        role_name = result["metadata"]["galaxy_info"].get("role_name") or \
-            alternate_role_name or github_repo.replace("ansible-role-", "")
+        if alternate_role_name:
+            role_name = alternate_role_name
+        elif result["metadata"]["galaxy_info"].get("role_name"):
+            role_name = result["metadata"]["galaxy_info"]["role_name"]
+        else:
+            role_name = github_repo.replace("ansible-role-", "")
+
         logger.info(f'enumerated role name {role_name}')
+        if real_role and real_role.name != role_name:
+            real_role = None
 
         galaxy_info = result["metadata"]["galaxy_info"]
         new_full_metadata = {
@@ -487,7 +558,7 @@ def legacy_role_import(
 
     logger.info('')
     logger.info('Import completed')
-    return True
+    return this_role
 
 
 def legacy_sync_from_upstream(

galaxy_ng/app/api/v1/viewsets/roles.py

@@ -263,18 +263,13 @@ def create(self, request):
         to the pulp tasking system and translating the
         task UUID to an integer to retain v1 compatibility.
         """
-        serializer_class = LegacyImportSerializer
-        serializer = serializer_class(data=request.data)
+        serializer = LegacyImportSerializer(data=request.data)
         serializer.is_valid(raise_exception=True)
         kwargs = dict(serializer.validated_data)
 
         # tell the defered task who started this job
         kwargs['request_username'] = request.user.username
 
-        # synthetically create the name for the response
-        role_name = kwargs.get('alternate_role_name') or \
-            kwargs['github_repo'].replace('ansible-role-', '')
-
         task_id, pulp_id = self.legacy_dispatch(legacy_role_import, kwargs=kwargs)
 
         return Response({
@@ -284,9 +279,12 @@ def create(self, request):
                 'github_user': kwargs['github_user'],
                 'github_repo': kwargs['github_repo'],
                 'github_reference': kwargs.get('github_reference'),
+                'alternate_namespace_name': kwargs.get('alternate_namespace_name'),
+                'alternate_role_name': kwargs.get('alternate_role_name'),
+                'alternate_clone_url': kwargs.get('alternate_clone_url'),
                 'summary_fields': {
                     'role': {
-                        'name': role_name
+                        'name': None
                     }
                 }
             }]

galaxy_ng/app/utils/galaxy.py

@@ -443,12 +443,13 @@ def upstream_role_iterator(
     pagenum = 0
     role_count = 0
     while next_url:
-        logger.info(f'fetch {pagenum} {next_url} role-count:{role_count}')
+        logger.info(f'fetch {pagenum} {next_url} role-count:{role_count} ...')
 
         page = safe_fetch(next_url)
 
         # Some upstream pages return ISEs for whatever reason.
         if page.status_code >= 500:
+            logger.error(f'{next_url} returned 500ISE. incrementing the page manually')
             if 'page=' in next_url:
                 next_url = next_url.replace(f'page={pagenum}', f'page={pagenum+1}')
             else:
@@ -520,9 +521,16 @@ def upstream_role_iterator(
         if ds.get('next'):
             next_url = ds['next']
         elif ds.get('next_link'):
-            next_url = _baseurl + ds['next_link']
+            next_url = ds['next_link']
         else:
             # break if no next page
             break
 
+        api_prefix = '/api/v1'
+        if not next_url.startswith(_baseurl):
+            if not next_url.startswith(api_prefix):
+                next_url = _baseurl + api_prefix + next_url
+            else:
+                next_url = _baseurl + next_url
+
         pagenum += 1

galaxy_ng/tests/integration/community/test_role_import_overrides.py

@@ -0,0 +1,105 @@
+"""test_community.py - Tests related to the community featureset.
+"""
+
+import pytest
+
+from ..utils import (
+    get_client,
+)
+from ..utils.legacy import (
+    cleanup_social_user,
+    LegacyRoleGitRepoBuilder,
+    wait_for_v1_task
+)
+
+pytestmark = pytest.mark.qa  # noqa: F821
+
+
+@pytest.mark.parametrize(
+    'spec',
+    [
+        {
+            'namespace': 'foo',
+            'name': 'bar',
+            'github_user': 'foo',
+            'github_repo': 'bar',
+            'meta_namespace': None,
+            'meta_name': None,
+            'alternate_namespace_name': 'foo',
+            'alternate_role_name': 'bar',
+        },
+        {
+            'namespace': 'jim32',
+            'name': 'bob32',
+            'github_user': 'jim',
+            'github_repo': 'bob',
+            'meta_namespace': 'jim32',
+            'meta_name': 'bob32',
+            'alternate_namespace_name': None,
+            'alternate_role_name': None,
+        }
+
+    ]
+)
+@pytest.mark.deployment_community
+def test_role_import_overrides(ansible_config, spec):
+    """" Validate setting namespace in meta/main.yml does the right thing """
+
+    admin_config = ansible_config("admin")
+    admin_client = get_client(
+        config=admin_config,
+        request_token=False,
+        require_auth=True
+    )
+
+    # all required namespaces ...
+    ns_names = [
+        spec['namespace'],
+        spec['github_user'],
+        spec['alternate_namespace_name'],
+        spec['meta_namespace']
+    ]
+    ns_names = sorted(set([x for x in ns_names if x]))
+
+    # cleanup
+    for ns_name in ns_names:
+        cleanup_social_user(ns_name, ansible_config)
+        try:
+            admin_client(f'/api/v3/namespaces/{ns_name}/', method='DELETE')
+        except Exception:
+            pass
+
+    # make the namespace(s)
+    for ns_name in ns_names:
+        v1 = admin_client('/api/v1/namespaces/', method='POST', args={'name': ns_name})
+        v3 = admin_client('/api/v3/namespaces/', method='POST', args={'name': ns_name})
+        admin_client(
+            f'/api/v1/namespaces/{v1["id"]}/providers/', method='POST', args={'id': v3['id']}
+        )
+
+    # make a local git repo
+    builder_kwargs = {
+        'namespace': spec['namespace'],
+        'name': spec['name'],
+        'meta_namespace': spec['meta_namespace'],
+        'meta_name': spec['meta_name'],
+    }
+    lr = LegacyRoleGitRepoBuilder(**builder_kwargs)
+
+    # run the import
+    payload = {'alternate_clone_url': lr.role_dir}
+    for key in ['github_user', 'github_repo', 'alternate_namespace_name', 'alternate_role_name']:
+        if spec.get(key):
+            payload[key] = spec[key]
+    resp = admin_client('/api/v1/imports/', method='POST', args=payload)
+    task_id = resp['results'][0]['id']
+    result = wait_for_v1_task(task_id=task_id, api_client=admin_client)
+    assert result['results'][0]['state'] == 'SUCCESS'
+
+    # find the role and check it's attributes ...
+    roles_search = admin_client(f'/api/v1/roles/?namespace={spec["namespace"]}&name={spec["name"]}')
+    assert roles_search['count'] == 1
+    assert roles_search['results'][0]['summary_fields']['namespace']['name'] == spec['namespace']
+    assert roles_search['results'][0]['name'] == spec['name']
+    assert roles_search['results'][0]['github_user'] == spec['github_user']
+    assert roles_search['results'][0]['github_repo'] == spec['github_repo']