diff --git a/CHANGES/2761.feature b/CHANGES/2761.feature new file mode 100644 index 0000000000..b12ae18928 --- /dev/null +++ b/CHANGES/2761.feature @@ -0,0 +1 @@ +Add _ui/v1/tags/collections and _ui/v1/tags/roles endpoints. Add sorting by name and count, and enable filtering by name (exact, partial and startswith match). diff --git a/galaxy_ng/app/api/ui/urls.py b/galaxy_ng/app/api/ui/urls.py index a238a46d28..fdaa1c8511 100644 --- a/galaxy_ng/app/api/ui/urls.py +++ b/galaxy_ng/app/api/ui/urls.py @@ -25,6 +25,10 @@ router.register('distributions', viewsets.DistributionViewSet, basename='distributions') router.register('my-distributions', viewsets.MyDistributionViewSet, basename='my-distributions') +router.register('tags/collections', viewsets.CollectionsTagsViewSet, basename='collections-tags') +router.register('tags/roles', viewsets.RolesTagsViewSet, basename='roles-tags') + + auth_views = [ path("login/", views.LoginView.as_view(), name="auth-login"), path("logout/", views.LogoutView.as_view(), name="auth-logout"), diff --git a/galaxy_ng/app/api/ui/viewsets/__init__.py b/galaxy_ng/app/api/ui/viewsets/__init__.py index 25c100eb12..50e7deac92 100644 --- a/galaxy_ng/app/api/ui/viewsets/__init__.py +++ b/galaxy_ng/app/api/ui/viewsets/__init__.py @@ -10,7 +10,11 @@ ) from .my_namespace import MyNamespaceViewSet from .my_synclist import MySyncListViewSet -from .tags import TagsViewSet +from .tags import ( + TagsViewSet, + CollectionsTagsViewSet, + RolesTagsViewSet +) from .user import UserViewSet, CurrentUserViewSet from .synclist import SyncListViewSet from .root import APIRootView @@ -30,6 +34,8 @@ 'CollectionImportViewSet', 'CollectionRemoteViewSet', 'TagsViewSet', + 'CollectionsTagsViewSet', + 'RolesTagsViewSet', 'CurrentUserViewSet', 'UserViewSet', 'SyncListViewSet', diff --git a/galaxy_ng/app/api/ui/viewsets/tags.py b/galaxy_ng/app/api/ui/viewsets/tags.py index 6a13070250..8021f4d82b 100644 --- a/galaxy_ng/app/api/ui/viewsets/tags.py +++ b/galaxy_ng/app/api/ui/viewsets/tags.py @@ -1,10 +1,16 @@ +from django.db.models import Count +from rest_framework import mixins +from django_filters import filters +from django_filters.rest_framework import DjangoFilterBackend, filterset + from pulp_ansible.app.models import Tag from pulp_ansible.app.serializers import TagSerializer from galaxy_ng.app.api import base as api_base -from galaxy_ng.app.access_control import access_policy - from galaxy_ng.app.api.ui import versioning +from galaxy_ng.app.access_control import access_policy +from galaxy_ng.app.api.v1.models import LegacyRoleTag +from galaxy_ng.app.api.v1.serializers import LegacyRoleTagSerializer class TagsViewSet(api_base.GenericViewSet): @@ -21,3 +27,95 @@ def list(self, request, *args, **kwargs): serializer = self.get_serializer(page, many=True) return self.get_paginated_response(serializer.data) + + +class CollectionTagFilterOrdering(filters.OrderingFilter): + def filter(self, qs, value): + if value is not None and any(v in ["count", "-count"] for v in value): + order = "-" if "-count" in value else "" + + return qs.filter( + ansible_collectionversion__ansible_crossrepositorycollectionversionindex__is_highest=True # noqa: E501 + ).annotate(count=Count('ansible_collectionversion')).order_by(f"{order}count") + + return super().filter(qs, value) + + +class CollectionTagFilter(filterset.FilterSet): + sort = CollectionTagFilterOrdering( + fields=( + ("name", "name"), + ('count', 'count') + ), + ) + + class Meta: + model = Tag + fields = { + "name": ["exact", "icontains", "contains", "startswith"], + } + + +class CollectionsTagsViewSet( + api_base.GenericViewSet, + mixins.ListModelMixin +): + """ + ViewSet for collections' tags within the system. + """ + serializer_class = TagSerializer + permission_classes = [access_policy.TagsAccessPolicy] + versioning_class = versioning.UIVersioning + filter_backends = (DjangoFilterBackend,) + filterset_class = CollectionTagFilter + + queryset = Tag.objects.all() + + def get_queryset(self): + qs = super().get_queryset() + return qs.annotate(count=Count("ansible_collectionversion")) + + +class RoleTagFilterOrdering(filters.OrderingFilter): + def filter(self, qs, value): + if value is not None and any(v in ["count", "-count"] for v in value): + order = "-" if "-count" in value else "" + + return qs.annotate(count=Count('legacyrole')).order_by(f"{order}count") + + return super().filter(qs, value) + + +class RoleTagFilter(filterset.FilterSet): + sort = RoleTagFilterOrdering( + fields=( + ("name", "name"), + ('count', 'count') + ), + ) + + class Meta: + model = LegacyRoleTag + fields = { + "name": ["exact", "icontains", "contains", "startswith"], + } + + +class RolesTagsViewSet( + api_base.GenericViewSet, + mixins.ListModelMixin +): + """ + ViewSet for roles' tags within the system. + Tags can be populated manually by running `django-admin populate-role-tags`. + """ + queryset = LegacyRoleTag.objects.all() + serializer_class = LegacyRoleTagSerializer + permission_classes = [access_policy.TagsAccessPolicy] + versioning_class = versioning.UIVersioning + filter_backends = (DjangoFilterBackend,) + filterset_class = RoleTagFilter + + def get_queryset(self): + qs = super().get_queryset() + return qs.annotate(count=Count("legacyrole")) diff --git a/galaxy_ng/app/api/v1/models.py b/galaxy_ng/app/api/v1/models.py index a263b1718b..ecfe291586 100644 --- a/galaxy_ng/app/api/v1/models.py +++ b/galaxy_ng/app/api/v1/models.py @@ -115,6 +115,16 @@ def __str__(self): return self.name +class LegacyRoleTag(models.Model): + name = models.CharField(max_length=64, unique=True, editable=False) + + def __repr__(self): + return f'' + + def __str__(self): + return self.name + + class LegacyRole(models.Model): """ A legacy v1 role, which is just an index for github. @@ -154,6 +164,8 @@ class LegacyRole(models.Model): default=dict ) + tags = models.ManyToManyField(LegacyRoleTag, editable=False, related_name="legacyrole") + def __repr__(self): return f'' diff --git a/galaxy_ng/app/api/v1/serializers.py b/galaxy_ng/app/api/v1/serializers.py index 893efb3a08..b84a92ae9e 100644 --- a/galaxy_ng/app/api/v1/serializers.py +++ b/galaxy_ng/app/api/v1/serializers.py @@ -6,7 +6,7 @@ from galaxy_ng.app.models.namespace import Namespace from galaxy_ng.app.utils.rbac import get_v3_namespace_owners from galaxy_ng.app.api.v1.models import LegacyNamespace -from galaxy_ng.app.api.v1.models import LegacyRole +from galaxy_ng.app.api.v1.models import LegacyRole, LegacyRoleTag from galaxy_ng.app.api.v1.models import LegacyRoleDownloadCount from galaxy_ng.app.api.v1.utils import sort_versions @@ -602,3 +602,12 @@ class LegacyTaskDetailSerializer(serializers.Serializer): class Meta: model = None fields = ['results'] + + +class LegacyRoleTagSerializer(serializers.ModelSerializer): + + count = serializers.IntegerField(read_only=True) + + class Meta: + model = LegacyRoleTag + fields = ['name', 'count'] diff --git a/galaxy_ng/app/management/commands/populate-role-tags.py b/galaxy_ng/app/management/commands/populate-role-tags.py new file mode 100644 index 0000000000..39ccda6183 --- /dev/null +++ b/galaxy_ng/app/management/commands/populate-role-tags.py @@ -0,0 +1,36 @@ +from gettext import gettext as _ + +import django_guid +from django.core.management.base import BaseCommand + +# from galaxy_ng.app.api.v1.tasks import legacy_sync_from_upstream +from galaxy_ng.app.api.v1.models import LegacyRole, LegacyRoleTag + + +# Set logging_uid, this does not seem to get generated when task called via management command +django_guid.set_guid(django_guid.utils.generate_guid()) + + +class Command(BaseCommand): + """ + Django management command for populating role tags ('_ui/v1/tags/roles/') within the system. + This command is run nightly on galaxy.ansible.com. + """ + + help = _("Populate the 'LegacyRoleTag' model with tags from LegacyRole 'full_metadata__tags'.") + + def handle(self, *args, **options): + created_tags = [] + roles = LegacyRole.objects.all() + for role in roles: + for name in role.full_metadata["tags"]: + tag, created = LegacyRoleTag.objects.get_or_create(name=name) + tag.legacyrole.add(role) + + if created: + created_tags.append(tag) + + self.stdout.write( + "Successfully populated {} tags " + "from {} roles.".format(len(created_tags), len(roles)) + ) diff --git a/galaxy_ng/app/migrations/0043_legacyroletag_legacyrole_tags.py b/galaxy_ng/app/migrations/0043_legacyroletag_legacyrole_tags.py new file mode 100644 index 0000000000..5f18af92da --- /dev/null +++ b/galaxy_ng/app/migrations/0043_legacyroletag_legacyrole_tags.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.6 on 2023-10-25 21:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("galaxy", "0042_namespace_created_namespace_updated"), + ] + + operations = [ + migrations.CreateModel( + name="LegacyRoleTag", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("name", models.CharField(editable=False, max_length=64, unique=True)), + ], + ), + migrations.AddField( + model_name="legacyrole", + name="tags", + field=models.ManyToManyField( + editable=False, related_name="legacyrole", to="galaxy.legacyroletag" + ), + ), + ] diff --git a/galaxy_ng/tests/integration/api/test_ui_paths.py b/galaxy_ng/tests/integration/api/test_ui_paths.py index b0756b9ec6..903c39951a 100644 --- a/galaxy_ng/tests/integration/api/test_ui_paths.py +++ b/galaxy_ng/tests/integration/api/test_ui_paths.py @@ -1,12 +1,16 @@ #!/usr/bin/env python3 import random +import json +import subprocess import pytest + +from orionutils.generator import build_collection from ansible.galaxy.api import GalaxyError from jsonschema import validate as validate_json -from ..constants import DEFAULT_DISTROS +from ..constants import DEFAULT_DISTROS, USERNAME_PUBLISHER from ..schemas import ( schema_collection_import, schema_collection_import_detail, @@ -26,9 +30,21 @@ schema_ui_collection_summary, schema_user, ) -from ..utils import UIClient, generate_unused_namespace, get_client, wait_for_task_ui_client +from ..utils import ( + UIClient, + generate_unused_namespace, + get_client, + wait_for_task_ui_client, + wait_for_task, +) +from ..utils.legacy import ( + clean_all_roles, + wait_for_v1_task +) + from .rbac_actions.utils import ReusableLocalContainer + REGEX_403 = r"HTTP Code: 403" @@ -767,6 +783,136 @@ def test_api_ui_v1_tags(ansible_config): # FIXME - ui tags api does not support POST? +# /api/automation-hub/_ui/v1/tags/collections/ +@pytest.mark.deployment_community +def test_api_ui_v1_tags_collections(ansible_config, upload_artifact): + + config = ansible_config("basic_user") + api_client = get_client(config) + + def build_upload_wait(tags): + artifact = build_collection( + "skeleton", + config={ + "namespace": USERNAME_PUBLISHER, + "tags": tags, + } + ) + resp = upload_artifact(config, api_client, artifact) + resp = wait_for_task(api_client, resp) + + build_upload_wait(["tools", "database", "postgresql"]) + build_upload_wait(["tools", "database", "mysql"]) + build_upload_wait(["tools", "database"]) + build_upload_wait(["tools"]) + + with UIClient(config=config) as uclient: + + # get the response + resp = uclient.get('_ui/v1/tags/collections') + assert resp.status_code == 200 + + ds = resp.json() + validate_json(instance=ds, schema=schema_objectlist) + + resp = uclient.get('_ui/v1/tags/collections?name=tools') + ds = resp.json() + assert len(ds["data"]) == 1 + + # result count should be 2 (mysql, postgresql) + resp = uclient.get('_ui/v1/tags/collections?name__icontains=sql') + ds = resp.json() + assert len(ds["data"]) == 2 + + resp = uclient.get('_ui/v1/tags/collections?name=test123') + ds = resp.json() + assert len(ds["data"]) == 0 + + # verify sort by name is correct + resp = uclient.get('_ui/v1/tags/collections?sort=name') + tags = [tag["name"] for tag in resp.json()["data"]] + assert tags == sorted(tags) + + # verify sort by -count is correct + resp = uclient.get('_ui/v1/tags/collections/?sort=-count') + data = resp.json()["data"] + assert data[0]["name"] == "tools" + assert data[1]["name"] == "database" + + +# /api/automation-hub/_ui/v1/tags/roles/ +@pytest.mark.deployment_community +def test_api_ui_v1_tags_roles(ansible_config): + """Test endpoint's sorting and filtering""" + + def _sync_role(github_user, role_name): + pargs = json.dumps({"github_user": github_user, "role_name": role_name}).encode('utf-8') + resp = api_admin_client('/api/v1/sync/', method='POST', args=pargs) + assert isinstance(resp, dict) + assert resp.get('task') is not None + wait_for_v1_task(resp=resp, api_client=api_admin_client) + + def _populate_tags_cmd(): + proc = subprocess.run( + "django-admin populate-role-tags", + stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True + ) + assert proc.returncode == 0 + + config = ansible_config("basic_user") + + admin_config = ansible_config("admin") + api_admin_client = get_client( + admin_config, + request_token=False, + require_auth=True + ) + + with UIClient(config=config) as uclient: + + # get the response + resp = uclient.get('_ui/v1/tags/roles') + assert resp.status_code == 200 + + ds = resp.json() + validate_json(instance=ds, schema=schema_objectlist) + + # clean all roles ... + clean_all_roles(ansible_config) + + # start the sync + _sync_role("geerlingguy", "docker") + + resp = uclient.get('_ui/v1/tags/roles') + resp.status_code == 200 + assert resp.json()["meta"]["count"] == 0 + + # run command to populate role tags table + _populate_tags_cmd() + + resp = uclient.get('_ui/v1/tags/roles') + resp.status_code == 200 + assert resp.json()["meta"]["count"] > 0 + + # add additional tags to test count + # tags ["docker", "system"] + _sync_role("6nsh", "docker") + # tags ["docker"] + _sync_role("0x28d", "docker_ce") + _populate_tags_cmd() + + resp = uclient.get('_ui/v1/tags/roles?sort=-count') + resp.status_code == 200 + assert resp.json()["meta"]["count"] > 0 + + # test correct count sorting + tags = [tag for tag in uclient.get('_ui/v1/tags/roles').json()["data"]] + + assert sorted(tags, key=lambda r: r["count"], reverse=True)[:2] == resp.json()["data"][:2] + assert resp.json()["data"][0]["name"] == "docker" + assert resp.json()["data"][1]["name"] == "system" + + # /api/automation-hub/_ui/v1/users/ @pytest.mark.deployment_standalone @pytest.mark.api_ui diff --git a/galaxy_ng/tests/unit/app/management/commands/test_populate_role_tags_commands.py b/galaxy_ng/tests/unit/app/management/commands/test_populate_role_tags_commands.py new file mode 100644 index 0000000000..be65b240d5 --- /dev/null +++ b/galaxy_ng/tests/unit/app/management/commands/test_populate_role_tags_commands.py @@ -0,0 +1,45 @@ +from django.core.management import call_command +from django.test import TestCase + +from galaxy_ng.app.api.v1.models import LegacyNamespace, LegacyRole, LegacyRoleTag + + +class TestPopulateRoleTagsCommand(TestCase): + + def _load_role(self, namespace, role, tags): + full_metadata = dict(tags=tags) + ln = LegacyNamespace.objects.get_or_create(name=namespace) + LegacyRole.objects.get_or_create(name=role, namespace=ln[0], full_metadata=full_metadata) + + def setUp(self): + super().setUp() + self._load_role("foo", "bar1", ["database", "network", "postgres"]) + self._load_role("foo", "bar2", ["database", "network"]) + + def test_populate_role_tags_command(self): + call_command('populate-role-tags') + + role_tags = LegacyRoleTag.objects.all() + tag_names = list(role_tags.values_list("name", flat=True)) + + self.assertEqual(3, role_tags.count()) + self.assertEqual(tag_names, ["database", "network", "postgres"]) + + def test_populate_twice_and_expect_same_results(self): + call_command('populate-role-tags') + role_tags_1 = LegacyRoleTag.objects.all() + self.assertEqual(3, role_tags_1.count()) + + call_command('populate-role-tags') + role_tags_2 = LegacyRoleTag.objects.all() + self.assertEqual(role_tags_1.count(), role_tags_2.count()) + + def test_populate_detected_changes(self): + call_command('populate-role-tags') + role_tags = LegacyRoleTag.objects.all() + self.assertEqual(3, role_tags.count()) + + self._load_role("foo", "bar3", ["database", "network", "mysql"]) + call_command('populate-role-tags') + role_tags = LegacyRoleTag.objects.all() + self.assertEqual(4, role_tags.count())