diff --git a/galaxy_ng/app/management/commands/sync-galaxy-collections.py b/galaxy_ng/app/management/commands/sync-galaxy-collections.py index eea74c43e0..8e3b78542c 100644 --- a/galaxy_ng/app/management/commands/sync-galaxy-collections.py +++ b/galaxy_ng/app/management/commands/sync-galaxy-collections.py @@ -10,6 +10,7 @@ from galaxy_ng.app.utils.galaxy import upstream_collection_iterator from galaxy_ng.app.utils.legacy import process_namespace +from pulp_ansible.app.models import CollectionVersion from pulp_ansible.app.models import CollectionRemote from pulp_ansible.app.models import AnsibleRepository from pulp_ansible.app.tasks.collections import sync @@ -40,15 +41,16 @@ def add_arguments(self, parser): parser.add_argument("--namespace", help="find and sync only this namespace name") parser.add_argument("--name", help="find and sync only this name") parser.add_argument("--limit", type=int) - # parser.add_argument("--start_page", type=int) def echo(self, message, style=None): style = style or self.style.SUCCESS - self.stdout.write(style(message)) + # self.stdout.write(style(message)) + logger.info(style(message)) def handle(self, *args, **options): counter = 0 + processed_namespaces = set() for namespace_info, collection_info, collection_versions in upstream_collection_iterator( baseurl=options['baseurl'], collection_namespace=options['namespace'], @@ -61,20 +63,42 @@ def handle(self, *args, **options): + f" versions:{len(collection_versions)}" ) - process_namespace(namespace_info['name'], namespace_info) + if namespace_info['name'] not in processed_namespaces: + process_namespace(namespace_info['name'], namespace_info) + processed_namespaces.add(namespace_info['name']) + + # pulp_ansible sync isn't smart enough to do this ... + should_sync = False + for cvdata in collection_versions: + if not CollectionVersion.objects.filter( + namespace=collection_info['namespace']['name'], + name=collection_info['name'], + version=cvdata['version'] + ).exists(): + should_sync = True + break + if not should_sync: + continue remote = CollectionRemote.objects.filter(name='community').first() repo = AnsibleRepository.objects.filter(name='published').first() + # build a single collection requirements requirements = { 'collections': [ collection_info['namespace']['name'] + '.' + collection_info['name'] ] } requirements_yaml = yaml.dump(requirements) + + # set the remote's requirements remote.requirements_file = requirements_yaml remote.save() + self.echo( + f"dispatching sync for {collection_info['namespace']['name']}" + + f".{collection_info['name']}" + ) self.do_dispatch( remote, repo, @@ -84,7 +108,7 @@ def handle(self, *args, **options): def do_dispatch(self, remote, repository, namespace, name): - # def sync(remote_pk, repository_pk, mirror, optimize) + # dispatch the real pulp_ansible sync code task = dispatch( sync, kwargs={ @@ -97,7 +121,8 @@ def do_dispatch(self, remote, repository, namespace, name): ) while task.state not in TASK_FINAL_STATES: - time.sleep(1) + time.sleep(2) + self.echo(f"Syncing {namespace}.{name} {task.state}") task.refresh_from_db() self.echo(f"Syncing {namespace}.{name} {task.state}") diff --git a/galaxy_ng/app/utils/galaxy.py b/galaxy_ng/app/utils/galaxy.py index 8102509f35..8938a1f3a7 100644 --- a/galaxy_ng/app/utils/galaxy.py +++ b/galaxy_ng/app/utils/galaxy.py @@ -312,6 +312,7 @@ def upstream_collection_iterator( pagenum = 0 collection_count = 0 + next_url = _baseurl + '/api/v2/collections/' while next_url: logger.info(f'fetch {pagenum} {next_url}')