Add support for multiple Azure Container Registries

docs/azure.mdx

@@ -552,6 +552,17 @@ Nextflow uses the following environment variables if the registry credentials ar
 
 Container images from public registries such as Docker Hub can be used without additional configuration, even when a private registry is specified in the `azure.registry` scope. The Docker runtime on Azure Batch VMs automatically uses the appropriate registry based on the container image specified for the task.
 
+To pull images from multiple private registries, provide a list of registry definitions instead of a single block:
+
+```groovy
+azure {
+    registry = [
+        [server: 'myregistry.azurecr.io', userName: '<registry-user>', password: '<registry-password>'],
+        [server: 'otherregistry.azurecr.io', userName: '<other-user>', password: '<other-password>'],
+    ]
+}
+```
+
 ### VM images
 
 When Nextflow creates a pool of compute nodes, it selects:

docs/reference/config.mdx

@@ -605,6 +605,10 @@ The client ID for an Azure [managed identity](https://learn.microsoft.com/en-us/
 
 When `true`, use the system-assigned [managed identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) to authenticate Azure resources. Defaults to environment variable `AZURE_MANAGED_IDENTITY_SYSTEM`.
 
+##### `azure.registry`
+
+The container registry from which to pull the Docker images. Accepts a single registry definition (`server`, `userName`, `password`), or a list of registry definitions to pull images from multiple Azure Container Registries.
+
 ##### `azure.registry.password`
 
 The password to connect to a private container registry.

plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy

@@ -871,24 +871,30 @@ class AzBatchService implements Closeable {
          * https://github.com/MicrosoftDocs/azure-docs/blob/master/articles/batch/batch-docker-container-workloads.md#:~:text=Run%20container%20applications%20on%20Azure,compatible%20containers%20on%20the%20nodes.
          */
         final containerConfig = new ContainerConfiguration(ContainerType.DOCKER_COMPATIBLE)
-        final registryOpts = config.registry()
-
-        if( registryOpts && registryOpts.isConfigured() ) {
-            final containerRegistries = new ArrayList<ContainerRegistryReference>(1)
-            containerRegistries << new ContainerRegistryReference()
-                    .setRegistryServer(registryOpts.server)
-                    .setUsername(registryOpts.userName)
-                    .setPassword(registryOpts.password)
+        final containerRegistries = containerRegistries()
+        if( containerRegistries )
             containerConfig.setContainerRegistries(containerRegistries)
-            log.debug "[AZURE BATCH] Connecting Azure Batch pool to Container Registry '$registryOpts.server'"
-        }
 
         final image = getImage(opts)
 
         new VirtualMachineConfiguration(image.imageReference, image.nodeAgentSkuId)
                 .setContainerConfiguration(containerConfig)
     }
 
+    protected List<ContainerRegistryReference> containerRegistries() {
+        final result = new ArrayList<ContainerRegistryReference>()
+        for( final registryOpts : config.registries() ) {
+            if( !registryOpts.isConfigured() )
+                continue
+            result << new ContainerRegistryReference()
+                    .setRegistryServer(registryOpts.server)
+                    .setUsername(registryOpts.userName)
+                    .setPassword(registryOpts.password)
+            log.debug "[AZURE BATCH] Connecting Azure Batch pool to Container Registry '$registryOpts.server'"
+        }
+        return result
+    }
+
 
     protected BatchStartTask createStartTask(AzStartTaskOpts opts) {
         log.trace "AzStartTaskOpts: ${opts}"

plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzConfig.groovy

@@ -19,6 +19,7 @@ package nextflow.cloud.azure.config
 import groovy.transform.CompileStatic
 import nextflow.Global
 import nextflow.Session
+import nextflow.config.spec.ConfigOption
 import nextflow.config.spec.ConfigScope
 import nextflow.config.spec.ScopeName
 import nextflow.script.dsl.Description
@@ -41,7 +42,11 @@ class AzConfig implements ConfigScope {
 
     private AzBatchOpts batch
 
-    private AzRegistryOpts registry
+    @ConfigOption(types=[Map, List])
+    @Description("""
+        The container registry from which to pull the Docker images. Accepts a single registry definition (`server`, `userName`, `password`), or a list of registry definitions to pull images from multiple Azure Container Registries.
+    """)
+    private final List<AzRegistryOpts> registry
 
     private AzRetryConfig retryPolicy
 
@@ -55,7 +60,7 @@ class AzConfig implements ConfigScope {
     AzConfig(Map azure) {
         this.batch = new AzBatchOpts( (Map)azure.batch ?: Collections.emptyMap() )
         this.storage = new AzStorageOpts( (Map)azure.storage ?: Collections.emptyMap() )
-        this.registry = new AzRegistryOpts( (Map)azure.registry ?: Collections.emptyMap() )
+        this.registry = parseRegistries(azure.registry)
         this.azcopy = new AzCopyOpts( (Map)azure.azcopy ?: Collections.emptyMap() )
         this.retryPolicy = new AzRetryConfig( (Map)azure.retryPolicy ?: Collections.emptyMap() )
         this.activeDirectory = new AzActiveDirectoryOpts((Map) azure.activeDirectory ?: Collections.emptyMap())
@@ -68,14 +73,20 @@ class AzConfig implements ConfigScope {
 
     AzStorageOpts storage() { storage }
 
-    AzRegistryOpts registry() { registry }
+    List<AzRegistryOpts> registries() { registry }
 
     AzRetryConfig retryConfig() { retryPolicy }
 
     AzActiveDirectoryOpts activeDirectory() { activeDirectory }
 
     AzManagedIdentityOpts managedIdentity() { managedIdentity }
 
+    private static List<AzRegistryOpts> parseRegistries(Object value) {
+        if( value instanceof List )
+            return ((List)value).collect { new AzRegistryOpts((Map)it ?: Collections.emptyMap()) }
+        return List.of( new AzRegistryOpts((Map)value ?: Collections.emptyMap()) )
+    }
+
     static AzConfig getConfig(Session session) {
         if( !session )
             throw new IllegalStateException("Missing Nextflow session")

plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzBatchServiceTest.groovy

@@ -26,6 +26,7 @@ import com.azure.compute.batch.models.AllocationState
 import com.azure.compute.batch.models.BatchPool
 import com.azure.compute.batch.models.BatchPoolState
 import com.azure.compute.batch.models.BatchJobCreateContent
+import com.azure.compute.batch.models.ContainerRegistryReference
 import com.azure.compute.batch.models.ElevationLevel
 import com.azure.compute.batch.models.EnvironmentSetting
 import com.azure.compute.batch.models.ResizeError
@@ -1293,4 +1294,62 @@ class AzBatchServiceTest extends Specification {
         poolWithResizeError(resizeErrors: [ new ResizeError(code: 'AllocationTimedout'), new ResizeError(code: 'AccountCoreQuotaReached') ]) | false
         poolWithResizeError(resizeErrors: [ new ResizeError(code: null) ])              | false
     }
+
+    def 'should return no container registries when none are configured' () {
+        given:
+        def exec = createExecutor(new AzConfig([:]))
+        AzBatchService svc = Spy(AzBatchService, constructorArgs:[exec])
+
+        expect:
+        svc.containerRegistries() == []
+    }
+
+    def 'should build a container registry reference for a single registry' () {
+        given:
+        def exec = createExecutor(new AzConfig([registry: [server: 'foo.azurecr.io', userName: 'foo', password: 'bar']]))
+        AzBatchService svc = Spy(AzBatchService, constructorArgs:[exec])
+
+        when:
+        def refs = svc.containerRegistries()
+        then:
+        refs.size() == 1
+        refs[0].registryServer == 'foo.azurecr.io'
+        refs[0].username == 'foo'
+        refs[0].password == 'bar'
+    }
+
+    def 'should build a container registry reference for each configured registry in a list' () {
+        given:
+        def exec = createExecutor(new AzConfig([registry: [
+                [server: 'foo.azurecr.io', userName: 'foo', password: 'p1'],
+                [server: 'bar.azurecr.io', userName: 'bar', password: 'p2'],
+        ]]))
+        AzBatchService svc = Spy(AzBatchService, constructorArgs:[exec])
+
+        when:
+        def refs = svc.containerRegistries()
+        then:
+        refs.size() == 2
+        refs[0].registryServer == 'foo.azurecr.io'
+        refs[0].username == 'foo'
+        refs[0].password == 'p1'
+        refs[1].registryServer == 'bar.azurecr.io'
+        refs[1].username == 'bar'
+        refs[1].password == 'p2'
+    }
+
+    def 'should skip unconfigured registries in a list' () {
+        given:
+        def exec = createExecutor(new AzConfig([registry: [
+                [server: 'docker.io'],
+                [server: 'foo.azurecr.io', userName: 'foo', password: 'p1'],
+        ]]))
+        AzBatchService svc = Spy(AzBatchService, constructorArgs:[exec])
+
+        when:
+        def refs = svc.containerRegistries()
+        then:
+        refs.size() == 1
+        refs[0].registryServer == 'foo.azurecr.io'
+    }
 }