wip cleanup synchronizers

clouddriver-api/src/main/java/com/netflix/spinnaker/clouddriver/security/CredentialsProvider.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2020 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.netflix.spinnaker.clouddriver.security;
+
+import java.util.Set;
+
+public interface CredentialsProvider<T extends AccountCredentials<?>> {
+  Set<T> getAll();
+
+  AccountCredentials getCredentials(String name);
+}

clouddriver-aws/clouddriver-aws.gradle

@@ -21,6 +21,7 @@ dependencies {
   implementation "com.netflix.spinnaker.kork:kork-aws"
   implementation "com.netflix.spinnaker.kork:kork-exceptions"
   implementation "com.netflix.spinnaker.kork:kork-security"
+  implementation "com.netflix.spinnaker.kork:kork-credentials"
   implementation "com.netflix.spinnaker.moniker:moniker"
   implementation "com.squareup.okhttp:okhttp"
   implementation "com.squareup.okhttp:okhttp-apache"

clouddriver-aws/src/main/groovy/com/netflix/spinnaker/clouddriver/aws/agent/CleanupAlarmsAgent.groovy

@@ -29,8 +29,8 @@ import com.netflix.spinnaker.clouddriver.aws.security.AmazonClientProvider
 import com.netflix.spinnaker.clouddriver.aws.security.AmazonCredentials
 import com.netflix.spinnaker.clouddriver.aws.security.NetflixAmazonCredentials
 import com.netflix.spinnaker.clouddriver.cache.CustomScheduledAgent
-import com.netflix.spinnaker.clouddriver.security.AccountCredentialsRepository
 import com.netflix.spinnaker.clouddriver.security.ProviderUtils
+import com.netflix.spinnaker.credentials.CredentialsRepository
 import groovy.util.logging.Slf4j
 import org.joda.time.DateTime
 
@@ -45,20 +45,20 @@ class CleanupAlarmsAgent implements RunnableAgent, CustomScheduledAgent {
   public static final Pattern ALARM_NAME_PATTERN = Pattern.compile(".+-v[0-9]{3}-alarm-.+")
 
   final AmazonClientProvider amazonClientProvider
-  final AccountCredentialsRepository accountCredentialsRepository
+  final CredentialsRepository<? extends NetflixAmazonCredentials> accountCredentialsRepository
   final long pollIntervalMillis
   final long timeoutMillis
   final int daysToLeave
 
 
   CleanupAlarmsAgent(AmazonClientProvider amazonClientProvider,
-                     AccountCredentialsRepository accountCredentialsRepository,
+                     CredentialsRepository<? extends NetflixAmazonCredentials> accountCredentialsRepository,
                      int daysToLeave) {
     this(amazonClientProvider, accountCredentialsRepository, POLL_INTERVAL_MILLIS, DEFAULT_TIMEOUT_MILLIS, daysToLeave)
   }
 
   CleanupAlarmsAgent(AmazonClientProvider amazonClientProvider,
-                     AccountCredentialsRepository accountCredentialsRepository,
+                     CredentialsRepository<? extends NetflixAmazonCredentials> accountCredentialsRepository,
                      long pollIntervalMillis,
                      long timeoutMills,
                      int daysToLeave) {
@@ -120,7 +120,7 @@ class CleanupAlarmsAgent implements RunnableAgent, CustomScheduledAgent {
   }
 
   private Set<NetflixAmazonCredentials> getAccounts() {
-    ProviderUtils.buildThreadSafeSetOfAccounts(accountCredentialsRepository, NetflixAmazonCredentials, AmazonCloudProvider.ID)
+    return accountCredentialsRepository.getAll()
   }
 
   private static Set<String> getAttachedAlarms(AmazonAutoScaling autoScaling) {

clouddriver-aws/src/main/groovy/com/netflix/spinnaker/clouddriver/aws/agent/CleanupDetachedInstancesAgent.groovy

@@ -28,8 +28,8 @@ import com.netflix.spinnaker.clouddriver.aws.security.NetflixAmazonCredentials
 import com.netflix.spinnaker.clouddriver.cache.CustomScheduledAgent
 import com.netflix.spinnaker.clouddriver.aws.deploy.ops.DetachInstancesAtomicOperation
 import com.netflix.spinnaker.clouddriver.aws.provider.AwsCleanupProvider
-import com.netflix.spinnaker.clouddriver.security.AccountCredentialsRepository
 import com.netflix.spinnaker.clouddriver.security.ProviderUtils
+import com.netflix.spinnaker.credentials.CredentialsRepository
 import groovy.util.logging.Slf4j
 
 import java.util.concurrent.TimeUnit
@@ -40,17 +40,17 @@ class CleanupDetachedInstancesAgent implements RunnableAgent, CustomScheduledAge
   public static final long DEFAULT_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(20)
 
   final AmazonClientProvider amazonClientProvider
-  final AccountCredentialsRepository accountCredentialsRepository
+  final CredentialsRepository<? extends NetflixAmazonCredentials> accountCredentialsRepository
   final long pollIntervalMillis
   final long timeoutMillis
 
   CleanupDetachedInstancesAgent(AmazonClientProvider amazonClientProvider,
-                                AccountCredentialsRepository accountCredentialsRepository) {
+                                CredentialsRepository<? extends NetflixAmazonCredentials> accountCredentialsRepository) {
     this(amazonClientProvider, accountCredentialsRepository, DEFAULT_POLL_INTERVAL_MILLIS, DEFAULT_TIMEOUT_MILLIS)
   }
 
   CleanupDetachedInstancesAgent(AmazonClientProvider amazonClientProvider,
-                                AccountCredentialsRepository accountCredentialsRepository,
+                                CredentialsRepository<? extends NetflixAmazonCredentials> accountCredentialsRepository,
                                 long pollIntervalMillis,
                                 long timeoutMills) {
     this.amazonClientProvider = amazonClientProvider
@@ -108,7 +108,7 @@ class CleanupDetachedInstancesAgent implements RunnableAgent, CustomScheduledAge
   }
 
   private Set<NetflixAmazonCredentials> getAccounts() {
-    ProviderUtils.buildThreadSafeSetOfAccounts(accountCredentialsRepository, NetflixAmazonCredentials, AmazonCloudProvider.ID)
+    return accountCredentialsRepository.getAll()
   }
 
   /**

clouddriver-aws/src/main/groovy/com/netflix/spinnaker/clouddriver/aws/deploy/handlers/BasicAmazonDeployHandler.groovy

@@ -46,7 +46,7 @@ import com.netflix.spinnaker.clouddriver.deploy.DeployDescription
 import com.netflix.spinnaker.clouddriver.deploy.DeployHandler
 import com.netflix.spinnaker.clouddriver.deploy.DeploymentResult
 import com.netflix.spinnaker.clouddriver.orchestration.events.CreateServerGroupEvent
-import com.netflix.spinnaker.clouddriver.security.AccountCredentialsRepository
+import com.netflix.spinnaker.credentials.CredentialsRepository
 import com.netflix.spinnaker.kork.dynamicconfig.DynamicConfigService
 import groovy.transform.PackageScope
 import groovy.util.logging.Slf4j
@@ -74,7 +74,7 @@ class BasicAmazonDeployHandler implements DeployHandler<BasicAmazonDeployDescrip
   }
 
   private final RegionScopedProviderFactory regionScopedProviderFactory
-  private final AccountCredentialsRepository accountCredentialsRepository
+  private final CredentialsRepository<? extends NetflixAmazonCredentials> accountCredentialsRepository
   private final AwsConfiguration.AmazonServerGroupProvider amazonServerGroupProvider
   private final AwsConfiguration.DeployDefaults deployDefaults
   private final ScalingPolicyCopier scalingPolicyCopier
@@ -84,7 +84,7 @@ class BasicAmazonDeployHandler implements DeployHandler<BasicAmazonDeployDescrip
   private List<CreateServerGroupEvent> deployEvents = []
 
   BasicAmazonDeployHandler(RegionScopedProviderFactory regionScopedProviderFactory,
-                           AccountCredentialsRepository accountCredentialsRepository,
+                           CredentialsRepository<? extends NetflixAmazonCredentials> accountCredentialsRepository,
                            AwsConfiguration.AmazonServerGroupProvider amazonServerGroupProvider,
                            AwsConfiguration.DeployDefaults deployDefaults,
                            ScalingPolicyCopier scalingPolicyCopier,

clouddriver-aws/src/main/groovy/com/netflix/spinnaker/clouddriver/aws/deploy/ops/actions/ModifyServerGroupLaunchTemplate.java

@@ -33,7 +33,7 @@
 import com.netflix.spinnaker.clouddriver.saga.SagaCommand;
 import com.netflix.spinnaker.clouddriver.saga.flow.SagaAction;
 import com.netflix.spinnaker.clouddriver.saga.models.Saga;
-import com.netflix.spinnaker.clouddriver.security.AccountCredentialsRepository;
+import com.netflix.spinnaker.credentials.CredentialsRepository;
 import java.util.Collections;
 import javax.annotation.Nonnull;
 import lombok.Builder;
@@ -46,12 +46,12 @@
 public class ModifyServerGroupLaunchTemplate
     implements SagaAction<ModifyServerGroupLaunchTemplate.ModifyServerGroupLaunchTemplateCommand> {
   private final BlockDeviceConfig blockDeviceConfig;
-  private final AccountCredentialsRepository credentialsRepository;
+  private final CredentialsRepository<? extends NetflixAmazonCredentials> credentialsRepository;
   private final RegionScopedProviderFactory regionScopedProviderFactory;
 
   public ModifyServerGroupLaunchTemplate(
       BlockDeviceConfig blockDeviceConfig,
-      AccountCredentialsRepository credentialsRepository,
+      CredentialsRepository<? extends NetflixAmazonCredentials> credentialsRepository,
       RegionScopedProviderFactory regionScopedProviderFactory) {
     this.blockDeviceConfig = blockDeviceConfig;
     this.credentialsRepository = credentialsRepository;

clouddriver-aws/src/main/groovy/com/netflix/spinnaker/clouddriver/aws/deploy/ops/actions/PrepareModifyServerGroupLaunchTemplate.java

@@ -20,7 +20,11 @@
 import com.amazonaws.services.autoscaling.model.AutoScalingGroup;
 import com.amazonaws.services.autoscaling.model.LaunchTemplateSpecification;
 import com.amazonaws.services.ec2.AmazonEC2;
-import com.amazonaws.services.ec2.model.*;
+import com.amazonaws.services.ec2.model.LaunchTemplateBlockDeviceMapping;
+import com.amazonaws.services.ec2.model.LaunchTemplateIamInstanceProfileSpecification;
+import com.amazonaws.services.ec2.model.LaunchTemplateInstanceMarketOptions;
+import com.amazonaws.services.ec2.model.LaunchTemplateVersion;
+import com.amazonaws.services.ec2.model.ResponseLaunchTemplateData;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
@@ -37,7 +41,7 @@
 import com.netflix.spinnaker.clouddriver.saga.SagaCommand;
 import com.netflix.spinnaker.clouddriver.saga.flow.SagaAction;
 import com.netflix.spinnaker.clouddriver.saga.models.Saga;
-import com.netflix.spinnaker.clouddriver.security.AccountCredentialsRepository;
+import com.netflix.spinnaker.credentials.CredentialsRepository;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -57,12 +61,12 @@ public class PrepareModifyServerGroupLaunchTemplate
     implements SagaAction<
         PrepareModifyServerGroupLaunchTemplate.PrepareModifyServerGroupLaunchTemplateCommand> {
   private final BlockDeviceConfig blockDeviceConfig;
-  private final AccountCredentialsRepository credentialsRepository;
+  private final CredentialsRepository<? extends NetflixAmazonCredentials> credentialsRepository;
   private final RegionScopedProviderFactory regionScopedProviderFactory;
 
   public PrepareModifyServerGroupLaunchTemplate(
       BlockDeviceConfig blockDeviceConfig,
-      AccountCredentialsRepository credentialsRepository,
+      CredentialsRepository<? extends NetflixAmazonCredentials> credentialsRepository,
       RegionScopedProviderFactory regionScopedProviderFactory) {
     this.blockDeviceConfig = blockDeviceConfig;
     this.credentialsRepository = credentialsRepository;

clouddriver-aws/src/main/groovy/com/netflix/spinnaker/clouddriver/aws/deploy/ops/actions/UpdateAutoScalingGroup.java

@@ -30,7 +30,7 @@
 import com.netflix.spinnaker.clouddriver.saga.SagaCommand;
 import com.netflix.spinnaker.clouddriver.saga.flow.SagaAction;
 import com.netflix.spinnaker.clouddriver.saga.models.Saga;
-import com.netflix.spinnaker.clouddriver.security.AccountCredentialsRepository;
+import com.netflix.spinnaker.credentials.CredentialsRepository;
 import javax.annotation.Nonnull;
 import lombok.Builder;
 import lombok.Value;
@@ -42,11 +42,11 @@
 public class UpdateAutoScalingGroup
     implements SagaAction<UpdateAutoScalingGroup.UpdateAutoScalingGroupCommand> {
   private final RegionScopedProviderFactory regionScopedProviderFactory;
-  private final AccountCredentialsRepository credentialsRepository;
+  private final CredentialsRepository<? extends NetflixAmazonCredentials> credentialsRepository;
 
   public UpdateAutoScalingGroup(
       RegionScopedProviderFactory regionScopedProviderFactory,
-      AccountCredentialsRepository credentialsRepository) {
+      CredentialsRepository<? extends NetflixAmazonCredentials> credentialsRepository) {
     this.regionScopedProviderFactory = regionScopedProviderFactory;
     this.credentialsRepository = credentialsRepository;
   }

clouddriver-aws/src/main/groovy/com/netflix/spinnaker/clouddriver/aws/deploy/ops/securitygroup/SecurityGroupLookupFactory.groovy

@@ -17,36 +17,23 @@
 package com.netflix.spinnaker.clouddriver.aws.deploy.ops.securitygroup
 
 import com.amazonaws.services.ec2.AmazonEC2
-import com.amazonaws.services.ec2.model.AuthorizeSecurityGroupIngressRequest
-import com.amazonaws.services.ec2.model.CreateSecurityGroupRequest
-import com.amazonaws.services.ec2.model.CreateTagsRequest
-import com.amazonaws.services.ec2.model.DeleteTagsRequest
-import com.amazonaws.services.ec2.model.DescribeSecurityGroupsRequest
-import com.amazonaws.services.ec2.model.DescribeTagsRequest
-import com.amazonaws.services.ec2.model.Filter
-import com.amazonaws.services.ec2.model.IpPermission
-import com.amazonaws.services.ec2.model.RevokeSecurityGroupIngressRequest
-import com.amazonaws.services.ec2.model.SecurityGroup
-import com.amazonaws.services.ec2.model.Tag
-import com.amazonaws.services.ec2.model.DescribeTagsResult
-import com.amazonaws.services.ec2.model.TagDescription
-import com.amazonaws.services.ec2.model.UpdateSecurityGroupRuleDescriptionsIngressRequest
+import com.amazonaws.services.ec2.model.*
 import com.google.common.collect.ImmutableSet
 import com.netflix.spinnaker.clouddriver.aws.deploy.description.UpsertSecurityGroupDescription
 import com.netflix.spinnaker.clouddriver.aws.security.AmazonClientProvider
 import com.netflix.spinnaker.clouddriver.aws.security.NetflixAmazonCredentials
-import com.netflix.spinnaker.clouddriver.security.AccountCredentialsRepository
+import com.netflix.spinnaker.credentials.CredentialsRepository
 import com.netflix.spinnaker.kork.core.RetrySupport
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 
 class SecurityGroupLookupFactory {
 
   private final AmazonClientProvider amazonClientProvider
-  private final AccountCredentialsRepository accountCredentialsRepository
+  private final CredentialsRepository<? extends NetflixAmazonCredentials> accountCredentialsRepository
 
   SecurityGroupLookupFactory(AmazonClientProvider amazonClientProvider,
-                             AccountCredentialsRepository accountCredentialsRepository) {
+                             CredentialsRepository<? extends NetflixAmazonCredentials> accountCredentialsRepository) {
     this.amazonClientProvider = amazonClientProvider
     this.accountCredentialsRepository = accountCredentialsRepository
   }

clouddriver-aws/src/main/groovy/com/netflix/spinnaker/clouddriver/aws/provider/AwsProvider.groovy

@@ -19,13 +19,13 @@ package com.netflix.spinnaker.clouddriver.aws.provider
 import com.netflix.spinnaker.cats.agent.Agent
 import com.netflix.spinnaker.cats.agent.AgentSchedulerAware
 import com.netflix.spinnaker.cats.cache.Cache
+import com.netflix.spinnaker.clouddriver.aws.data.Keys
 import com.netflix.spinnaker.clouddriver.aws.security.NetflixAmazonCredentials
 import com.netflix.spinnaker.clouddriver.cache.KeyParser
 import com.netflix.spinnaker.clouddriver.cache.SearchableProvider
-import com.netflix.spinnaker.clouddriver.eureka.provider.agent.EurekaAwareProvider
-import com.netflix.spinnaker.clouddriver.security.AccountCredentialsRepository
-import com.netflix.spinnaker.clouddriver.aws.data.Keys
 import com.netflix.spinnaker.clouddriver.core.provider.agent.HealthProvidingCachingAgent
+import com.netflix.spinnaker.clouddriver.eureka.provider.agent.EurekaAwareProvider
+import com.netflix.spinnaker.credentials.CredentialsRepository
 
 import static com.netflix.spinnaker.clouddriver.core.provider.agent.Namespace.*
 
@@ -35,7 +35,7 @@ class AwsProvider extends AgentSchedulerAware implements SearchableProvider, Eur
 
   final KeyParser keyParser = new Keys()
 
-  final AccountCredentialsRepository accountCredentialsRepository
+  final CredentialsRepository<? extends NetflixAmazonCredentials> accountCredentialsRepository
 
   final Set<String> defaultCaches = [
     LOAD_BALANCERS.ns,
@@ -58,7 +58,7 @@ class AwsProvider extends AgentSchedulerAware implements SearchableProvider, Eur
   final Collection<Agent> agents
   private Collection<HealthProvidingCachingAgent> healthAgents
 
-  AwsProvider(AccountCredentialsRepository accountCredentialsRepository, Collection<Agent> agents) {
+  AwsProvider(CredentialsRepository<? extends NetflixAmazonCredentials> accountCredentialsRepository, Collection<Agent> agents) {
     this.agents = agents
     this.accountCredentialsRepository = accountCredentialsRepository
     synchronizeHealthAgents()

clouddriver-aws/src/main/groovy/com/netflix/spinnaker/clouddriver/aws/provider/agent/AbstractAmazonLoadBalancerCachingAgent.groovy

@@ -129,7 +129,7 @@ abstract class AbstractAmazonLoadBalancerCachingAgent implements CachingAgent, O
   abstract CacheResult loadDataInternal(ProviderCache providerCache)
 
   @Override
-  Collection<Map<String, ?>> pendingOnDemandRequests(ProviderCache providerCache) {
+  Collection<Map<String, Object>> pendingOnDemandRequests(ProviderCache providerCache) {
     return []
   }
 

clouddriver-aws/src/main/groovy/com/netflix/spinnaker/clouddriver/aws/provider/agent/AmazonApplicationLoadBalancerCachingAgent.groovy

@@ -382,7 +382,7 @@ class AmazonApplicationLoadBalancerCachingAgent extends AbstractAmazonLoadBalanc
   }
 
   @Override
-  Collection<Map<String, ?>> pendingOnDemandRequests(ProviderCache providerCache) {
+  Collection<Map<String, Object>> pendingOnDemandRequests(ProviderCache providerCache) {
     Collection<String> keys = providerCache.filterIdentifiers(
       ON_DEMAND.ns,
       Keys.getLoadBalancerKey("*", "*", "*", "*", "*")

clouddriver-aws/src/main/groovy/com/netflix/spinnaker/clouddriver/aws/provider/agent/AmazonInstanceTypeCachingAgent.java

@@ -32,8 +32,9 @@
 import com.netflix.spinnaker.clouddriver.aws.cache.Keys;
 import com.netflix.spinnaker.clouddriver.aws.provider.AwsInfrastructureProvider;
 import com.netflix.spinnaker.clouddriver.aws.security.AmazonCredentials;
+import com.netflix.spinnaker.clouddriver.aws.security.NetflixAmazonCredentials;
 import com.netflix.spinnaker.clouddriver.security.AccountCredentials;
-import com.netflix.spinnaker.clouddriver.security.AccountCredentialsRepository;
+import com.netflix.spinnaker.credentials.CredentialsRepository;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
@@ -45,6 +46,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -65,22 +67,24 @@ public class AmazonInstanceTypeCachingAgent implements CachingAgent {
 
   // https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/us-west-2/index.json
   private final String region;
-  private final AccountCredentialsRepository accountCredentialsRepository;
+  private final CredentialsRepository<? extends NetflixAmazonCredentials>
+      accountCredentialsRepository;
   private final URI pricingUri;
   private final HttpHost pricingHost;
   private final HttpClient httpClient;
   private final ObjectMapper objectMapper =
       new ObjectMapper().disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
 
   public AmazonInstanceTypeCachingAgent(
-      String region, AccountCredentialsRepository accountCredentialsRepository) {
+      String region,
+      CredentialsRepository<? extends NetflixAmazonCredentials> accountCredentialsRepository) {
     this(region, accountCredentialsRepository, HttpClients.createDefault());
   }
 
   // VisibleForTesting
   AmazonInstanceTypeCachingAgent(
       String region,
-      AccountCredentialsRepository accountCredentialsRepository,
+      CredentialsRepository<? extends NetflixAmazonCredentials> accountCredentialsRepository,
       HttpClient httpClient) {
     this.region = region;
     this.accountCredentialsRepository = accountCredentialsRepository;
@@ -102,7 +106,7 @@ public CacheResult loadData(ProviderCache providerCache) {
     try {
       Set<String> matchingAccounts =
           accountCredentialsRepository.getAll().stream()
-              .filter(AmazonCredentials.class::isInstance)
+              .filter(Objects::nonNull)
               .map(AmazonCredentials.class::cast)
               .filter(ac -> ac.getRegions().stream().anyMatch(r -> region.equals(r.getName())))
               .map(AccountCredentials::getName)