Merge pull request #1515 from RoadieHQ/21550-allow-more-dynamic-provi…

…der-configuration Make AWS providers more configurable
RoadieHQ · Jul 30, 2024 · dd0b0a6 · dd0b0a6
2 parents 49003e4 + 315a822
commit dd0b0a6
Show file tree

Hide file tree

Showing 14 changed files with 296 additions and 108 deletions.
diff --git a/.changeset/slow-kings-hunt.md b/.changeset/slow-kings-hunt.md
@@ -0,0 +1,5 @@
+---
+'@roadiehq/catalog-backend-module-aws': major
+---
+
+Modify AWS providers to allow fully runtime customizable dynamic run configuration. Change provider naming convention, remove existing region argument.
diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableProvider.ts
@@ -27,6 +27,8 @@ import {
   relationshipsFromTags,
 } from '../utils/tags';
 import { CatalogApi } from '@backstage/catalog-client';
+import { DynamicAccountConfig } from '../types';
+import { duration } from '../utils/timer';
 
 /**
  * Provides entities from AWS DynamoDB service.
@@ -56,29 +58,31 @@ export class AWSDynamoDbTableProvider extends AWSEntityProvider {
   }
 
   getProviderName(): string {
-    return `aws-dynamo-db-table-${this.accountId}-${this.providerId ?? 0}`;
+    return `aws-dynamo-db-table-${this.providerId ?? 0}`;
   }
 
-  private async getDdb() {
+  private async getDdb(dynamicAccountConfig?: DynamicAccountConfig) {
     const credentials = this.useTemporaryCredentials
-      ? this.getCredentials()
+      ? this.getCredentials(dynamicAccountConfig)
       : await this.getCredentialsProvider();
     return this.useTemporaryCredentials
       ? new DynamoDB({ credentials })
       : new DynamoDB(credentials);
   }
 
-  async run(): Promise<void> {
+  async run(dynamicAccountConfig?: DynamicAccountConfig): Promise<void> {
     if (!this.connection) {
       throw new Error('Not initialized');
     }
+    const startTimestamp = process.hrtime();
+    const { accountId } = this.getParsedConfig(dynamicAccountConfig);
     const groups = await this.getGroups();
 
-    const defaultAnnotations = await this.buildDefaultAnnotations(this.region);
-    const ddb = await this.getDdb();
-    this.logger.info(
-      `Retrieving all DynamoDB tables for account ${this.accountId}`,
+    const defaultAnnotations = await this.buildDefaultAnnotations(
+      dynamicAccountConfig,
     );
+    const ddb = await this.getDdb(dynamicAccountConfig);
+    this.logger.info(`Retrieving all DynamoDB tables for account ${accountId}`);
 
     const paginatorConfig = {
       client: ddb,
@@ -141,5 +145,10 @@ export class AWSDynamoDbTableProvider extends AWSEntityProvider {
         locationKey: this.getProviderName(),
       })),
     });
+
+    this.logger.info(
+      `Finished providing ${ddbComponents.length} DynamoDB tables for account ${accountId}`,
+      { run_duration: duration(startTimestamp) },
+    );
   }
 }
diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSEC2Provider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSEC2Provider.ts
@@ -27,6 +27,8 @@ import {
   relationshipsFromTags,
 } from '../utils/tags';
 import { CatalogApi } from '@backstage/catalog-client';
+import { DynamicAccountConfig } from '../types';
+import { duration } from '../utils/timer';
 
 /**
  * Provides entities from AWS Elastic Compute Cloud.
@@ -56,31 +58,35 @@ export class AWSEC2Provider extends AWSEntityProvider {
   }
 
   getProviderName(): string {
-    return `aws-ec2-provider-${this.accountId}-${this.providerId ?? 0}`;
+    return `aws-ec2-provider-${this.providerId ?? 0}`;
   }
 
-  private async getEc2(discoveryRegion: string) {
+  private async getEc2(dynamicAccountConfig?: DynamicAccountConfig) {
+    const { region } = this.getParsedConfig(dynamicAccountConfig);
     const credentials = this.useTemporaryCredentials
-      ? this.getCredentials()
+      ? this.getCredentials(dynamicAccountConfig)
       : await this.getCredentialsProvider();
     return this.useTemporaryCredentials
-      ? new EC2({ credentials, region: discoveryRegion })
+      ? new EC2({ credentials, region: region })
       : new EC2(credentials);
   }
 
-  async run(region?: string): Promise<void> {
-    const discoveryRegion = region ?? this.region;
+  async run(dynamicAccountConfig?: DynamicAccountConfig): Promise<void> {
     if (!this.connection) {
       throw new Error('Not initialized');
     }
+    const startTimestamp = process.hrtime();
+
+    const { region, accountId } = this.getParsedConfig(dynamicAccountConfig);
     const groups = await this.getGroups();
 
-    this.logger.info(`Providing ec2 resources from aws: ${this.accountId}`);
+    this.logger.info(`Providing ec2 resources from aws: ${accountId}`);
     const ec2Resources: ResourceEntity[] = [];
 
-    const ec2 = await this.getEc2(discoveryRegion);
+    const ec2 = await this.getEc2(dynamicAccountConfig);
 
-    const defaultAnnotations = this.buildDefaultAnnotations(discoveryRegion);
+    const defaultAnnotations =
+      this.buildDefaultAnnotations(dynamicAccountConfig);
 
     const instances = await ec2.describeInstances({
       Filters: [{ Name: 'instance-state-name', Values: ['running'] }],
@@ -90,7 +96,7 @@ export class AWSEC2Provider extends AWSEntityProvider {
       if (reservation.Instances) {
         for (const instance of reservation.Instances) {
           const instanceId = instance.InstanceId;
-          const arn = `arn:aws:ec2:${discoveryRegion}:${this.accountId}:instance/${instanceId}`;
+          const arn = `arn:aws:ec2:${region}:${accountId}:instance/${instanceId}`;
           const consoleLink = new ARN(arn).consoleLink;
           const resource: ResourceEntity = {
             kind: 'Resource',
@@ -137,5 +143,10 @@ export class AWSEC2Provider extends AWSEntityProvider {
         locationKey: this.getProviderName(),
       })),
     });
+
+    this.logger.info(
+      `Finished providing ${ec2Resources.length} EC2 resources from AWS: ${accountId}`,
+      { run_duration: duration(startTimestamp) },
+    );
   }
 }
diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSEKSClusterProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSEKSClusterProvider.ts
@@ -30,6 +30,8 @@ import {
   relationshipsFromTags,
 } from '../utils/tags';
 import { CatalogApi } from '@backstage/catalog-client';
+import { DynamicAccountConfig } from '../types';
+import { duration } from '../utils/timer';
 
 /**
  * Provides entities from AWS EKS Cluster service.
@@ -59,33 +61,35 @@ export class AWSEKSClusterProvider extends AWSEntityProvider {
   }
 
   getProviderName(): string {
-    return `aws-eks-cluster-${this.accountId}-${this.providerId ?? 0}`;
+    return `aws-eks-cluster-${this.providerId ?? 0}`;
   }
 
-  private async getEks(discoveryRegion: string) {
+  private async getEks(dynamicAccountConfig?: DynamicAccountConfig) {
+    const { region } = this.getParsedConfig(dynamicAccountConfig);
     const credentials = this.useTemporaryCredentials
-      ? this.getCredentials()
+      ? this.getCredentials(dynamicAccountConfig)
       : await this.getCredentialsProvider();
     return this.useTemporaryCredentials
-      ? new EKS({ credentials, region: discoveryRegion })
+      ? new EKS({ credentials, region })
       : new EKS(credentials);
   }
 
-  async run(region?: string): Promise<void> {
+  async run(dynamicAccountConfig?: DynamicAccountConfig): Promise<void> {
     if (!this.connection) {
       throw new Error('Not initialized');
     }
-    const discoveryRegion = region ?? this.region;
+
+    const startTimestamp = process.hrtime();
+    const { accountId } = this.getParsedConfig(dynamicAccountConfig);
     const groups = await this.getGroups();
 
-    this.logger.info(
-      `Providing eks cluster resources from aws: ${this.accountId}`,
-    );
+    this.logger.info(`Providing EKS cluster resources from AWS: ${accountId}`);
     const eksResources: ResourceEntity[] = [];
 
-    const eks = await this.getEks(discoveryRegion);
+    const eks = await this.getEks(dynamicAccountConfig);
 
-    const defaultAnnotations = this.buildDefaultAnnotations(discoveryRegion);
+    const defaultAnnotations =
+      this.buildDefaultAnnotations(dynamicAccountConfig);
 
     const paginatorConfig = {
       client: eks,
@@ -144,5 +148,10 @@ export class AWSEKSClusterProvider extends AWSEntityProvider {
         locationKey: this.getProviderName(),
       })),
     });
+
+    this.logger.info(
+      `Finished providing ${eksResources.length} EKS cluster resources from AWS: ${accountId}`,
+      { run_duration: duration(startTimestamp) },
+    );
   }
 }
diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSEntityProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSEntityProvider.ts
@@ -19,7 +19,7 @@ import {
   EntityProviderConnection,
 } from '@backstage/plugin-catalog-node';
 import * as winston from 'winston';
-import { AccountConfig } from '../types';
+import { AccountConfig, DynamicAccountConfig } from '../types';
 import { STS } from '@aws-sdk/client-sts';
 import {
   ANNOTATION_ORIGIN_LOCATION,
@@ -45,7 +45,9 @@ export abstract class AWSEntityProvider implements EntityProvider {
   protected readonly labelValueMapper: LabelValueMapper | undefined;
 
   public abstract getProviderName(): string;
-  public abstract run(region?: string): Promise<void>;
+  public abstract run(
+    dynamicAccountConfig?: DynamicAccountConfig,
+  ): Promise<void>;
 
   protected constructor(
     account: AccountConfig,
@@ -86,19 +88,33 @@ export abstract class AWSEntityProvider implements EntityProvider {
     return labelsFromTags(tags, this.labelValueMapper);
   }
 
-  protected getCredentials() {
-    const arnParse = parseArn(this.account.roleArn ?? this.account.roleName);
-
-    const region = this.region ?? arnParse.region;
+  protected getCredentials(dynamicAccountConfig?: DynamicAccountConfig) {
+    const { roleArn, externalId, region } =
+      this.getParsedConfig(dynamicAccountConfig);
     return fromTemporaryCredentials({
       params: {
-        RoleArn: this.account.roleArn,
-        ExternalId: this.account.externalId,
+        RoleArn: roleArn,
+        ExternalId: externalId,
       },
       clientConfig: region ? { region: region } : undefined,
     });
   }
 
+  protected getParsedConfig(dynamicAccountConfig?: DynamicAccountConfig) {
+    const { roleArn, externalId, region } = dynamicAccountConfig
+      ? dynamicAccountConfig
+      : { roleArn: undefined, externalId: undefined, region: undefined };
+
+    const arn = roleArn ?? this.account.roleArn ?? this.account.roleName;
+    const arnParse = parseArn(arn);
+    return {
+      accountId: arnParse?.accountId,
+      region: region ?? this.region ?? arnParse.region,
+      externalId: externalId ?? this.account.externalId,
+      roleArn: arn,
+    };
+  }
+
   protected async getCredentialsProvider() {
     const awsCredentialProvider =
       await this.credentialsManager.getCredentialProvider({
@@ -127,9 +143,12 @@ export abstract class AWSEntityProvider implements EntityProvider {
     this.connection = connection;
   }
 
-  protected async buildDefaultAnnotations(region: string) {
+  protected async buildDefaultAnnotations(
+    dynamicAccountConfig?: DynamicAccountConfig,
+  ) {
+    const { region, roleArn } = this.getParsedConfig(dynamicAccountConfig);
     const credentials = this.useTemporaryCredentials
-      ? this.getCredentials()
+      ? this.getCredentials(dynamicAccountConfig)
       : await this.getCredentialsProvider();
     const sts = this.useTemporaryCredentials
       ? new STS({ credentials: credentials, region: region })
@@ -138,12 +157,8 @@ export abstract class AWSEntityProvider implements EntityProvider {
     const account = await sts.getCallerIdentity({});
 
     const defaultAnnotations: { [name: string]: string } = {
-      [ANNOTATION_LOCATION]: `${this.getProviderName()}:${
-        this.account.roleName
-      }`,
-      [ANNOTATION_ORIGIN_LOCATION]: `${this.getProviderName()}:${
-        this.account.roleName
-      }`,
+      [ANNOTATION_LOCATION]: `${this.getProviderName()}:${roleArn}`,
+      [ANNOTATION_ORIGIN_LOCATION]: `${this.getProviderName()}:${roleArn}`,
     };
 
     if (account.Account) {

diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMRoleProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMRoleProvider.ts
@@ -28,6 +28,8 @@ import {
   relationshipsFromTags,
 } from '../utils/tags';
 import { CatalogApi } from '@backstage/catalog-client';
+import { DynamicAccountConfig } from '../types';
+import { duration } from '../utils/timer';
 
 /**
  * Provides entities from AWS IAM Role service.
@@ -57,33 +59,35 @@ export class AWSIAMRoleProvider extends AWSEntityProvider {
   }
 
   getProviderName(): string {
-    return `aws-iam-role-${this.accountId}-${this.providerId ?? 0}`;
+    return `aws-iam-role-${this.providerId ?? 0}`;
   }
 
-  private async getIam(discoveryRegion: string) {
+  private async getIam(dynamicAccountConfig?: DynamicAccountConfig) {
+    const { region } = this.getParsedConfig(dynamicAccountConfig);
     const credentials = this.useTemporaryCredentials
-      ? this.getCredentials()
+      ? this.getCredentials(dynamicAccountConfig)
       : await this.getCredentialsProvider();
     return this.useTemporaryCredentials
-      ? new IAM({ credentials, region: discoveryRegion })
+      ? new IAM({ credentials, region })
       : new IAM(credentials);
   }
 
-  async run(region?: string): Promise<void> {
+  async run(dynamicAccountConfig?: DynamicAccountConfig): Promise<void> {
     if (!this.connection) {
       throw new Error('Not initialized');
     }
-    const discoveryRegion = region ?? this.region;
+    const startTimestamp = process.hrtime();
+    const { accountId } = this.getParsedConfig(dynamicAccountConfig);
+
     const groups = await this.getGroups();
 
-    this.logger.info(
-      `Providing iam role resources from aws: ${this.accountId}`,
-    );
+    this.logger.info(`Providing IAM role resources from AWS: ${accountId}`);
     const roleResources: ResourceEntity[] = [];
 
-    const defaultAnnotations = this.buildDefaultAnnotations(discoveryRegion);
+    const defaultAnnotations =
+      this.buildDefaultAnnotations(dynamicAccountConfig);
 
-    const iam = await this.getIam(discoveryRegion);
+    const iam = await this.getIam(dynamicAccountConfig);
 
     const paginatorConfig = {
       client: iam,
@@ -128,5 +132,10 @@ export class AWSIAMRoleProvider extends AWSEntityProvider {
         locationKey: this.getProviderName(),
       })),
     });
+
+    this.logger.info(
+      `Finished providing ${roleResources.length} IAM role resources from AWS: ${accountId}`,
+      { run_duration: duration(startTimestamp) },
+    );
   }
 }