Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Aws test container #134

Merged
merged 5 commits into from
Jun 24, 2021
Merged

Aws test container #134

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
506709e
Integration with localstack and testcontainers
belosh59 Jun 1, 2021
b814d65
Init AWS clients through the configurator
belosh59 Jun 17, 2021
4150a19
Complete test suit for free version of localstack components, also re…
belosh59 Jun 21, 2021
eafb28e
Adding a test logging configuration to make viewing output easier
kickroot Jun 21, 2021
ce755e4
failsafe integration for verify step, cleanup policies
belosh59 Jun 21, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 69 additions & 22 deletions magpie-aws/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
<aws-sigv4-auth-cassandra-java-driver-plugin.version>4.0.3</aws-sigv4-auth-cassandra-java-driver-plugin.version>
<magpie.api.version>0.1.2</magpie.api.version>
<sentry.version>1.7.28</sentry.version>
<mockito.version>3.10.0</mockito.version>
<testcontainer.version>1.15.3</testcontainer.version>
</properties>

<dependencies>
Expand Down Expand Up @@ -207,6 +209,31 @@
<artifactId>cloudwatch</artifactId>
<version>${aws.sdk.version}</version>
</dependency>
<dependency>
<groupId>software.aws.mcs</groupId>
<artifactId>aws-sigv4-auth-cassandra-java-driver-plugin</artifactId>
kickroot marked this conversation as resolved.
Show resolved Hide resolved
<version>${aws-sigv4-auth-cassandra-java-driver-plugin.version}</version>
<exclusions>
<exclusion>
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-json</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.7.0</version>
<scope>compile</scope>
</dependency>

<dependency>
<groupId>com.google.code.findbugs</groupId>
Expand Down Expand Up @@ -242,36 +269,56 @@
<version>1.2.3</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>software.aws.mcs</groupId>
<artifactId>aws-sigv4-auth-cassandra-java-driver-plugin</artifactId>
<version>${aws-sigv4-auth-cassandra-java-driver-plugin.version}</version>
kickroot marked this conversation as resolved.
Show resolved Hide resolved
<exclusions>
<exclusion>
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-json</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
</exclusion>
</exclusions>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.7.0</version>
<scope>compile</scope>
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>${testcontainer.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>localstack</artifactId>
<version>${testcontainer.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>cloudformation</artifactId>
<version>${aws.sdk.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>3.0.0-M5</version>
<configuration>
<runOrder>random</runOrder>
</configuration>
<executions>
<execution>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>properties-maven-plugin</artifactId>
Expand Down
17 changes: 16 additions & 1 deletion magpie-aws/src/main/java/io/openraven/magpie/plugins/aws/discovery/AWSUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.javatuples.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.awscore.client.builder.AwsClientBuilder;
import software.amazon.awssdk.core.exception.SdkClientException;
import software.amazon.awssdk.core.exception.SdkServiceException;
import software.amazon.awssdk.regions.Region;
Expand All @@ -36,6 +37,7 @@

import javax.annotation.Nullable;
import java.io.IOException;
import java.net.URI;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.List;
Expand Down Expand Up @@ -181,7 +183,7 @@ public static Pair<Double, GetMetricStatisticsResponse> getCloudwatchDoubleMetri

public static GetMetricStatisticsResponse getCloudwatchMetricStatistics( String regionID, String namespace, String metric, Statistic statistic, List<Dimension> dimensions) {

try (final CloudWatchClient client = CloudWatchClient.builder().region(Region.of(regionID)).build()) {
try (final CloudWatchClient client = configure(CloudWatchClient.builder(), Region.of(regionID))) {

// The start time is t-minus 2 days (48 hours) because an asset is considered "active" if it's been updated within
// 48hrs, otherwise it is considered "terminated/deleted", so start capturing at the longest possible period
Expand All @@ -200,7 +202,20 @@ public static GetMetricStatisticsResponse getCloudwatchMetricStatistics( String
return client.getMetricStatistics(request);
}
}

public static String getAwsAccountId() {
return StsClient.create().getCallerIdentity().account();
}

public static <BuilderT extends AwsClientBuilder<BuilderT, ClientT>, ClientT> ClientT
configure(AwsClientBuilder<BuilderT, ClientT> builder, Region region) {
// Remap magpie clients to local environment
String magpieAwsEndpoint = System.getProperty("MAGPIE_AWS_ENDPOINT");
if (magpieAwsEndpoint != null) {
builder.endpointOverride(URI.create(magpieAwsEndpoint));
}
// Build for region only
builder.region(region);
return builder.build();
}
}
2 changes: 1 addition & 1 deletion ...aws/src/main/java/io/openraven/magpie/plugins/aws/discovery/services/AthenaDiscovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ public List<Region> getSupportedRegions() {

@Override
public void discover(ObjectMapper mapper, Session session, Region region, Emitter emitter, Logger logger, String account) {
final var client = AthenaClient.builder().region(region).build();
final var client = AWSUtils.configure(AthenaClient.builder(), region);
final String RESOURCE_TYPE = "AWS::Athena::DataCatalog";

try {
Expand Down
2 changes: 1 addition & 1 deletion ...aws/src/main/java/io/openraven/magpie/plugins/aws/discovery/services/BackupDiscovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ public List<Region> getSupportedRegions() {

@Override
public void discover(ObjectMapper mapper, Session session, Region region, Emitter emitter, Logger logger, String account) {
final var client = BackupClient.builder().region(region).build();
final var client = AWSUtils.configure(BackupClient.builder(), region);

final String RESOURCE_TYPE = "AWS::Backup::BackupVault";

Expand Down
3 changes: 2 additions & 1 deletion ...-aws/src/main/java/io/openraven/magpie/plugins/aws/discovery/services/BatchDiscovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import io.openraven.magpie.api.Emitter;
import io.openraven.magpie.api.Session;
import io.openraven.magpie.plugins.aws.discovery.AWSResource;
import io.openraven.magpie.plugins.aws.discovery.AWSUtils;
import io.openraven.magpie.plugins.aws.discovery.DiscoveryExceptions;
import io.openraven.magpie.plugins.aws.discovery.VersionedMagpieEnvelopeProvider;
import org.slf4j.Logger;
Expand All @@ -46,7 +47,7 @@ public List<Region> getSupportedRegions() {

@Override
public void discover(ObjectMapper mapper, Session session, Region region, Emitter emitter, Logger logger, String account) {
final var client = BatchClient.builder().region(region).build();
final var client = AWSUtils.configure(BatchClient.builder(), region);

discoverComputeEnvironments(mapper, session, client, region, emitter, account);
discoverJobQueues(mapper, session, client, region, emitter, account);
Expand Down
2 changes: 1 addition & 1 deletion ...src/main/java/io/openraven/magpie/plugins/aws/discovery/services/CloudFrontDiscovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ public List<Region> getSupportedRegions() {

@Override
public void discover(ObjectMapper mapper, Session session, Region region, Emitter emitter, Logger logger, String account) {
final var client = CloudFrontClient.builder().region(region).build();
final var client = AWSUtils.configure(CloudFrontClient.builder(), region);
String RESOURCE_TYPE = "AWS::CloudFront::Distribution";

try {
Expand Down
2 changes: 1 addition & 1 deletion ...rc/main/java/io/openraven/magpie/plugins/aws/discovery/services/CloudSearchDiscovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public List<Region> getSupportedRegions() {

@Override
public void discover(ObjectMapper mapper, Session session, Region region, Emitter emitter, Logger logger, String account) {
final var client = CloudSearchClient.builder().region(region).build();
final var client = AWSUtils.configure(CloudSearchClient.builder(), region);
final String RESOURCE_TYPE = "AWS::CloudSearch::Domain";

try {
Expand Down
2 changes: 1 addition & 1 deletion ...src/main/java/io/openraven/magpie/plugins/aws/discovery/services/CloudTrailDiscovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public List<Region> getSupportedRegions() {

@Override
public void discover(ObjectMapper mapper, Session session, Region region, Emitter emitter, Logger logger, String account) {
final var client = CloudTrailClient.builder().region(region).build();
final var client = AWSUtils.configure(CloudTrailClient.builder(), region);
final String RESOURCE_TYPE = "AWS::CloudTrail::Trail";

try {
Expand Down
2 changes: 1 addition & 1 deletion ...src/main/java/io/openraven/magpie/plugins/aws/discovery/services/CloudWatchDiscovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public List<Region> getSupportedRegions() {

@Override
public void discover(ObjectMapper mapper, Session session, Region region, Emitter emitter, Logger logger, String account) {
final var client = CloudWatchClient.builder().region(region).build();
final var client = AWSUtils.configure(CloudWatchClient.builder(), region);

discoverAlarms(mapper, session, region, emitter, client, account);
discoverDashboards(mapper, session, region, emitter, client, account);
Expand Down
7 changes: 4 additions & 3 deletions ...s/src/main/java/io/openraven/magpie/plugins/aws/discovery/services/DynamoDbDiscovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import java.util.Map;
import java.util.stream.Collectors;

import static io.openraven.magpie.plugins.aws.discovery.AWSUtils.configure;
import static io.openraven.magpie.plugins.aws.discovery.AWSUtils.getAwsResponse;

public class DynamoDbDiscovery implements AWSDiscovery {
Expand All @@ -54,13 +55,13 @@ public List<Region> getSupportedRegions() {

@Override
public void discover(ObjectMapper mapper, Session session, Region region, Emitter emitter, Logger logger, String account) {
final var client = DynamoDbClient.builder().region(region).build();
final var client = configure(DynamoDbClient.builder(), region);

discoverGlobalTables(mapper, session, region, emitter, client, account);
discoverTables(mapper, session, region, emitter, client, account);
}

private void discoverGlobalTables(ObjectMapper mapper, Session session, Region region, Emitter emitter, DynamoDbClient client, String account) {
protected void discoverGlobalTables(ObjectMapper mapper, Session session, Region region, Emitter emitter, DynamoDbClient client, String account) {
final String RESOURCE_TYPE = "AWS::DynamoDB::GlobalTable";
try {
client.listGlobalTables().globalTables().stream()
Expand All @@ -79,7 +80,7 @@ private void discoverGlobalTables(ObjectMapper mapper, Session session, Region r
}
}

private void discoverTables(ObjectMapper mapper, Session session, Region region, Emitter emitter, DynamoDbClient client, String account) {
protected void discoverTables(ObjectMapper mapper, Session session, Region region, Emitter emitter, DynamoDbClient client, String account) {
final String RESOURCE_TYPE = "AWS::DynamoDB::Table";

try {
Expand Down
2 changes: 1 addition & 1 deletion magpie-aws/src/main/java/io/openraven/magpie/plugins/aws/discovery/services/EBDiscovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ public List<Region> getSupportedRegions() {

@Override
public void discover(ObjectMapper mapper, Session session, Region region, Emitter emitter, Logger logger, String account) {
final var client = ElasticBeanstalkClient.builder().region(region).build();
final var client = AWSUtils.configure(ElasticBeanstalkClient.builder(), region);
final String RESOURCE_TYPE = "AWS::ElasticBeanstalk";

try {
Expand Down
2 changes: 1 addition & 1 deletion ...ie-aws/src/main/java/io/openraven/magpie/plugins/aws/discovery/services/EC2Discovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public class EC2Discovery implements AWSDiscovery {
private static final String SERVICE = "ec2";

public void discover(ObjectMapper mapper, Session session, Region region, Emitter emitter, Logger logger, String account) {
final var client = Ec2Client.builder().region(region).build();
final var client = AWSUtils.configure(Ec2Client.builder(), region);

discoverEc2Instances(mapper, session, client, region, emitter, account);
discoverEIPs(mapper, session, client, region, emitter, account);
Expand Down
2 changes: 1 addition & 1 deletion ...ie-aws/src/main/java/io/openraven/magpie/plugins/aws/discovery/services/ECSDiscovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ public List<Region> getSupportedRegions() {

@Override
public void discover(ObjectMapper mapper, Session session, Region region, Emitter emitter, Logger logger, String account) {
final var client = EcsClient.builder().region(region).build();
final var client = AWSUtils.configure(EcsClient.builder(), region);
final String RESOURCE_TYPE = "AWS::ECS::Cluster";

try {
Expand Down
2 changes: 1 addition & 1 deletion ...ie-aws/src/main/java/io/openraven/magpie/plugins/aws/discovery/services/EFSDiscovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public List<Region> getSupportedRegions() {

@Override
public void discover(ObjectMapper mapper, Session session, Region region, Emitter emitter, Logger logger, String account) {
final var client = EfsClient.builder().region(region).build();
final var client = AWSUtils.configure(EfsClient.builder(), region);
final String RESOURCE_TYPE = "AWS::EFS::FileSystem";

try {
Expand Down
2 changes: 1 addition & 1 deletion ...ie-aws/src/main/java/io/openraven/magpie/plugins/aws/discovery/services/EKSDiscovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public List<Region> getSupportedRegions() {

@Override
public void discover(ObjectMapper mapper, Session session, Region region, Emitter emitter, Logger logger, String account) {
final var client = EksClient.builder().region(region).build();
final var client = AWSUtils.configure(EksClient.builder(), region);
final String RESOURCE_TYPE = "AWS::EKS::Cluster";

try {
Expand Down
2 changes: 1 addition & 1 deletion ...ie-aws/src/main/java/io/openraven/magpie/plugins/aws/discovery/services/ELBDiscovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public List<Region> getSupportedRegions() {

@Override
public void discover(ObjectMapper mapper, Session session, Region region, Emitter emitter, Logger logger, String account) {
final var client = ElasticLoadBalancingClient.builder().region(region).build();
final var client = AWSUtils.configure(ElasticLoadBalancingClient.builder(), region);
final String RESOURCE_TYPE = "AWS::ElasticLoadBalancing::LoadBalancer";

try {
Expand Down
2 changes: 1 addition & 1 deletion ...-aws/src/main/java/io/openraven/magpie/plugins/aws/discovery/services/ELBV2Discovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public List<Region> getSupportedRegions() {

@Override
public void discover(ObjectMapper mapper, Session session, Region region, Emitter emitter, Logger logger, String account) {
final var client = ElasticLoadBalancingV2Client.builder().region(region).build();
final var client = AWSUtils.configure(ElasticLoadBalancingV2Client.builder(), region);
final String RESOURCE_TYPE = "AWS::ElasticLoadBalancingV2::LoadBalancer";

try {
Expand Down
2 changes: 1 addition & 1 deletion ...ie-aws/src/main/java/io/openraven/magpie/plugins/aws/discovery/services/EMRDiscovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public List<Region> getSupportedRegions() {

@Override
public void discover(ObjectMapper mapper, Session session, Region region, Emitter emitter, Logger logger, String account) {
final var client = EmrClient.builder().region(region).build();
final var client = AWSUtils.configure(EmrClient.builder(), region);
final String RESOURCE_TYPE = "AWS::EMR::Cluster";

try {
Expand Down
2 changes: 1 addition & 1 deletion magpie-aws/src/main/java/io/openraven/magpie/plugins/aws/discovery/services/ESDiscovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public List<Region> getSupportedRegions() {

@Override
public void discover(ObjectMapper mapper, Session session, Region region, Emitter emitter, Logger logger, String account) {
final var client = ElasticsearchClient.builder().region(region).build();
final var client = AWSUtils.configure(ElasticsearchClient.builder(), region);
final String RESOURCE_TYPE = "AWS::Elasticsearch::Domain";

try {
Expand Down
2 changes: 1 addition & 1 deletion ...rc/main/java/io/openraven/magpie/plugins/aws/discovery/services/ElastiCacheDiscovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public List<Region> getSupportedRegions() {

@Override
public void discover(ObjectMapper mapper, Session session, Region region, Emitter emitter, Logger logger, String account) {
final var client = ElastiCacheClient.builder().region(region).build();
final var client = AWSUtils.configure(ElastiCacheClient.builder(), region);
final String RESOURCE_TYPE = "AWS::ElastiCache::Cluster";

try {
Expand Down
2 changes: 1 addition & 1 deletion ...ie-aws/src/main/java/io/openraven/magpie/plugins/aws/discovery/services/FSXDiscovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public List<Region> getSupportedRegions() {

@Override
public void discover(ObjectMapper mapper, Session session, Region region, Emitter emitter, Logger logger, String account) {
final var client = FSxClient.builder().region(region).build();
final var client = AWSUtils.configure(FSxClient.builder(), region);
final String RESOURCE_TYPE = "AWS::FSx::FileSystem";

try {
Expand Down
2 changes: 1 addition & 1 deletion ...ws/src/main/java/io/openraven/magpie/plugins/aws/discovery/services/GlacierDiscovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public List<Region> getSupportedRegions() {

@Override
public void discover(ObjectMapper mapper, Session session, Region region, Emitter emitter, Logger logger, String account) {
final var client = GlacierClient.builder().region(region).build();
final var client = AWSUtils.configure(GlacierClient.builder(), region);
final String RESOURCE_TYPE = "AWS::Glacier::Vault";

try {
Expand Down
Loading