diff --git a/pom.xml b/pom.xml
index 52b2f18..c2fdc75 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
com.airbnb.billow
billow
- 2.21
+ 2.24
@@ -207,8 +207,8 @@
maven-compiler-plugin
${maven-compiler-plugin.version}
- 1.7
- 1.7
+ 8
+ 8
diff --git a/src/main/java/com/airbnb/billow/AWSDatabase.java b/src/main/java/com/airbnb/billow/AWSDatabase.java
index 5d16a60..ed0e938 100644
--- a/src/main/java/com/airbnb/billow/AWSDatabase.java
+++ b/src/main/java/com/airbnb/billow/AWSDatabase.java
@@ -7,14 +7,12 @@
import com.amazonaws.services.elasticache.model.ReplicationGroup;
import com.amazonaws.services.elasticsearch.model.DescribeElasticsearchDomainRequest;
import com.amazonaws.services.elasticsearch.model.DescribeElasticsearchDomainResult;
-import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.RequiredArgsConstructor;
+import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
@@ -63,20 +61,22 @@
import com.google.common.collect.ImmutableMultimap;
@Slf4j
-@Data
+@ToString
+@EqualsAndHashCode
+@RequiredArgsConstructor
public class AWSDatabase {
- private final ImmutableMultimap ec2Instances;
- private final ImmutableMultimap dynamoTables;
- private final ImmutableMultimap rdsInstances;
- private final ImmutableMultimap ec2SGs;
- private final ImmutableMultimap sqsQueues;
- private final ImmutableMultimap elasticacheClusters;
- private final ImmutableMultimap elasticsearchClusters;
- private final ImmutableList iamUsers;
- private final long timestamp;
+ private TimestampedData> ec2Instances;
+ private TimestampedData> dynamoTables;
+ private TimestampedData> rdsInstances;
+ private TimestampedData> ec2SGs;
+ private TimestampedData> sqsQueues;
+ private TimestampedData> elasticacheClusters;
+ private TimestampedData> elasticsearchClusters;
+ private TimestampedData> iamUsers;
private String awsAccountNumber;
private String awsARNPartition;
+
AWSDatabase(final Map ec2Clients,
final Map rdsClients,
final Map dynamoClients,
@@ -86,19 +86,8 @@ public class AWSDatabase {
final AmazonIdentityManagement iamClient,
final String configAWSAccountNumber,
final String configAWSARNPartition) {
- timestamp = System.currentTimeMillis();
- log.info("Building AWS DB with timestamp {}", timestamp);
-
- log.info("Getting EC2 instances");
- final ImmutableMultimap.Builder ec2InstanceBuilder = new ImmutableMultimap.Builder<>();
- final ImmutableMultimap.Builder dynamoTableBuilder = new ImmutableMultimap.Builder<>();
- final ImmutableMultimap.Builder sqsQueueBuilder = new ImmutableMultimap.Builder<>();
- final ImmutableMultimap.Builder elasticacheClusterBuilder =
- new ImmutableMultimap.Builder<>();
- final ImmutableMultimap.Builder elasticsearchClusterBuilder =
- new ImmutableMultimap.Builder<>();
-
if (configAWSAccountNumber == null) {
+ log.info("No AWS account number given");
awsAccountNumber = "";
} else {
log.info("using account number '{}' from config", configAWSAccountNumber);
@@ -112,15 +101,117 @@ public class AWSDatabase {
awsARNPartition = configAWSARNPartition;
}
- /*
- * IAM keys
- * Put this in the beginning to populate the awsAccountNumber.
- */
+ refreshIamUsers(iamClient);
+ refreshElasticacheClusters(elasticacheClients);
+ refreshElasticsearchClusters(elasticsearchClients);
+ refreshSqsQueues(sqsClients);
+ refreshDynamoTables(dynamoClients);
+ refreshEc2Instances(ec2Clients);
+ refreshEc2SGs(ec2Clients);
+ refreshRdsInstances(rdsClients);
+ log.info("Done building AWS DB");
+ }
+
+ /**
+ * @return the earliest timestamp of all data in the DB
+ */
+ public long getTimestamp() {
+ return Collections.min(Arrays.asList(
+ ec2Instances.getTimestamp(),
+ dynamoTables.getTimestamp(),
+ rdsInstances.getTimestamp(),
+ ec2SGs.getTimestamp(),
+ sqsQueues.getTimestamp(),
+ elasticsearchClusters.getTimestamp(),
+ elasticsearchClusters.getTimestamp(),
+ iamUsers.getTimestamp()
+ ));
+ }
+
+ public long getAgeInMs() {
+ return System.currentTimeMillis() - getTimestamp();
+ }
+
+ /**
+ * Public getters for AWS data to hide the TimestampedData wrapped class from users of this class
+ */
+
+ public ImmutableMultimap getEc2Instances() {
+ return ec2Instances.getData();
+ }
+
+ public ImmutableMultimap getDynamoTables() {
+ return dynamoTables.getData();
+ }
+
+ public ImmutableMultimap getRdsInstances() {
+ return rdsInstances.getData();
+ }
+
+ public ImmutableMultimap getEc2SGs() {
+ return ec2SGs.getData();
+ }
+
+ public ImmutableMultimap getSqsQueues() {
+ return sqsQueues.getData();
+ }
+
+ public ImmutableMultimap getElasticacheClusters() {
+ return elasticacheClusters.getData();
+ }
+
+ public ImmutableList getIamUsers() {
+ return iamUsers.getData();
+ }
+
+ public ImmutableMultimap getElasticsearchClusters() {
+ return elasticsearchClusters.getData();
+ }
+
+ /**
+ * Public API for updating data after initial build
+ */
+
+ public void refreshEc2Instances(final Map ec2Clients) {
+ this.ec2Instances = TimestampedData.withTimestamp(() -> loadEC2Instances(ec2Clients));
+ }
+
+ public void refreshDynamoTables(final Map dynamoClients) {
+ this.dynamoTables = TimestampedData.withTimestamp(() -> loadDynamo(dynamoClients));
+ }
+
+ public void refreshRdsInstances(final Map rdsClients) {
+ this.rdsInstances = TimestampedData.withTimestamp(() -> loadRDS(rdsClients));
+ }
+ public void refreshEc2SGs(final Map ec2Clients) {
+ this.ec2SGs = TimestampedData.withTimestamp(() -> loadEC2SGs(ec2Clients));
+ }
+
+ public void refreshSqsQueues(final Map sqsClients) {
+ this.sqsQueues = TimestampedData.withTimestamp(() -> loadSQS(sqsClients));
+ }
+
+ public void refreshElasticacheClusters(Map elasticacheClients) {
+ this.elasticacheClusters = TimestampedData.withTimestamp(() -> loadElasticache(elasticacheClients));
+ }
+
+ public void refreshIamUsers(final AmazonIdentityManagement iamClient) {
+ this.iamUsers = TimestampedData.withTimestamp(() -> loadIAM(iamClient));
+ }
+
+ public void refreshElasticsearchClusters(final Map elasticsearchClients) {
+ this.elasticsearchClusters = TimestampedData.withTimestamp(() -> loadElasticsearch(elasticsearchClients));
+ }
+
+ /**
+ * Private helper methods for loading data from the AWS API
+ */
+ private ImmutableList loadIAM(final AmazonIdentityManagement iamClient) {
log.info("Getting IAM keys");
final ImmutableList.Builder usersBuilder = new ImmutableList.Builder<>();
- final ListUsersRequest listUsersRequest = new ListUsersRequest();
+ ListUsersRequest listUsersRequest = new ListUsersRequest();
ListUsersResult listUsersResult;
do {
log.debug("Performing IAM request: {}", listUsersRequest);
@@ -141,13 +232,11 @@ public class AWSDatabase {
}
listUsersRequest.setMarker(listUsersResult.getMarker());
} while (listUsersResult.isTruncated());
- this.iamUsers = usersBuilder.build();
-
-
- /*
- * ElasticCache
- */
+ return usersBuilder.build();
+ }
+ private ImmutableMultimap loadElasticache(final Map elasticacheClients) {
+ ImmutableMultimap.Builder elasticacheClusterBuilder = ImmutableMultimap.builder();
for (Map.Entry clientPair : elasticacheClients.entrySet()) {
final String regionName = clientPair.getKey();
final AmazonElastiCacheClient client = clientPair.getValue();
@@ -181,11 +270,11 @@ public class AWSDatabase {
for (CacheCluster cluster : describeCacheClustersResult.getCacheClusters()) {
com.amazonaws.services.elasticache.model.ListTagsForResourceRequest tagsRequest =
- new com.amazonaws.services.elasticache.model.ListTagsForResourceRequest()
- .withResourceName(elasticacheARN(awsARNPartition, regionName, awsAccountNumber, cluster));
+ new com.amazonaws.services.elasticache.model.ListTagsForResourceRequest()
+ .withResourceName(elasticacheARN(awsARNPartition, regionName, awsAccountNumber, cluster));
com.amazonaws.services.elasticache.model.ListTagsForResourceResult tagsResult =
- client.listTagsForResource(tagsRequest);
+ client.listTagsForResource(tagsRequest);
elasticacheClusterBuilder.putAll(regionName, new ElasticacheCluster(cluster, clusterIdToNodeGroupMember.get(cluster.getCacheClusterId()), tagsResult.getTagList()));
cntClusters++;
}
@@ -194,15 +283,14 @@ public class AWSDatabase {
describeCacheClustersRequest.setMarker(describeCacheClustersResult.getMarker());
} while (describeCacheClustersResult.getMarker() != null);
-
-
}
- this.elasticacheClusters = elasticacheClusterBuilder.build();
- /*
- * Elasticsearch
- */
+ return elasticacheClusterBuilder.build();
+ }
+ private ImmutableMultimap loadElasticsearch(final Map elasticsearchClients) {
+ final ImmutableMultimap.Builder elasticsearchClusterBuilder =
+ new ImmutableMultimap.Builder<>();
for (Map.Entry clientPair : elasticsearchClients.entrySet()) {
final String regionName = clientPair.getKey();
final AWSElasticsearchClient client = clientPair.getValue();
@@ -224,12 +312,11 @@ public class AWSDatabase {
log.debug("Found {} Elasticsearch domains in {}", domainInfoList.size(), regionName);
}
- this.elasticsearchClusters = elasticsearchClusterBuilder.build();
-
- /*
- * SQS Queues
- */
+ return elasticsearchClusterBuilder.build();
+ }
+ private ImmutableMultimap loadSQS(final Map sqsClients) {
+ final ImmutableMultimap.Builder sqsQueueBuilder = new ImmutableMultimap.Builder<>();
for (Map.Entry clientPair : sqsClients.entrySet()) {
final String regionName = clientPair.getKey();
final AmazonSQSClient client = clientPair.getValue();
@@ -254,10 +341,10 @@ public class AWSDatabase {
String queueArn = map.get(SQSQueue.ATTR_QUEUE_ARN);
SQSQueue queue = new SQSQueue(url, Long.valueOf(approximateNumberOfMessagesDelayed),
- Long.valueOf(receiveMessageWaitTimeSeconds), Long.valueOf(createdTimestamp),
- Long.valueOf(delaySeconds), Long.valueOf(messageRetentionPeriod), Long.valueOf(maximumMessageSize),
- Long.valueOf(visibilityTimeout), Long.valueOf(approximateNumberOfMessages),
- Long.valueOf(lastModifiedTimestamp), queueArn);
+ Long.valueOf(receiveMessageWaitTimeSeconds), Long.valueOf(createdTimestamp),
+ Long.valueOf(delaySeconds), Long.valueOf(messageRetentionPeriod), Long.valueOf(maximumMessageSize),
+ Long.valueOf(visibilityTimeout), Long.valueOf(approximateNumberOfMessages),
+ Long.valueOf(lastModifiedTimestamp), queueArn);
sqsQueueBuilder.putAll(regionName, queue);
cnt++;
@@ -265,12 +352,11 @@ public class AWSDatabase {
log.debug("Found {} queues in {}", cnt, regionName);
}
- this.sqsQueues = sqsQueueBuilder.build();
-
- /*
- * DynamoDB Tables
- */
+ return sqsQueueBuilder.build();
+ }
+ private ImmutableMultimap loadDynamo(final Map dynamoClients) {
+ final ImmutableMultimap.Builder dynamoTableBuilder = new ImmutableMultimap.Builder<>();
for (Map.Entry clientPair : dynamoClients.entrySet()) {
final String regionName = clientPair.getKey();
final AmazonDynamoDBClient client = clientPair.getValue();
@@ -288,12 +374,12 @@ public class AWSDatabase {
log.debug("Found {} dynamodbs in {}", cnt, regionName);
}
- this.dynamoTables = dynamoTableBuilder.build();
-
- /*
- * EC2 Instances
- */
+ return dynamoTableBuilder.build();
+ }
+ private ImmutableMultimap loadEC2Instances(final Map ec2Clients) {
+ final ImmutableMultimap.Builder ec2InstanceBuilder = new ImmutableMultimap.Builder<>();
+ log.info("Getting EC2 instances");
for (Map.Entry clientPair : ec2Clients.entrySet()) {
final String regionName = clientPair.getKey();
final AmazonEC2Client client = clientPair.getValue();
@@ -307,12 +393,10 @@ public class AWSDatabase {
ec2InstanceBuilder.putAll(regionName, new EC2Instance(instance));
}
}
- this.ec2Instances = ec2InstanceBuilder.build();
-
- /*
- * EC2 security groups
- */
+ return ec2InstanceBuilder.build();
+ }
+ private ImmutableMultimap loadEC2SGs(final Map ec2Clients) {
log.info("Getting EC2 security groups");
final ImmutableMultimap.Builder ec2SGbuilder = new ImmutableMultimap.Builder();
for (Map.Entry clientPair : ec2Clients.entrySet()) {
@@ -322,13 +406,10 @@ public class AWSDatabase {
log.debug("Found {} security groups in {}", securityGroups.size(), regionName);
ec2SGbuilder.putAll(regionName, securityGroups);
}
- this.ec2SGs = ec2SGbuilder.build();
-
-
- /*
- * RDS Instances
- */
+ return ec2SGbuilder.build();
+ }
+ private ImmutableMultimap loadRDS(final Map rdsClients) {
log.info("Getting RDS instances and clusters");
final ImmutableMultimap.Builder rdsBuilder = new ImmutableMultimap.Builder();
@@ -374,36 +455,30 @@ public class AWSDatabase {
List snapshots = new ArrayList<>();
// Get snapshot for masters only.
if (RDSInstance.checkIfMaster(instance, instanceIdToCluster.get(instance.getDBInstanceIdentifier()))) {
- if ("aurora".equals(instance.getEngine()) || "aurora-mysql".equals(instance.getEngine())) {
- DescribeDBClusterSnapshotsRequest snapshotsRequest = new DescribeDBClusterSnapshotsRequest()
- .withDBClusterIdentifier(instance.getDBClusterIdentifier());
- DescribeDBClusterSnapshotsResult snapshotsResult = client.describeDBClusterSnapshots(snapshotsRequest);
- for (DBClusterSnapshot s : snapshotsResult.getDBClusterSnapshots()) {
- snapshots.add(s.getDBClusterSnapshotIdentifier());
- }
- } else {
- DescribeDBSnapshotsRequest snapshotsRequest = new DescribeDBSnapshotsRequest()
- .withDBInstanceIdentifier(instance.getDBInstanceIdentifier());
- DescribeDBSnapshotsResult snapshotsResult = client.describeDBSnapshots(snapshotsRequest);
- for (DBSnapshot s : snapshotsResult.getDBSnapshots()) {
- snapshots.add(s.getDBSnapshotIdentifier());
- }
- }
+ if ("aurora".equals(instance.getEngine()) || "aurora-mysql".equals(instance.getEngine())) {
+ DescribeDBClusterSnapshotsRequest snapshotsRequest = new DescribeDBClusterSnapshotsRequest()
+ .withDBClusterIdentifier(instance.getDBClusterIdentifier());
+ DescribeDBClusterSnapshotsResult snapshotsResult = client.describeDBClusterSnapshots(snapshotsRequest);
+ for (DBClusterSnapshot s : snapshotsResult.getDBClusterSnapshots()) {
+ snapshots.add(s.getDBClusterSnapshotIdentifier());
+ }
+ } else {
+ DescribeDBSnapshotsRequest snapshotsRequest = new DescribeDBSnapshotsRequest()
+ .withDBInstanceIdentifier(instance.getDBInstanceIdentifier());
+ DescribeDBSnapshotsResult snapshotsResult = client.describeDBSnapshots(snapshotsRequest);
+ for (DBSnapshot s : snapshotsResult.getDBSnapshots()) {
+ snapshots.add(s.getDBSnapshotIdentifier());
+ }
+ }
}
rdsBuilder.putAll(regionName, new RDSInstance(instance,
- instanceIdToCluster.get(instance.getDBInstanceIdentifier()), tagsResult.getTagList(), snapshots));
+ instanceIdToCluster.get(instance.getDBInstanceIdentifier()), tagsResult.getTagList(), snapshots));
}
rdsRequest.setMarker(result.getMarker());
} while (result.getMarker() != null);
}
- this.rdsInstances = rdsBuilder.build();
-
- log.info("Done building AWS DB");
- }
-
- public long getAgeInMs() {
- return System.currentTimeMillis() - getTimestamp();
+ return rdsBuilder.build();
}
private String rdsARN(String partition, String regionName, String accountNumber, DBInstance instance) {
diff --git a/src/main/java/com/airbnb/billow/AWSDatabaseHolder.java b/src/main/java/com/airbnb/billow/AWSDatabaseHolder.java
index 7572134..438f7d7 100644
--- a/src/main/java/com/airbnb/billow/AWSDatabaseHolder.java
+++ b/src/main/java/com/airbnb/billow/AWSDatabaseHolder.java
@@ -1,5 +1,6 @@
package com.airbnb.billow;
+import com.google.common.collect.ImmutableList;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@@ -124,17 +125,77 @@ public AWSDatabaseHolder(Config config) {
rebuild();
}
+ /**
+ * Build a fresh version of the DB completely from scratch
+ */
public void rebuild() {
+ log.info(String.format("Using AWS Account number: '%s'", awsAccountNumber));
current = new AWSDatabase(
- ec2Clients,
- rdsClients,
- dynamoDBClients,
- sqsClients,
- elasticacheClients,
- elasticsearchClients,
- iamClient,
- awsAccountNumber,
- awsARNPartition);
+ ec2Clients,
+ rdsClients,
+ dynamoDBClients,
+ sqsClients,
+ elasticacheClients,
+ elasticsearchClients,
+ iamClient,
+ awsAccountNumber,
+ awsARNPartition);
+ }
+
+ /**
+ * Incrementally refresh ec2 instance data
+ */
+ public void refreshEc2Instances() {
+ current.refreshEc2Instances(ec2Clients);
+ }
+
+ /**
+ * Incrementally refresh dyanmo data
+ */
+ public void refreshDynamoTables() {
+ current.refreshDynamoTables(dynamoDBClients);
+ }
+
+ /**
+ * Incrementally refresh rds instance data
+ */
+ public void refreshRdsInstances() {
+ current.refreshRdsInstances(rdsClients);
+ }
+
+ /**
+ * Incrementally refresh EC2 security group data
+ */
+ public void refreshEc2SGs() {
+ current.refreshEc2SGs(ec2Clients);
+ }
+
+ /**
+ * Incrementally refresh sqs data
+ */
+ public void refreshSqsQueues() {
+ current.refreshSqsQueues(sqsClients);
+ }
+
+ /**
+ * Incrementally refresh elasticache data
+ */
+ public void refreshElasticacheClusters() {
+ current.refreshElasticacheClusters(elasticacheClients);
+ }
+
+ /**
+ * Incrementally refresh IAM data
+ */
+ public void refreshIamUsers() {
+ current.refreshIamUsers(iamClient);
+ }
+
+ /**
+ * Incrementally refresh elasticsearch data
+ */
+ public void refreshElasticsearchClusters() {
+ current.refreshElasticsearchClusters(elasticsearchClients);
}
public HealthCheck.Result healthy() {
diff --git a/src/main/java/com/airbnb/billow/Main.java b/src/main/java/com/airbnb/billow/Main.java
index 85dc13d..ddb4131 100644
--- a/src/main/java/com/airbnb/billow/Main.java
+++ b/src/main/java/com/airbnb/billow/Main.java
@@ -1,5 +1,6 @@
package com.airbnb.billow;
+import com.airbnb.billow.jobs.*;
import com.codahale.metrics.CachedGauge;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.MetricRegistry;
@@ -22,6 +23,7 @@
import org.eclipse.jetty.servlet.ServletHolder;
import org.quartz.JobDetail;
import org.quartz.Scheduler;
+import org.quartz.SchedulerException;
import org.quartz.SimpleTrigger;
import org.quartz.impl.StdSchedulerFactory;
@@ -71,23 +73,21 @@ protected Long loadValue() {
metricRegistry.register(databaseAgeMetricName, cacheAgeGauge);
final Scheduler scheduler = StdSchedulerFactory.getDefaultScheduler();
- scheduler.getContext().put(AWSDatabaseHolderRefreshJob.DB_KEY, dbHolder);
- scheduler.getContext().put(AWSDatabaseHolderRefreshJob.START_COUNTER_KEY, metricRegistry.counter(jobStartMetricName));
- scheduler.getContext().put(AWSDatabaseHolderRefreshJob.FAILURE_COUNTER_KEY, metricRegistry.counter(jobFailureMetricName));
- scheduler.getContext().put(AWSDatabaseHolderRefreshJob.SUCCESS_COUNTER_KEY, metricRegistry.counter(jobSuccessMetricName));
+ scheduler.getContext().put(BaseAWSDatabaseHolderRefreshJob.DB_KEY, dbHolder);
+ scheduler.getContext().put(BaseAWSDatabaseHolderRefreshJob.START_COUNTER_KEY, metricRegistry.counter(jobStartMetricName));
+ scheduler.getContext().put(BaseAWSDatabaseHolderRefreshJob.FAILURE_COUNTER_KEY, metricRegistry.counter(jobFailureMetricName));
+ scheduler.getContext().put(BaseAWSDatabaseHolderRefreshJob.SUCCESS_COUNTER_KEY, metricRegistry.counter(jobSuccessMetricName));
scheduler.start();
- final SimpleTrigger trigger = newTrigger().
- withIdentity(AWSDatabaseHolderRefreshJob.NAME).
- startNow().
- withSchedule(simpleSchedule().withIntervalInMilliseconds(refreshRate).repeatForever()).
- build();
+ scheduleJob(scheduler, DynamoRefreshJob.class, refreshRate);
+ scheduleJob(scheduler, Ec2InstanceRefreshJob.class, refreshRate);
+ scheduleJob(scheduler, Ec2SGRefreshJob.class, refreshRate);
+ scheduleJob(scheduler, ElasticacheRefreshJob.class, refreshRate);
+ scheduleJob(scheduler, ElasticsearchRefreshJob.class, refreshRate);
+ scheduleJob(scheduler, IamRefreshJob.class, refreshRate);
+ scheduleJob(scheduler, RdsRefreshJob.class, refreshRate);
+ scheduleJob(scheduler, SqsRefreshJob.class, refreshRate);
- final JobDetail jobDetail = newJob(AWSDatabaseHolderRefreshJob.class).
- withIdentity(AWSDatabaseHolderRefreshJob.NAME).
- build();
-
- scheduler.scheduleJob(jobDetail, trigger);
log.info("Creating age health check");
healthCheckRegistry.register("DB", new HealthCheck() {
@@ -175,4 +175,18 @@ private static void configureConnectors(Server server) {
}
}
}
+
+ private static void scheduleJob(Scheduler scheduler, Class jobClass, Long refreshRate) throws SchedulerException {
+ final SimpleTrigger trigger = newTrigger().
+ withIdentity(jobClass.toString()).
+ startNow().
+ withSchedule(simpleSchedule().withIntervalInMilliseconds(refreshRate).repeatForever()).
+ build();
+
+ final JobDetail jobDetail = newJob(jobClass).
+ withIdentity(jobClass.toString()).
+ build();
+
+ scheduler.scheduleJob(jobDetail, trigger);
+ }
}
diff --git a/src/main/java/com/airbnb/billow/TimestampedData.java b/src/main/java/com/airbnb/billow/TimestampedData.java
new file mode 100644
index 0000000..c44ce2d
--- /dev/null
+++ b/src/main/java/com/airbnb/billow/TimestampedData.java
@@ -0,0 +1,18 @@
+package com.airbnb.billow;
+
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.function.Supplier;
+
+@Data
+@Slf4j
+public class TimestampedData {
+ private final T data;
+ private final long timestamp;
+
+ public static TimestampedData withTimestamp(Supplier supplier) {
+ long timestamp = System.currentTimeMillis();
+ return new TimestampedData(supplier.get(), timestamp);
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/airbnb/billow/AWSDatabaseHolderRefreshJob.java b/src/main/java/com/airbnb/billow/jobs/BaseAWSDatabaseHolderRefreshJob.java
similarity index 75%
rename from src/main/java/com/airbnb/billow/AWSDatabaseHolderRefreshJob.java
rename to src/main/java/com/airbnb/billow/jobs/BaseAWSDatabaseHolderRefreshJob.java
index fb83019..571fcce 100644
--- a/src/main/java/com/airbnb/billow/AWSDatabaseHolderRefreshJob.java
+++ b/src/main/java/com/airbnb/billow/jobs/BaseAWSDatabaseHolderRefreshJob.java
@@ -1,30 +1,36 @@
-package com.airbnb.billow;
+package com.airbnb.billow.jobs;
+import com.airbnb.billow.AWSDatabaseHolder;
import com.codahale.metrics.Counter;
+import lombok.extern.slf4j.Slf4j;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.quartz.SchedulerException;
-public class AWSDatabaseHolderRefreshJob implements Job {
+@Slf4j
+public abstract class BaseAWSDatabaseHolderRefreshJob implements Job {
public static final String DB_KEY = "db";
public static final String FAILURE_COUNTER_KEY = "failure_counter";
public static final String START_COUNTER_KEY = "start_counter";
public static final String SUCCESS_COUNTER_KEY = "success_counter";
- public static final String NAME = "dbRefresh";
@Override
public void execute(JobExecutionContext context) throws JobExecutionException {
try {
increment(START_COUNTER_KEY, context);
((AWSDatabaseHolder) context.getScheduler().getContext().get(DB_KEY)).rebuild();
+ log.info("[job success] {} completed", this.getClass().toString());
increment(SUCCESS_COUNTER_KEY, context);
} catch (SchedulerException e) {
+ log.error("[job failure] {} completed", this.getClass().toString());
increment(FAILURE_COUNTER_KEY, context);
throw new JobExecutionException(e);
}
}
+ abstract void refresh(AWSDatabaseHolder dbHolder);
+
private void increment(String counterName, JobExecutionContext context) {
try {
((Counter) context.getScheduler().getContext().get(counterName)).inc();
diff --git a/src/main/java/com/airbnb/billow/jobs/DynamoRefreshJob.java b/src/main/java/com/airbnb/billow/jobs/DynamoRefreshJob.java
new file mode 100644
index 0000000..d477537
--- /dev/null
+++ b/src/main/java/com/airbnb/billow/jobs/DynamoRefreshJob.java
@@ -0,0 +1,12 @@
+package com.airbnb.billow.jobs;
+
+import com.airbnb.billow.AWSDatabaseHolder;
+
+public class DynamoRefreshJob extends BaseAWSDatabaseHolderRefreshJob {
+ public static final String NAME = "dynamo_job";
+
+ @Override
+ void refresh(AWSDatabaseHolder dbHolder) {
+ dbHolder.refreshDynamoTables();
+ }
+}
diff --git a/src/main/java/com/airbnb/billow/jobs/Ec2InstanceRefreshJob.java b/src/main/java/com/airbnb/billow/jobs/Ec2InstanceRefreshJob.java
new file mode 100644
index 0000000..3edc7ab
--- /dev/null
+++ b/src/main/java/com/airbnb/billow/jobs/Ec2InstanceRefreshJob.java
@@ -0,0 +1,12 @@
+package com.airbnb.billow.jobs;
+
+import com.airbnb.billow.AWSDatabaseHolder;
+
+public class Ec2InstanceRefreshJob extends BaseAWSDatabaseHolderRefreshJob {
+ public static final String NAME = "ec2_instance_job";
+
+ @Override
+ void refresh(AWSDatabaseHolder dbHolder) {
+ dbHolder.refreshEc2Instances();
+ }
+}
diff --git a/src/main/java/com/airbnb/billow/jobs/Ec2SGRefreshJob.java b/src/main/java/com/airbnb/billow/jobs/Ec2SGRefreshJob.java
new file mode 100644
index 0000000..015018d
--- /dev/null
+++ b/src/main/java/com/airbnb/billow/jobs/Ec2SGRefreshJob.java
@@ -0,0 +1,12 @@
+package com.airbnb.billow.jobs;
+
+import com.airbnb.billow.AWSDatabaseHolder;
+
+public class Ec2SGRefreshJob extends BaseAWSDatabaseHolderRefreshJob {
+ public static final String NAME = "ec2_sg_job";
+
+ @Override
+ void refresh(AWSDatabaseHolder dbHolder) {
+ dbHolder.refreshEc2SGs();
+ }
+}
diff --git a/src/main/java/com/airbnb/billow/jobs/ElasticacheRefreshJob.java b/src/main/java/com/airbnb/billow/jobs/ElasticacheRefreshJob.java
new file mode 100644
index 0000000..a14e2b9
--- /dev/null
+++ b/src/main/java/com/airbnb/billow/jobs/ElasticacheRefreshJob.java
@@ -0,0 +1,12 @@
+package com.airbnb.billow.jobs;
+
+import com.airbnb.billow.AWSDatabaseHolder;
+
+public class ElasticacheRefreshJob extends BaseAWSDatabaseHolderRefreshJob {
+ public static final String NAME = "elasticache_job";
+
+ @Override
+ void refresh(AWSDatabaseHolder dbHolder) {
+ dbHolder.refreshElasticacheClusters();
+ }
+}
diff --git a/src/main/java/com/airbnb/billow/jobs/ElasticsearchRefreshJob.java b/src/main/java/com/airbnb/billow/jobs/ElasticsearchRefreshJob.java
new file mode 100644
index 0000000..f1557d8
--- /dev/null
+++ b/src/main/java/com/airbnb/billow/jobs/ElasticsearchRefreshJob.java
@@ -0,0 +1,12 @@
+package com.airbnb.billow.jobs;
+
+import com.airbnb.billow.AWSDatabaseHolder;
+
+public class ElasticsearchRefreshJob extends BaseAWSDatabaseHolderRefreshJob {
+ public static final String NAME = "elasticsearch_job";
+
+ @Override
+ void refresh(AWSDatabaseHolder dbHolder) {
+ dbHolder.refreshElasticsearchClusters();
+ }
+}
diff --git a/src/main/java/com/airbnb/billow/jobs/IamRefreshJob.java b/src/main/java/com/airbnb/billow/jobs/IamRefreshJob.java
new file mode 100644
index 0000000..f869e42
--- /dev/null
+++ b/src/main/java/com/airbnb/billow/jobs/IamRefreshJob.java
@@ -0,0 +1,12 @@
+package com.airbnb.billow.jobs;
+
+import com.airbnb.billow.AWSDatabaseHolder;
+
+public class IamRefreshJob extends BaseAWSDatabaseHolderRefreshJob {
+ public static final String NAME = "iam_job";
+
+ @Override
+ void refresh(AWSDatabaseHolder dbHolder) {
+ dbHolder.refreshIamUsers();
+ }
+}
diff --git a/src/main/java/com/airbnb/billow/jobs/RdsRefreshJob.java b/src/main/java/com/airbnb/billow/jobs/RdsRefreshJob.java
new file mode 100644
index 0000000..fcbe475
--- /dev/null
+++ b/src/main/java/com/airbnb/billow/jobs/RdsRefreshJob.java
@@ -0,0 +1,12 @@
+package com.airbnb.billow.jobs;
+
+import com.airbnb.billow.AWSDatabaseHolder;
+
+public class RdsRefreshJob extends BaseAWSDatabaseHolderRefreshJob{
+ public static final String NAME = "rds_job";
+
+ @Override
+ void refresh(AWSDatabaseHolder dbHolder) {
+ dbHolder.refreshRdsInstances();
+ }
+}
diff --git a/src/main/java/com/airbnb/billow/jobs/SqsRefreshJob.java b/src/main/java/com/airbnb/billow/jobs/SqsRefreshJob.java
new file mode 100644
index 0000000..f10172a
--- /dev/null
+++ b/src/main/java/com/airbnb/billow/jobs/SqsRefreshJob.java
@@ -0,0 +1,12 @@
+package com.airbnb.billow.jobs;
+
+import com.airbnb.billow.AWSDatabaseHolder;
+
+public class SqsRefreshJob extends BaseAWSDatabaseHolderRefreshJob {
+ public static final String NAME = "sqs_job";
+
+ @Override
+ void refresh(AWSDatabaseHolder dbHolder) {
+ dbHolder.refreshSqsQueues();
+ }
+}