Paginate all vpc calls.

Author: bergmannf

vpc.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"sync"
 	"time"
 
@@ -153,13 +154,17 @@ func (c *VPCCollector) collectVpcsPerRegionQuota(ch chan<- prometheus.Metric) {
 func (c *VPCCollector) collectVpcsPerRegionUsage(ch chan<- prometheus.Metric) {
 	ctx, cancelFunc := context.WithTimeout(context.Background(), c.e.timeout)
 	defer cancelFunc()
-	describeVpcsOutput, err := c.ec2.DescribeVpcsWithContext(ctx, &ec2.DescribeVpcsInput{})
+	var vpcs []*ec2.Vpc
+	err := c.ec2.DescribeVpcsPagesWithContext(ctx, &ec2.DescribeVpcsInput{}, func(page *ec2.DescribeVpcsOutput, lastPage bool) bool {
+		vpcs = append(vpcs, page.Vpcs...)
+		return !lastPage
+	})
 	if err != nil {
 		level.Error(c.e.logger).Log("msg", "Call to DescribeVpcs failed", "region", c.region, "err", err)
 		exporterMetrics.IncrementErrors()
 		return
 	}
-	usage := len(describeVpcsOutput.Vpcs)
+	usage := len(vpcs)
 	ch <- prometheus.MustNewConstMetric(c.e.VpcsPerRegionUsage, prometheus.GaugeValue, float64(usage), *c.region)
 }
 
@@ -176,18 +181,21 @@ func (c *VPCCollector) collectSubnetsPerVpcQuota(ch chan<- prometheus.Metric) {
 func (c *VPCCollector) collectSubnetsPerVpcUsage(ch chan<- prometheus.Metric, vpc *ec2.Vpc) {
 	ctx, cancelFunc := context.WithTimeout(context.Background(), c.e.timeout)
 	defer cancelFunc()
-	describeSubnetsOutput, err := c.ec2.DescribeSubnetsWithContext(ctx, &ec2.DescribeSubnetsInput{
-		Filters: []*ec2.Filter{&ec2.Filter{
+	var subnets []*ec2.Subnet
+	err := c.ec2.DescribeSubnetsPagesWithContext(ctx, &ec2.DescribeSubnetsInput{
+		Filters: []*ec2.Filter{{
 			Name:   aws.String("vpc-id"),
 			Values: []*string{vpc.VpcId},
-		}},
+		}}}, func(page *ec2.DescribeSubnetsOutput, lastPage bool) bool {
+		subnets = append(subnets, page.Subnets...)
+		return !lastPage
 	})
 	if err != nil {
 		level.Error(c.e.logger).Log("msg", "Call to DescribeSubnets failed", "region", c.region, "err", err)
 		exporterMetrics.IncrementErrors()
 		return
 	}
-	usage := len(describeSubnetsOutput.Subnets)
+	usage := len(subnets)
 	ch <- prometheus.MustNewConstMetric(c.e.SubnetsPerVpcUsage, prometheus.GaugeValue, float64(usage), *c.region, *vpc.VpcId)
 }
 
@@ -204,15 +212,25 @@ func (c *VPCCollector) collectRoutesPerRouteTableQuota(ch chan<- prometheus.Metr
 func (c *VPCCollector) collectRoutesPerRouteTableUsage(ch chan<- prometheus.Metric, rtb *ec2.RouteTable) {
 	ctx, cancelFunc := context.WithTimeout(context.Background(), c.e.timeout)
 	defer cancelFunc()
-	descRouteTableOutput, err := c.ec2.DescribeRouteTablesWithContext(ctx, &ec2.DescribeRouteTablesInput{
+	var routes []*ec2.Route
+	var unexpectedNumberOfRouteTables error
+	err := c.ec2.DescribeRouteTablesPagesWithContext(ctx, &ec2.DescribeRouteTablesInput{
 		RouteTableIds: []*string{rtb.RouteTableId},
+	}, func(page *ec2.DescribeRouteTablesOutput, lastPage bool) bool {
+		if len(page.RouteTables) != 1 {
+			// Indicate that one of the calls return too many routetables making data invalid
+			unexpectedNumberOfRouteTables = errors.New("Unexpected number of routetables returned")
+			return false
+		}
+		routes = append(routes, page.RouteTables[0].Routes...)
+		return !lastPage
 	})
-	if err != nil {
+	if err != nil || unexpectedNumberOfRouteTables != nil {
 		level.Error(c.e.logger).Log("msg", "Call to DescribeRouteTables failed", "region", c.region, "err", err)
 		exporterMetrics.IncrementErrors()
 		return
 	}
-	quota := len(descRouteTableOutput.RouteTables)
+	quota := len(routes)
 	ch <- prometheus.MustNewConstMetric(c.e.RoutesPerRouteTableUsage, prometheus.GaugeValue, float64(quota), *c.region, *rtb.VpcId, *rtb.RouteTableId)
 }
 
@@ -229,19 +247,27 @@ func (c *VPCCollector) collectInterfaceVpcEndpointsPerVpcQuota(ch chan<- prometh
 func (c *VPCCollector) collectInterfaceVpcEndpointsPerVpcUsage(ch chan<- prometheus.Metric, vpc *ec2.Vpc) {
 	ctx, cancelFunc := context.WithTimeout(context.Background(), c.e.timeout)
 	defer cancelFunc()
-	descVpcEndpoints, err := c.ec2.DescribeVpcEndpointsWithContext(ctx, &ec2.DescribeVpcEndpointsInput{
-		Filters: []*ec2.Filter{{
-			Name:   aws.String("vpc-id"),
-			Values: []*string{vpc.VpcId},
-		}},
+
+	numEndpoints := 0
+	descEndpointsInput := &ec2.DescribeVpcEndpointsInput{
+		Filters:    []*ec2.Filter{{Name: aws.String("vpc-id"), Values: []*string{vpc.VpcId}}},
+		MaxResults: aws.Int64(1000),
+	}
+
+	var vpcEndpoints []*ec2.VpcEndpoint
+	err := c.ec2.DescribeVpcEndpointsPagesWithContext(ctx, descEndpointsInput, func(page *ec2.DescribeVpcEndpointsOutput, lastPage bool) bool {
+		vpcEndpoints = append(vpcEndpoints, page.VpcEndpoints...)
+		return !lastPage
 	})
+
 	if err != nil {
 		level.Error(c.e.logger).Log("msg", "Call to DescribeVpcEndpoints failed", "region", c.region, "err", err)
 		exporterMetrics.IncrementErrors()
 		return
 	}
-	quota := len(descVpcEndpoints.VpcEndpoints)
-	ch <- prometheus.MustNewConstMetric(c.e.InterfaceVpcEndpointsPerVpcUsage, prometheus.GaugeValue, float64(quota), *c.region, *vpc.VpcId)
+	numEndpoints += len(vpcEndpoints)
+
+	ch <- prometheus.MustNewConstMetric(c.e.InterfaceVpcEndpointsPerVpcUsage, prometheus.GaugeValue, float64(numEndpoints), *c.region, *vpc.VpcId)
 }
 
 func (c *VPCCollector) collectRoutesTablesPerVpcQuota(ch chan<- prometheus.Metric) {
@@ -257,18 +283,22 @@ func (c *VPCCollector) collectRoutesTablesPerVpcQuota(ch chan<- prometheus.Metri
 func (c *VPCCollector) collectRoutesTablesPerVpcUsage(ch chan<- prometheus.Metric, vpc *ec2.Vpc) {
 	ctx, cancelFunc := context.WithTimeout(context.Background(), c.e.timeout)
 	defer cancelFunc()
-	descRouteTables, err := c.ec2.DescribeRouteTablesWithContext(ctx, &ec2.DescribeRouteTablesInput{
+	var routeTables []*ec2.RouteTable
+	input := &ec2.DescribeRouteTablesInput{
 		Filters: []*ec2.Filter{{
 			Name:   aws.String("vpc-id"),
 			Values: []*string{vpc.VpcId},
-		}},
+		}}}
+	err := c.ec2.DescribeRouteTablesPagesWithContext(ctx, input, func(page *ec2.DescribeRouteTablesOutput, lastPage bool) bool {
+		routeTables = append(routeTables, page.RouteTables...)
+		return !lastPage
 	})
 	if err != nil {
 		level.Error(c.e.logger).Log("msg", "Call to DescribeRouteTables failed", "region", c.region, "err", err)
 		exporterMetrics.IncrementErrors()
 		return
 	}
-	quota := len(descRouteTables.RouteTables)
+	quota := len(routeTables)
 	ch <- prometheus.MustNewConstMetric(c.e.RouteTablesPerVpcUsage, prometheus.GaugeValue, float64(quota), *c.region, *vpc.VpcId)
 }
 
@@ -285,18 +315,28 @@ func (c *VPCCollector) collectIPv4BlocksPerVpcQuota(ch chan<- prometheus.Metric)
 func (c *VPCCollector) collectIPv4BlocksPerVpcUsage(ch chan<- prometheus.Metric, vpc *ec2.Vpc) {
 	ctx, cancelFunc := context.WithTimeout(context.Background(), c.e.timeout)
 	defer cancelFunc()
-	descVpcs, err := c.ec2.DescribeVpcsWithContext(ctx, &ec2.DescribeVpcsInput{
+	input := &ec2.DescribeVpcsInput{
 		VpcIds: []*string{vpc.VpcId},
+	}
+	var unexpectedNumberOfVpcs error
+	var cidrBlockAssociations []*ec2.VpcCidrBlockAssociation
+	err := c.ec2.DescribeVpcsPagesWithContext(ctx, input, func(page *ec2.DescribeVpcsOutput, lastPage bool) bool {
+		if len(page.Vpcs) != 1 {
+			unexpectedNumberOfVpcs = errors.New("Unexpected number of vpcs received")
+			return false
+		}
+		cidrBlockAssociations = append(cidrBlockAssociations, page.Vpcs[0].CidrBlockAssociationSet...)
+		return !lastPage
 	})
 	if err != nil {
 		level.Error(c.e.logger).Log("msg", "Call to DescribeVpcs failed", "region", c.region, "err", err)
 		exporterMetrics.IncrementErrors()
 		return
 	}
-	if len(descVpcs.Vpcs) != 1 {
+	if unexpectedNumberOfVpcs != nil {
 		level.Error(c.e.logger).Log("msg", "Unexpected numbers of VPCs (!= 1) returned", "region", c.region, "vpcId", vpc.VpcId)
 	}
-	quota := len(descVpcs.Vpcs[0].CidrBlockAssociationSet)
+	quota := len(cidrBlockAssociations)
 	ch <- prometheus.MustNewConstMetric(c.e.IPv4BlocksPerVpcUsage, prometheus.GaugeValue, float64(quota), *c.region, *vpc.VpcId)
 }