Skip to content

Commit be88092

Browse files
bergmannfbergmannf
committed
Paginate all vpc calls.
1 parent c50aaa2 commit be88092

File tree

1 file changed

+36
-22
lines changed

1 file changed

+36
-22
lines changed

‎vpc.go‎

Lines changed: 36 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ func (e *VPCExporter) CollectInRegion(session *session.Session, region *string,
9595
if err != nil {
9696
level.Error(e.logger).Log("msg", "Call to DescribeVpcs failed", "region", region, "err", err)
9797
} else {
98-
for i, _ := range allVpcs.Vpcs {
98+
for i := range allVpcs.Vpcs {
9999
e.collectSubnetsPerVpcUsage(allVpcs.Vpcs[i], ec2Svc, *region)
100100
e.collectInterfaceVpcEndpointsPerVpcUsage(allVpcs.Vpcs[i], ec2Svc, *region)
101101
e.collectRoutesTablesPerVpcUsage(allVpcs.Vpcs[i], ec2Svc, *region)
@@ -109,7 +109,7 @@ func (e *VPCExporter) CollectInRegion(session *session.Session, region *string,
109109
if err != nil {
110110
level.Error(e.logger).Log("msg", "Call to DescribeRouteTables failed", "region", region, "err", err)
111111
} else {
112-
for i, _ := range allRouteTables.RouteTables {
112+
for i := range allRouteTables.RouteTables {
113113
e.collectRoutesPerRouteTableUsage(allRouteTables.RouteTables[i], ec2Svc, *region)
114114
}
115115
}
@@ -120,7 +120,7 @@ func (e *VPCExporter) CollectLoop() {
120120

121121
wg := &sync.WaitGroup{}
122122
wg.Add(len(e.sessions))
123-
for i, _ := range e.sessions {
123+
for i := range e.sessions {
124124
session := e.sessions[i]
125125
region := session.Config.Region
126126
go e.CollectInRegion(session, region, wg)
@@ -167,14 +167,17 @@ func (e *VPCExporter) collectVpcsPerRegionQuota(client *servicequotas.ServiceQuo
167167
func (e *VPCExporter) collectVpcsPerRegionUsage(ec2Svc *ec2.EC2, region string) {
168168
ctx, cancelFunc := context.WithTimeout(context.Background(), e.timeout)
169169
defer cancelFunc()
170-
describeVpcsOutput, err := ec2Svc.DescribeVpcsWithContext(ctx, &ec2.DescribeVpcsInput{})
170+
numVpcs := 0
171+
err := ec2Svc.DescribeVpcsPagesWithContext(ctx, &ec2.DescribeVpcsInput{}, func(page *ec2.DescribeVpcsOutput, lastPage bool) bool {
172+
numVpcs += len(page.Vpcs)
173+
return !lastPage
174+
})
171175
if err != nil {
172176
level.Error(e.logger).Log("msg", "Call to DescribeVpcs failed", "region", region, "err", err)
173177
exporterMetrics.IncrementErrors()
174178
return
175179
}
176-
usage := len(describeVpcsOutput.Vpcs)
177-
e.cache.AddMetric(prometheus.MustNewConstMetric(e.VpcsPerRegionUsage, prometheus.GaugeValue, float64(usage), region))
180+
e.cache.AddMetric(prometheus.MustNewConstMetric(e.VpcsPerRegionUsage, prometheus.GaugeValue, float64(numVpcs), region))
178181
}
179182

180183
func (e *VPCExporter) collectSubnetsPerVpcQuota(client *servicequotas.ServiceQuotas, region string) {
@@ -190,19 +193,21 @@ func (e *VPCExporter) collectSubnetsPerVpcQuota(client *servicequotas.ServiceQuo
190193
func (e *VPCExporter) collectSubnetsPerVpcUsage(vpc *ec2.Vpc, ec2Svc *ec2.EC2, region string) {
191194
ctx, cancelFunc := context.WithTimeout(context.Background(), e.timeout)
192195
defer cancelFunc()
193-
describeSubnetsOutput, err := ec2Svc.DescribeSubnetsWithContext(ctx, &ec2.DescribeSubnetsInput{
194-
Filters: []*ec2.Filter{&ec2.Filter{
196+
numSubnets := 0
197+
err := ec2Svc.DescribeSubnetsPagesWithContext(ctx, &ec2.DescribeSubnetsInput{
198+
Filters: []*ec2.Filter{{
195199
Name: aws.String("vpc-id"),
196200
Values: []*string{vpc.VpcId},
197-
}},
201+
}}}, func(page *ec2.DescribeSubnetsOutput, lastPage bool) bool {
202+
numSubnets += len(page.Subnets)
203+
return !lastPage
198204
})
199205
if err != nil {
200206
level.Error(e.logger).Log("msg", "Call to DescribeSubnets failed", "region", region, "err", err)
201207
exporterMetrics.IncrementErrors()
202208
return
203209
}
204-
usage := len(describeSubnetsOutput.Subnets)
205-
e.cache.AddMetric(prometheus.MustNewConstMetric(e.SubnetsPerVpcUsage, prometheus.GaugeValue, float64(usage), region, *vpc.VpcId))
210+
e.cache.AddMetric(prometheus.MustNewConstMetric(e.SubnetsPerVpcUsage, prometheus.GaugeValue, float64(numSubnets), region, *vpc.VpcId))
206211
}
207212

208213
func (e *VPCExporter) collectRoutesPerRouteTableQuota(client *servicequotas.ServiceQuotas, region string) {
@@ -221,6 +226,10 @@ func (e *VPCExporter) collectRoutesPerRouteTableUsage(rtb *ec2.RouteTable, ec2Sv
221226
descRouteTableOutput, err := ec2Svc.DescribeRouteTablesWithContext(ctx, &ec2.DescribeRouteTablesInput{
222227
RouteTableIds: []*string{rtb.RouteTableId},
223228
})
229+
if len(descRouteTableOutput.RouteTables) != 1 {
230+
level.Error(e.logger).Log("msg", "Unexpected number of routetables (!= 1) returned from DescribeRouteTables")
231+
return
232+
}
224233
if err != nil {
225234
level.Error(e.logger).Log("msg", "Call to DescribeRouteTables failed", "region", region, "err", err)
226235
exporterMetrics.IncrementErrors()
@@ -243,19 +252,21 @@ func (e *VPCExporter) collectInterfaceVpcEndpointsPerVpcQuota(client *servicequo
243252
func (e *VPCExporter) collectInterfaceVpcEndpointsPerVpcUsage(vpc *ec2.Vpc, ec2Svc *ec2.EC2, region string) {
244253
ctx, cancelFunc := context.WithTimeout(context.Background(), e.timeout)
245254
defer cancelFunc()
246-
descVpcEndpoints, err := ec2Svc.DescribeVpcEndpointsWithContext(ctx, &ec2.DescribeVpcEndpointsInput{
247-
Filters: []*ec2.Filter{{
248-
Name: aws.String("vpc-id"),
249-
Values: []*string{vpc.VpcId},
250-
}},
255+
256+
numEndpoints := 0
257+
descEndpointsInput := &ec2.DescribeVpcEndpointsInput{
258+
Filters: []*ec2.Filter{{Name: aws.String("vpc-id"), Values: []*string{vpc.VpcId}}},
259+
}
260+
err := ec2Svc.DescribeVpcEndpointsPagesWithContext(ctx, descEndpointsInput, func(page *ec2.DescribeVpcEndpointsOutput, lastPage bool) bool {
261+
numEndpoints += len(page.VpcEndpoints)
262+
return !lastPage
251263
})
252264
if err != nil {
253265
level.Error(e.logger).Log("msg", "Call to DescribeVpcEndpoints failed", "region", region, "err", err)
254266
exporterMetrics.IncrementErrors()
255267
return
256268
}
257-
quota := len(descVpcEndpoints.VpcEndpoints)
258-
e.cache.AddMetric(prometheus.MustNewConstMetric(e.InterfaceVpcEndpointsPerVpcUsage, prometheus.GaugeValue, float64(quota), region, *vpc.VpcId))
269+
e.cache.AddMetric(prometheus.MustNewConstMetric(e.InterfaceVpcEndpointsPerVpcUsage, prometheus.GaugeValue, float64(numEndpoints), region, *vpc.VpcId))
259270
}
260271

261272
func (e *VPCExporter) collectRoutesTablesPerVpcQuota(client *servicequotas.ServiceQuotas, region string) {
@@ -271,19 +282,22 @@ func (e *VPCExporter) collectRoutesTablesPerVpcQuota(client *servicequotas.Servi
271282
func (e *VPCExporter) collectRoutesTablesPerVpcUsage(vpc *ec2.Vpc, ec2Svc *ec2.EC2, region string) {
272283
ctx, cancelFunc := context.WithTimeout(context.Background(), e.timeout)
273284
defer cancelFunc()
274-
descRouteTables, err := ec2Svc.DescribeRouteTablesWithContext(ctx, &ec2.DescribeRouteTablesInput{
285+
var numRouteTables int
286+
input := &ec2.DescribeRouteTablesInput{
275287
Filters: []*ec2.Filter{{
276288
Name: aws.String("vpc-id"),
277289
Values: []*string{vpc.VpcId},
278-
}},
290+
}}}
291+
err := ec2Svc.DescribeRouteTablesPagesWithContext(ctx, input, func(page *ec2.DescribeRouteTablesOutput, lastPage bool) bool {
292+
numRouteTables += len(page.RouteTables)
293+
return !lastPage
279294
})
280295
if err != nil {
281296
level.Error(e.logger).Log("msg", "Call to DescribeRouteTables failed", "region", region, "err", err)
282297
exporterMetrics.IncrementErrors()
283298
return
284299
}
285-
quota := len(descRouteTables.RouteTables)
286-
e.cache.AddMetric(prometheus.MustNewConstMetric(e.RouteTablesPerVpcUsage, prometheus.GaugeValue, float64(quota), region, *vpc.VpcId))
300+
e.cache.AddMetric(prometheus.MustNewConstMetric(e.RouteTablesPerVpcUsage, prometheus.GaugeValue, float64(numRouteTables), region, *vpc.VpcId))
287301
}
288302

289303
func (e *VPCExporter) collectIPv4BlocksPerVpcQuota(client *servicequotas.ServiceQuotas, region string) {

0 commit comments

Comments
 (0)