Skip to content

Commit 2b467a1

Browse files
leandrodamascenaleandrodamascena
authored
Merge pull request #17 from leandrodamascena/developer
New checks, performance and others.
2 parents b5b68a5 + 2d321d2 commit 2b467a1

File tree

12 files changed

+363
-98
lines changed

12 files changed

+363
-98
lines changed

‎README.md‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ This script has been written in python3+ using AWS-CLI and it works in Linux, Wi
2626
- Make sure the latest version of AWS-CLI is installed on your workstation, and other components needed, with Python pip already installed:
2727

2828
```sh
29-
$ pip install awscli boto3
29+
$ pip install -r requirements.txt
3030
```
3131

3232
- Make sure you have properly configured your AWS-CLI with a valid Access Key and Region:
@@ -71,7 +71,6 @@ $ python msgfmt.py -o locales/NEWFOLDER/LC_MESSAGES/messages.mo locales/NEWFOLDE
7171

7272
### TODO
7373

74-
- Use role instance rather than aws-cli
7574
- Improve documentation and code comments
7675
- More services that uses VPC (I'll try add one a week)
7776
- Custom logging control and reporting improvement.

‎requirements.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
boto3
2+
ipaddress

‎shared/awscommands.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
from shared.common import *
22
from shared.internal.security import IAM, IAMPOLICY
3-
from shared.internal.network import VPC
3+
from shared.internal.network import VPC, IGW, NATGATEWAY
44
from shared.internal.compute import LAMBDA, EC2
55
from shared.internal.database import RDS, ELASTICACHE, DOCUMENTDB
66
from shared.internal.storage import EFS, S3POLICY
7-
from shared.internal.analytics import ELASTICSEARCH
7+
from shared.internal.analytics import ELASTICSEARCH, MSK
88
from shared.internal.application import SQSPOLICY
99

1010

@@ -26,3 +26,6 @@ def run(self):
2626
ELASTICSEARCH(self.vpc_options).run()
2727
DOCUMENTDB(self.vpc_options).run()
2828
SQSPOLICY(self.vpc_options).run()
29+
MSK(self.vpc_options).run()
30+
IGW(self.vpc_options).run()
31+
NATGATEWAY(self.vpc_options).run()

‎shared/common.py‎

Lines changed: 79 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
from typing import NamedTuple
22
import datetime
33
import boto3
4+
import re
5+
from ipaddress import ip_network, ip_address
6+
7+
8+
VPCE_REGEX = re.compile(r'(?<=sourcevpce")(\s*:\s*")(vpce-[a-zA-Z0-9]+)', re.DOTALL)
9+
SOURCE_IP_ADDRESS_REGEX = re.compile(r'(?<=sourceip")(\s*:\s*")([a-fA-F0-9.:/%]+)', re.DOTALL)
410

511

612
class bcolors:
@@ -19,6 +25,9 @@ class VpcOptions(NamedTuple):
1925
vpc_id: str
2026
region_name: str
2127

28+
def client(self, service_name: str):
29+
return self.session.client(service_name, region_name=self.region_name)
30+
2231

2332
def generate_session(profile_name):
2433
try:
@@ -29,14 +38,83 @@ def generate_session(profile_name):
2938

3039

3140
def exit_critical(message):
32-
print(bcolors.colors.get('FAIL'), message, bcolors.colors.get('ENDC'), sep="")
41+
log_critical(message)
3342
raise SystemExit
3443

3544

45+
def log_critical(message):
46+
print(bcolors.colors.get('FAIL'), message, bcolors.colors.get('ENDC'), sep="")
47+
48+
3649
def message_handler(message, position):
3750
print(bcolors.colors.get(position), message, bcolors.colors.get('ENDC'), sep="")
3851

3952

4053
def datetime_to_string(o):
4154
if isinstance(o, datetime.datetime):
4255
return o.__str__()
56+
57+
58+
def check_ipvpc_inpolicy(document, vpc_options: VpcOptions):
59+
document = document.replace("\\", "").lower()
60+
61+
""" Checking if VPC is inside document, it's a 100% true information """
62+
if vpc_options.vpc_id in document:
63+
return "direct VPC reference"
64+
else:
65+
"""
66+
Vpc_id not found, trying to discover if it's a potencial subnet IP or VPCE is allowed
67+
"""
68+
if "aws:sourcevpce" in document:
69+
70+
""" Get VPCE found """
71+
aws_sourcevpces = []
72+
for vpce_tuple in VPCE_REGEX.findall(document):
73+
aws_sourcevpces.append(vpce_tuple[1])
74+
75+
""" Get all VPCE of this VPC """
76+
ec2 = vpc_options.client('ec2')
77+
78+
filters = [{'Name': 'vpc-id',
79+
'Values': [vpc_options.vpc_id]}]
80+
81+
vpc_endpoints = ec2.describe_vpc_endpoints(Filters=filters)
82+
83+
""" iterate VPCEs found found """
84+
if len(vpc_endpoints['VpcEndpoints']) > 0:
85+
matching_vpces = []
86+
""" Iterate VPCE to match vpce in Policy Document """
87+
for data in vpc_endpoints['VpcEndpoints']:
88+
if data['VpcEndpointId'] in aws_sourcevpces:
89+
matching_vpces.append(data['VpcEndpointId'])
90+
return "VPC Endpoint(s): " + (", ".join(matching_vpces))
91+
92+
if "aws:sourceip" in document:
93+
94+
""" Get ip found """
95+
aws_sourceips = []
96+
for vpce_tuple in SOURCE_IP_ADDRESS_REGEX.findall(document):
97+
aws_sourceips.append(vpce_tuple[1])
98+
""" Get subnets cidr block """
99+
ec2 = vpc_options.client('ec2')
100+
101+
filters = [{'Name': 'vpc-id',
102+
'Values': [vpc_options.vpc_id]}]
103+
104+
subnets = ec2.describe_subnets(Filters=filters)
105+
overlapping_subnets = []
106+
""" iterate ips found """
107+
for ipfound in aws_sourceips:
108+
109+
""" Iterate subnets to match ipaddress """
110+
for subnet in list(subnets['Subnets']):
111+
ipfound = ip_network(ipfound)
112+
network_addres = ip_network(subnet['CidrBlock'])
113+
114+
if ipfound.overlaps(network_addres):
115+
overlapping_subnets.append("{} ({})".format(str(network_addres), subnet['SubnetId']))
116+
if len(overlapping_subnets) != 0:
117+
return "source IP(s): {} -> subnet CIDR(s): {}"\
118+
.format(", ".join(aws_sourceips), ", ".join(overlapping_subnets))
119+
120+
return False

‎shared/internal/analytics.py‎

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ def __init__(self, vpc_options: VpcOptions):
88

99
def run(self):
1010
try:
11-
client = self.vpc_options.session.client('es', region_name=self.vpc_options.region_name)
11+
client = self.vpc_options.client('es')
1212

1313
response = client.list_domain_names()
1414

@@ -27,9 +27,12 @@ def run(self):
2727

2828
document = json.dumps(documentpolicy, default=datetime_to_string)
2929

30+
""" check either vpc_id or potencial subnet ip are found """
31+
ipvpc_found = check_ipvpc_inpolicy(document=document, vpc_options=self.vpc_options)
32+
3033
""" elasticsearch uses accesspolicies too, so check both situation """
3134
if elasticsearch_domain['DomainStatus']['VPCOptions']['VPCId'] == self.vpc_options.vpc_id \
32-
or self.vpc_options.vpc_id in document:
35+
or ipvpc_found is True:
3336
found += 1
3437
message = message + "\nDomainId: {0} - DomainName: {1} - VpcId {2}".format(
3538
elasticsearch_domain['DomainStatus']['DomainId'],
@@ -40,4 +43,53 @@ def run(self):
4043

4144
except Exception as e:
4245
message = "Can't list ElasticSearch Domains\nError {0}".format(str(e))
46+
exit_critical(message)
47+
48+
class MSK(object):
49+
50+
def __init__(self, vpc_options: VpcOptions):
51+
self.vpc_options = vpc_options
52+
53+
def run(self):
54+
try:
55+
client = self.vpc_options.client('kafka')
56+
57+
""" get all cache clusters """
58+
response = client.list_clusters()
59+
60+
message_handler("\nChecking MSK CLUSTERS...", "HEADER")
61+
62+
if len(response['ClusterInfoList']) == 0:
63+
message_handler("Found 0 MSK Clusters in region {0}".format(self.vpc_options.region_name), "OKBLUE")
64+
else:
65+
found = 0
66+
message = ""
67+
68+
""" iterate cache clusters to get subnet groups """
69+
for data in response['ClusterInfoList']:
70+
71+
msk_subnets = ", ".join(data['BrokerNodeGroupInfo']['ClientSubnets'])
72+
73+
ec2 = self.vpc_options.session.resource('ec2', region_name=self.vpc_options.region_name)
74+
75+
filters = [{'Name':'vpc-id',
76+
'Values':[self.vpc_options.vpc_id]}]
77+
78+
subnets = ec2.subnets.filter(Filters=filters)
79+
80+
for subnet in list(subnets):
81+
82+
if subnet.id in msk_subnets:
83+
84+
found += 1
85+
message = message + "\nClusterName: {0} - VpcId: {1}".format(
86+
data['ClusterName'],
87+
self.vpc_options.vpc_id
88+
)
89+
break
90+
91+
message_handler("Found {0} MSK Clusters using VPC {1} {2}".format(str(found), self.vpc_options.vpc_id, message),'OKBLUE')
92+
93+
except Exception as e:
94+
message = "Can't list MSK Clusters\nError {0}".format(str(e))
4395
exit_critical(message)

‎shared/internal/application.py‎

Lines changed: 43 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,52 +1,58 @@
1+
from concurrent.futures.thread import ThreadPoolExecutor
2+
13
from shared.common import *
24
import json
35

6+
47
class SQSPOLICY(object):
5-
8+
69
def __init__(self, vpc_options: VpcOptions):
710
self.vpc_options = vpc_options
811

912
def run(self):
1013
try:
11-
client = self.vpc_options.session.client('sqs', region_name=self.vpc_options.region_name)
12-
14+
client = self.vpc_options.client('sqs')
1315
response = client.list_queues()
16+
except Exception as e:
17+
message = "Can't list SQS Queue Policy\nError {0}".format(str(e))
18+
log_critical(message)
19+
return
20+
message_handler("\nChecking SQS QUEUE POLICY...", "HEADER")
21+
22+
if "QueueUrls" not in response:
23+
message_handler("Found 0 SQS Queues in region {0}".format(self.vpc_options.region_name), "OKBLUE")
24+
else:
25+
found = 0
26+
message = ""
27+
28+
with ThreadPoolExecutor(15) as executor:
29+
results = executor.map(lambda data: self.analyze_queues(client, data[1]), enumerate(response["QueueUrls"]))
30+
31+
for result in results:
32+
if result[0] is True:
33+
found += 1
34+
message += result[1]
35+
message_handler("Found {0} SQS Queue Policy using VPC {1} {2}".format(str(found), self.vpc_options.vpc_id, message),'OKBLUE')
36+
37+
def analyze_queues(self, client, queue):
38+
try:
39+
sqs_queue_policy = client.get_queue_attributes(QueueUrl=queue, AttributeNames=['Policy'])
40+
if "Attributes" in sqs_queue_policy:
1441

15-
message_handler("\nChecking SQS QUEUE POLICY...", "HEADER")
16-
17-
if not "QueueUrls" in response:
18-
message_handler("Found 0 SQS Queues in region {0}".format(self.vpc_options.region_name), "OKBLUE")
19-
else:
20-
found = 0
21-
message = ""
22-
23-
""" SQS Queue doesn't returns a dict"""
24-
for idx, queue in enumerate(response["QueueUrls"]):
25-
26-
sqs_queue_policy = client.get_queue_attributes(QueueUrl=queue,
27-
AttributeNames=['Policy'])
28-
29-
30-
if "Attributes" in sqs_queue_policy:
31-
32-
""" Not sure about boto3 return """
33-
try:
34-
documentpolicy = sqs_queue_policy['Attributes']['Policy']
35-
36-
document = json.dumps(documentpolicy, default=datetime_to_string)
42+
""" Not sure about boto3 return """
3743

38-
if self.vpc_options.vpc_id in document:
39-
found += 1
40-
message = message + "\nQueueUrl: {0} - VpcId {1}".format(
41-
queue,
42-
self.vpc_options.vpc_id
43-
)
44-
except:
45-
pass
44+
documentpolicy = sqs_queue_policy['Attributes']['Policy']
45+
document = json.dumps(documentpolicy, default=datetime_to_string)
4646

47+
""" check either vpc_id or potencial subnet ip are found """
48+
ipvpc_found = check_ipvpc_inpolicy(document=document, vpc_options=self.vpc_options)
4749

48-
message_handler("Found {0} SQS Queue Policy using VPC {1} {2}".format(str(found), self.vpc_options.vpc_id, message),'OKBLUE')
49-
50+
if ipvpc_found is not False:
51+
return True, "\nQueueUrl: {0} -> {1} -> VPC {2}".format(
52+
queue,
53+
ipvpc_found,
54+
self.vpc_options.vpc_id
55+
)
5056
except Exception as e:
51-
message = "Can't list SQS Queue Policy\nError {0}".format(str(e))
52-
exit_critical(message)
57+
log_critical(str(e))
58+
return False, None

‎shared/internal/compute.py‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ def __init__(self, vpc_options: VpcOptions):
88

99
def run(self):
1010
try:
11-
client = self.vpc_options.session.client('lambda', region_name=self.vpc_options.region_name)
11+
client = self.vpc_options.client('lambda')
1212

1313
response = client.list_functions()
1414

@@ -22,11 +22,10 @@ def run(self):
2222
for data in response["Functions"]:
2323
if 'VpcConfig' in data and data['VpcConfig']['VpcId'] == self.vpc_options.vpc_id:
2424
found += 1
25-
message = message + "\nFunctionName: {0} - Runtime: {1} - VpcId {2} - SubnetIds: {3}".format(
26-
data["FunctionName"],
27-
data["Runtime"],
28-
data['VpcConfig']['VpcId'],
29-
", ".join(data['VpcConfig']['SubnetIds'])
25+
message = message + "\nFunctionName: {} -> Subnet id(s): {} -> VPC id {}".format(
26+
data["FunctionName"],
27+
", ".join(data['VpcConfig']['SubnetIds']),
28+
data['VpcConfig']['VpcId']
3029
)
3130
message_handler("Found {0} Lambda Functions using VPC {1} {2}".format(str(found), self.vpc_options.vpc_id, message),'OKBLUE')
3231
except Exception as e:
@@ -42,7 +41,7 @@ def __init__(self, vpc_options: VpcOptions):
4241
def run(self):
4342
try:
4443

45-
client = self.vpc_options.session.client('ec2', region_name=self.vpc_options.region_name)
44+
client = self.vpc_options.client('ec2')
4645

4746
response = client.describe_instances()
4847

@@ -58,11 +57,11 @@ def run(self):
5857
if "VpcId" in instances:
5958
if instances['VpcId'] == self.vpc_options.vpc_id:
6059
found += 1
61-
message = message + "\nInstanceId: {0} - PrivateIpAddress: {1} - VpcId {2} - SubnetIds: {3}".format(
60+
message = message + "\nInstanceId: {} - PrivateIpAddress: {} -> Subnet id(s): {} - VpcId {}".format(
6261
instances["InstanceId"],
6362
instances["PrivateIpAddress"],
64-
instances['VpcId'],
65-
instances['SubnetId']
63+
instances['SubnetId'],
64+
instances['VpcId']
6665
)
6766
message_handler("Found {0} EC2 Instances using VPC {1} {2}".format(str(found), self.vpc_options.vpc_id, message),'OKBLUE')
6867
except Exception as e:

‎shared/internal/database.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ def __init__(self, vpc_options: VpcOptions):
88

99
def run(self):
1010
try:
11-
client = self.vpc_options.session.client('rds', region_name=self.vpc_options.region_name)
11+
client = self.vpc_options.client('rds')
1212

1313
response = client.describe_db_instances(Filters=[
1414
{'Name': 'engine',
@@ -46,7 +46,7 @@ def __init__(self, vpc_options: VpcOptions):
4646

4747
def run(self):
4848
try:
49-
client = self.vpc_options.session.client('elasticache', region_name=self.vpc_options.region_name)
49+
client = self.vpc_options.client('elasticache')
5050

5151
""" get all cache clusters """
5252
response = client.describe_cache_clusters()
@@ -85,7 +85,7 @@ def __init__(self, vpc_options: VpcOptions):
8585

8686
def run(self):
8787
try:
88-
client = self.vpc_options.session.client('docdb', region_name=self.vpc_options.region_name)
88+
client = self.vpc_options.client('docdb')
8989

9090
response = client.describe_db_instances(Filters=[
9191
{'Name': 'engine',

0 commit comments

Comments
 (0)