From 43beb977c47e34777f5903fbffb872a59a099fc5 Mon Sep 17 00:00:00 2001 From: Richard Knechtel Date: Wed, 29 Jun 2022 13:07:08 -0500 Subject: [PATCH 1/4] Added showing Network Interfaces and SG is attached to and showing all lambdas an SG is attached to and added usage for a python virtual environment --- sg-tool/README.md | 45 ++++++++++++-- sg-tool/requirements.txt | 16 +++++ sg-tool/sg.py | 128 +++++++++++++++++++++++++++++++-------- 3 files changed, 158 insertions(+), 31 deletions(-) create mode 100644 sg-tool/requirements.txt diff --git a/sg-tool/README.md b/sg-tool/README.md index 40bcfd7..8bb8391 100644 --- a/sg-tool/README.md +++ b/sg-tool/README.md @@ -1,9 +1,44 @@ -# SG Tool -This tool can be used to find ALL information related to a Security Group: +# SG Tool + +This tool can be used to find ALL information related to a Security Group: + - Inbound + Outbound Rules - Attached Resources - References in other SGs -Examples: -``` -./sg.py sg-abc123456 \ No newline at end of file + +## Python Virtual Environment Setup (Linux) + +**Create the Virtual Environment (Example):** +python3 -m venv ~/projects/aws-tools/sg-tool/v-env + +**Activate the Virtual Environment (Example):** +source ~/projects/aws-tools/sg-tool/v-env/bin/activate + +**Generate Requirements for project:** +To create requirements.txt: + +1) Setup virtual environment +2) Install all python packages + Example: +~/projects/aws-tools/sg-tool/v-env/bin/pip3 install +3) Note: Make sure to upgrade pip +~/projects/aws-tools/sg-tool/v-env/bin/pip3 install --upgrade pip +4) run: +[Path to Virtual Environment Bin Directory]/pip3 freeze > requirements.txt +Example (Linux): +~/projects/aws-tools/sg-tool/v-env/bin/pip3 freeze > requirements.txt + +**Install the Requirements/Dependancies (Example):** +~/projects/aws-tools/sg-tool/v-env/bin/pip3 install -r requirements.txt + +**Example usage:** + +**Template:** + +`./sg.py ` + +**Example Call:** + +`./sg.py sg-abc123456` + diff --git a/sg-tool/requirements.txt b/sg-tool/requirements.txt new file mode 100644 index 0000000..697627d --- /dev/null +++ b/sg-tool/requirements.txt @@ -0,0 +1,16 @@ +boto3==1.24.15 +botocore==1.27.15 +click==8.1.3 +DateTime==4.4 +dnspython==2.2.1 +futures3==1.0.0 +jmespath==1.0.1 +netaddr==0.8.0 +pickle5==0.0.11 +pkg_resources==0.0.0 +python-dateutil==2.8.2 +pytz==2022.1 +s3transfer==0.6.0 +six==1.16.0 +urllib3==1.26.9 +zope.interface==5.4.0 diff --git a/sg-tool/sg.py b/sg-tool/sg.py index 03df251..0592143 100755 --- a/sg-tool/sg.py +++ b/sg-tool/sg.py @@ -1,25 +1,26 @@ -#!/usr/bin/env python -import re -import sys -import json +#!/usr/bin/env python3 import boto3 -import click -import pickle import botocore -import dns.resolver -from netaddr import IPNetwork, IPAddress -from datetime import datetime, timedelta +import click from concurrent.futures import as_completed from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta +import dns.resolver +import json +from netaddr import IPNetwork, IPAddress +import pickle +import re +import sys +import time +# Console Text Colors: BLUE = '\033[94m' RED = '\033[91m' GREEN = '\033[92m' ENDC = '\033[0m' YELLOW = '\033[93m' - def print_green(message, nl=True): click.echo(click.style(str(message), fg='bright_green'), nl=nl) @@ -85,15 +86,17 @@ def get_client(self, service_name): except KeyError: throttle_count += 1 throttle_wait_ms = 2**throttle_count/10 - sleep(throttle_wait_ms) + time.sleep(throttle_wait_ms) continue self.boto_clients[service_name] = client_service return client_service def aws(self, service_name, function_name, max_results=0, quiet=False, cache=True, **kwargs): + full_req_name = f'{service_name}:{function_name}:{kwargs}' if cache and full_req_name in self.aws_cache: return self.aws_cache[full_req_name] + aws_functions_dict = { 'ec2:describe_network_interfaces': {'token_name': 'NextToken', 'max_name': 'MaxResults', 'max_items': 1000, 'no_max_args': ['NetworkInterfaceIds']}, 'elasticache:describe_cache_clusters': {'token_name': 'Marker', 'max_name': 'MaxRecords', 'max_items': 100}, @@ -110,10 +113,11 @@ def aws(self, service_name, function_name, max_results=0, quiet=False, cache=Tru 'ecs:list_tasks': {'token_name': 'nextToken', 'max_name': 'maxResults', 'max_items': 100}, 'sagemaker:list_endpoints': {'token_name': 'NextToken', 'max_name': 'MaxResults', 'max_items': 100}, 'cloudtrail:lookup_events': {'token_name': 'NextToken', 'max_name': 'MaxResults', 'max_items': 50}, + 'lambda:list_functions': {'token_name': 'Marker', 'max_name': 'MaxItems', 'max_items': 100}, } - next_token_key_names = ['NextMarker', - 'NextToken', 'Marker', 'nextToken'] + + next_token_key_names = ['NextMarker', 'NextToken', 'Marker', 'nextToken'] object_list = [] token_name = '' @@ -143,13 +147,13 @@ def aws(self, service_name, function_name, max_results=0, quiet=False, cache=Tru except botocore.exceptions.EndpointConnectionError: throttle_count += 1 throttle_wait_ms = 2**throttle_count/10 - sleep(throttle_wait_ms) + time.sleep(throttle_wait_ms) continue except client.exceptions.ClientError as err: if err.response['Error']['Code'] == 'Throttling': throttle_count += 1 throttle_wait_ms = 2**throttle_count/10 - sleep(throttle_wait_ms) + time.sleep(throttle_wait_ms) continue else: err_msg = err.response['Error']['Message'] @@ -286,8 +290,7 @@ def get_referenced_sgs(full_sg_name, all_sgs, sgs_names): def get_elasticache_instances(): - es_instances = Boto().aws( - 'elasticache', 'describe_cache_clusters', ShowCacheNodeInfo=True) + es_instances = Boto().aws('elasticache', 'describe_cache_clusters', ShowCacheNodeInfo=True) result = {} for instance in es_instances: name = instance['CacheClusterId'] @@ -629,8 +632,35 @@ def get_neptune_instances(): return result +# Get any Lambdas attached to a Security Group +def run_lambda_action(action_name): + return action_name, Boto().aws('lambda', action_name, 'NextMarker') + +def get_lambdas(): + + lambda_client = boto3.client('lambda') + + all_lambdas = [] + next_marker = None + response = lambda_client.list_functions() + all_lambdas.append(response) + while next_marker != '': + next_marker = '' + functions = response['Functions'] + if not functions: + continue + + # Verify if there is next marker + if 'NextMarker' in response: + next_marker = response['NextMarker'] + response = lambda_client.list_functions(Marker=next_marker) + all_lambdas.append(response) + + + return all_lambdas + +def get_attached_resources(interfaces, sg_id): -def get_attached_resources(interfaces): attached_entities = {} saved_redshift_clusters = '' saved_rds_instances = '' @@ -639,7 +669,12 @@ def get_attached_resources(interfaces): saved_neptune_clusters = '' saved_ecs_clusters = '' saved_sm_endpoints = '' + + NetworkInterfaces = [] + Lambda_ips = [] # get list of IPs for lambdas sharing same security group + for eni in interfaces: + description = eni['Description'] int_type = eni['InterfaceType'] eni_id = eni['NetworkInterfaceId'] @@ -654,11 +689,16 @@ def get_attached_resources(interfaces): all_eni_ips = private_ips + # Get any Network Interfaces a Security Group is attached to + if eni_id is not None: + NetworkInterfaces.append(eni_id) + requester_id = eni.get('RequesterId') status = eni.get('Status') managed = eni.get('RequesterManaged') attachment = eni.get('Attachment') instance_id = '' + if attachment: instance_id = attachment.get('InstanceId') @@ -669,6 +709,7 @@ def get_attached_resources(interfaces): name = re.sub( r'^AWS Lambda VPC ENI\-(.+)\-[^\-]+\-[^\-]+\-[^\-]+\-[^\-]+\-[^\-]+$', r'\1', description) service = 'Lambda Function' + Lambda_ips += private_ips elif int_type == 'nat_gateway': service = 'NAT Gateway' @@ -876,12 +917,46 @@ def get_attached_resources(interfaces): else: service = 'Unknown' name = description + + # Create list of Attached Resources full_res_name = f'{service} {name}' + if not full_res_name in attached_entities: attached_entities[full_res_name] = { 'service': service, 'name': name, 'ips': private_ips} else: attached_entities[full_res_name]['ips'] += private_ips + + # Add Lambdas - if any - to Attached Resources + all_lambdas = get_lambdas() + + all_functions = [] + + for lambdas in all_lambdas: + all_functions = lambdas['Functions'] + + for lambda_funtion in all_functions: + + # If no VpcConfig in the lambda Function - skip it + if 'VpcConfig' in lambda_funtion: + Function_Name = lambda_funtion['FunctionName'] + Security_Groups = lambda_funtion['VpcConfig']['SecurityGroupIds'] + + if sg_id in Security_Groups: + service = 'Lambda Function' + name = Function_Name + + full_res_name = f'{service} {name}' + if full_res_name not in attached_entities: + attached_entities[full_res_name] = {'service': service, 'name': name, 'ips': Lambda_ips} + else: + for private_ip in private_ips: + if private_ip not in Lambda_ips: + attached_entities[full_res_name]['ips'] += private_ips + + # Add Network Interfaces - if any - to Attached Resources + if NetworkInterfaces is not None and len(NetworkInterfaces) > 0: + attached_entities['Network Interfaces'] = {'service': 'Network Interface', 'name': 'ENIs', 'ips': NetworkInterfaces} return attached_entities @@ -893,10 +968,8 @@ def run_ec2_action(action_name): def get_interfaces_and_sgs(): threads = [] with ThreadPoolExecutor(max_workers=2) as executor: - threads.append(executor.submit( - run_ec2_action, 'describe_security_groups')) - threads.append(executor.submit( - run_ec2_action, 'describe_network_interfaces')) + threads.append(executor.submit(run_ec2_action, 'describe_security_groups')) + threads.append(executor.submit(run_ec2_action, 'describe_network_interfaces')) for future in as_completed(threads): action_name, data = future.result() @@ -904,10 +977,11 @@ def get_interfaces_and_sgs(): all_sgs = data elif action_name == 'describe_network_interfaces': all_ec2_interfaces = data - return all_ec2_interfaces, all_sgs + return all_ec2_interfaces, all_sgs def main(): + if len(sys.argv) < 2: all_sgs = Boto().aws('ec2', 'describe_security_groups') sgs_list = [] @@ -933,6 +1007,7 @@ def main(): else: cache = False all_ec2_interfaces, all_sgs = get_interfaces_and_sgs() + save_data('/tmp/ec2_interfaces.pickle', all_ec2_interfaces) save_data('/tmp/sgs.pickle', all_sgs) @@ -968,8 +1043,8 @@ def main(): sg_interfaces = [x for x in all_ec2_interfaces if sg_id in [ y['GroupId'] for y in x['Groups']]] - - attached_resources = get_attached_resources(sg_interfaces) + + attached_resources = get_attached_resources(sg_interfaces, sg_id) if attached_resources: print_blue('Attached to Resources: ') @@ -1010,7 +1085,7 @@ def main(): print(f'\t- {ref_sg}', end='') ref_sg_interfaces = [x for x in all_ec2_interfaces if ref_sg_id in [ y['GroupId'] for y in x['Groups']]] - ref_attached_resources = get_attached_resources(ref_sg_interfaces) + ref_attached_resources = get_attached_resources(ref_sg_interfaces, sg_id) ref_sg_data = [x for x in all_sgs if x['GroupId'] == ref_sg_id][0] ref_inbound_rules_raw = ref_sg_data['IpPermissions'] ref_inbound_rules = get_rules(ref_inbound_rules_raw, sgs_names) @@ -1053,3 +1128,4 @@ def main(): main() except KeyboardInterrupt: print_red("\r \nInterrupted by Ctrl+C\n") + From 8612f7b54a08ef987748451340618da1906112b4 Mon Sep 17 00:00:00 2001 From: Richard Knechtel Date: Mon, 12 Sep 2022 13:03:12 -0500 Subject: [PATCH 2/4] Added commnets and changelog --- sg-tool/README.md | 9 +++++++++ sg-tool/sg.py | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/sg-tool/README.md b/sg-tool/README.md index 8bb8391..80cd026 100644 --- a/sg-tool/README.md +++ b/sg-tool/README.md @@ -6,6 +6,15 @@ This tool can be used to find ALL information related to a Security Group: - Attached Resources - References in other SGs +-- + +**Note:** +*Updates to this tool were done by:* +Name: Richard Knechtel +Company: Blast Motion +Date: 06/29/2022 + +-- ## Python Virtual Environment Setup (Linux) diff --git a/sg-tool/sg.py b/sg-tool/sg.py index 0592143..5a4cecc 100755 --- a/sg-tool/sg.py +++ b/sg-tool/sg.py @@ -13,6 +13,16 @@ import sys import time +# ****************************************************************************************************************** +# Change Log: +# Date Who Company Change(s): +# ------------------------------------------------------------------------------------------------------------------ +# 06/29/2022 Richard Knechtel Blast Motion Showing Network Interfaces and Security Group is attached to, +# Showing all lambdas an SG is attached to, +# Added usage for a python virtual environment +# ------------------------------------------------------------------------------------------------------------------ +# +# ****************************************************************************************************************** # Console Text Colors: BLUE = '\033[94m' From 40c8ddfed643b31fbca207f11bfbcede4298e82e Mon Sep 17 00:00:00 2001 From: Richard Knechtel Date: Wed, 14 Sep 2022 12:32:17 -0500 Subject: [PATCH 3/4] Added Python Virtual Environment stuff for eni-tool --- eni-tool/README.md | 44 ++++++++++++++++++++++++++++++++----- eni-tool/requirements.txt | 46 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 5 deletions(-) create mode 100644 eni-tool/requirements.txt diff --git a/eni-tool/README.md b/eni-tool/README.md index c1e269b..d503954 100644 --- a/eni-tool/README.md +++ b/eni-tool/README.md @@ -1,7 +1,41 @@ # ENI Tool -This tool can be used to find the resource that is using a specific IP address, or all resources using network interfaces +This tool can be used to find the resource that is using a specific IP address, or all resources using network interfaces + + +## Python Virtual Environment Setup (Linux) + +**Create the Virtual Environment (Example):** +python3 -m venv ~/projects/aws-tools/eni-tool/v-env + +**Activate the Virtual Environment (Example):** +source ~/projects/aws-tools/eni-tool/v-env/bin/activate + +**Generate Requirements for project:** +To create requirements.txt: + +1) Setup virtual environment +2) Install all python packages + Example: +~/projects/aws-tools/eni-tool/v-env/bin/pip3 install +3) Note: Make sure to upgrade pip +~/projects/aws-tools/eni-tool/v-env/bin/pip3 install --upgrade pip +4) run: +[Path to Virtual Environment Bin Directory]/pip3 freeze > requirements.txt +Example (Linux): +~/projects/aws-tools/eni-tool/v-env/bin/pip3 freeze > requirements.txt + +**Install the Requirements/Dependancies (Example):** +~/projects/aws-tools/eni-tool/v-env/bin/pip3 install -r requirements.txt + +**Example usage:** + +**Template:** + +`./eni.py ` + +**Example Call:** + +`./eni.py 192.168.10.10` +`./eni.py all` + -Examples: -``` -./eni.py 192.168.10.10 -./eni.py all \ No newline at end of file diff --git a/eni-tool/requirements.txt b/eni-tool/requirements.txt new file mode 100644 index 0000000..78448be --- /dev/null +++ b/eni-tool/requirements.txt @@ -0,0 +1,46 @@ +async-generator==1.10 +attrs==22.1.0 +beautifulsoup4==4.11.1 +boto3==1.24.72 +botocore==1.27.72 +certifi==2022.6.15.2 +charset-normalizer==2.1.1 +cycler==0.11.0 +DateTime==4.7 +demjson==2.2.4 +dnspython==2.2.1 +fonttools==4.37.1 +h11==0.13.0 +idna==3.4 +jmespath==1.0.1 +jtutils==0.0.8 +kiwisolver==1.4.4 +leven==1.0.4 +matplotlib==3.5.3 +netaddr==0.8.0 +nose==1.3.7 +numpy==1.23.3 +outcome==1.2.0 +packaging==21.3 +pandas==1.4.4 +Pillow==9.2.0 +pkg_resources==0.0.0 +pyparsing==3.0.9 +PySocks==1.7.1 +python-csv==0.0.13 +python-dateutil==2.8.2 +pytz==2022.2.1 +requests==2.28.1 +s3transfer==0.6.0 +selenium==4.4.3 +six==1.16.0 +sniffio==1.3.0 +sortedcontainers==2.4.0 +soupsieve==2.3.2.post1 +trio==0.21.0 +trio-websocket==0.9.2 +urllib3==1.26.12 +wsproto==1.2.0 +xlrd==2.0.1 +xmltodict==0.13.0 +zope.interface==5.4.0 From b99f342d16b5a63564b5e6b76d10015caa1cbc10 Mon Sep 17 00:00:00 2001 From: Richard Knechtel <5502779+rknechtel@users.noreply.github.com> Date: Fri, 23 Sep 2022 09:28:26 -0500 Subject: [PATCH 4/4] Updated sg-tool to handl Security Group rules with both IPV4 and IPV6 IP addresses --- sg-tool/sg.py | 67 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/sg-tool/sg.py b/sg-tool/sg.py index 5a4cecc..889e4d8 100755 --- a/sg-tool/sg.py +++ b/sg-tool/sg.py @@ -13,16 +13,6 @@ import sys import time -# ****************************************************************************************************************** -# Change Log: -# Date Who Company Change(s): -# ------------------------------------------------------------------------------------------------------------------ -# 06/29/2022 Richard Knechtel Blast Motion Showing Network Interfaces and Security Group is attached to, -# Showing all lambdas an SG is attached to, -# Added usage for a python virtual environment -# ------------------------------------------------------------------------------------------------------------------ -# -# ****************************************************************************************************************** # Console Text Colors: BLUE = '\033[94m' @@ -200,7 +190,8 @@ def get_rules(raw_list, sgs_names): ip_protocol = rule.get('IpProtocol') from_port = rule.get('FromPort') to_port = rule.get('ToPort') - cidr_info = rule.get('IpRanges') + cidr4_info = rule.get('IpRanges') # IPV4 + cidr6_info = rule.get('Ipv6Ranges') # IPV6 group_info = rule.get('UserIdGroupPairs') if ip_protocol: @@ -219,18 +210,49 @@ def get_rules(raw_list, sgs_names): else: ports = "%s %s-%s" % (ip_protocol, from_port, to_port) - if cidr_info: - for cidr_one_info in cidr_info: - cidr = cidr_one_info.get('CidrIp') - description = cidr_one_info.get('Description') - if cidr: - cidr = re.sub(r'^(.+)/32$', r'\1', cidr) - cidr = re.sub(r'^0\.0\.0\.0/0$', 'ANY', cidr) - + # IPV4 + if cidr4_info: + for cidr4_one_info in cidr4_info: + + description = cidr4_one_info.get('Description') if not description: description = '' - - rules.append([ports, cidr, description]) + + cidr4 = cidr4_one_info.get('CidrIp') + if cidr4: + cidr4 = re.sub(r'^(.+)/32$', r'\1', cidr4) + cidr4 = re.sub(r'^0\.0\.0\.0/0$', 'ANY', cidr4) + + rules.append([ports, cidr4, description]) + + # IPV6 + if cidr6_info: + for cidr6_one_info in cidr6_info: + + description = cidr6_one_info.get('Description') + if not description: + description = '' + + cidr6 = cidr6_one_info.get('CidrIpv6') + if cidr6: + cidr6 = re.sub(r'''(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}| + ([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:) + {1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1 + ,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4} + :){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{ + 1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA + -F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a + -fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0 + -9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0, + 4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1} + :){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9 + ])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0 + -9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4] + |1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4] + |1{0,1}[0-9]){0,1}[0-9]))''', r'\1', cidr6) + cidr6 = re.sub(r'^::/0$', 'ANY', cidr6) + + rules.append([ports, cidr6, description]) if group_info: for group_one_info in group_info: @@ -1095,7 +1117,7 @@ def main(): print(f'\t- {ref_sg}', end='') ref_sg_interfaces = [x for x in all_ec2_interfaces if ref_sg_id in [ y['GroupId'] for y in x['Groups']]] - ref_attached_resources = get_attached_resources(ref_sg_interfaces, sg_id) + ref_attached_resources = get_attached_resources(ref_sg_interfaces, ref_sg) ref_sg_data = [x for x in all_sgs if x['GroupId'] == ref_sg_id][0] ref_inbound_rules_raw = ref_sg_data['IpPermissions'] ref_inbound_rules = get_rules(ref_inbound_rules_raw, sgs_names) @@ -1138,4 +1160,3 @@ def main(): main() except KeyboardInterrupt: print_red("\r \nInterrupted by Ctrl+C\n") -