Merge pull request #64 from kuettai/main

Tagging supports #55 and Fixed issue #62

README.md

@@ -28,7 +28,11 @@ Running this tool is free as it is covered under the AWS Free Tier. If you have
 1. [Log in to your AWS account](https://docs.aws.amazon.com/cloudshell/latest/userguide/getting-started.html#start-session) using the IAM User with sufficient permissions described above. 
 2. Launch [AWS CloudShell](https://docs.aws.amazon.com/cloudshell/latest/userguide/getting-started.html#launch-region-shell) in any region. 
 
+<details>
+<summary>Launch AWS Cloudshell Walkthrough</summary>
+
 ![Launch AWS CloudShell](https://d39bs20xyg7k53.cloudfront.net/services-screener/p1-cloudshell.gif)
+</details>
 
 In the AWS CloudShell terminal, run this script this to install the dependencies:
 ```bash
@@ -42,8 +46,11 @@ pip install -r requirements.txt
 alias screener="python3 $(pwd)/main.py"
 
 ```
-
+<details>
+<summary>Install Dependecies Walkthrough</summary>
+
 ![Install dependencies](https://d39bs20xyg7k53.cloudfront.net/services-screener/p2-dependencies.gif)
+</details>
 
 ## Using Service Screener
 When running Service Screener, you will need to specify the regions and services you would like it to run on. It currently supports Amazon Cloudfront, AWS Cloudtrail, Amazon Dynamodb, Amazon EC2, Amazon EFS, Amazon RDS, Amazon EKS, Amazon Elasticache, Amazon Guardduty, AWS IAM, Amazon Opensearch, AWS Lambda, and Amazon S3.
@@ -72,7 +79,6 @@ screener --regions ap-southeast-1,us-east-1 --services rds,iam
 
 **Example 5: Run in the Singapore region, filter resources based on tags (e.g: Name=env Values=prod and Name=department Values=hr,coe)**
 ```
-## NOT SUPPORTED YET, TO BE RELEASED SOON
 screener --regions ap-southeast-1 --filters env=prod%department=hr,coe
 ```
 
@@ -90,12 +96,19 @@ screener --regions ALL
 # api-raw: raw findings
 # report: generate default web html
 ```
+<details>
+<summary>Get Report Walkthrough</summary>
+
 ![Get Report](https://d39bs20xyg7k53.cloudfront.net/services-screener/p3-getreport.gif)
+</details>
 
 ### Downloading the report
 The output is generated as a ~/service-screener-v2/output.zip file. 
 You can [download the file](https://docs.aws.amazon.com/cloudshell/latest/userguide/working-with-cloudshell.html#files-storage) in the CloudShell console by clicking the *Download file* button under the *Actions* menu on the top right of the Cloudshell console. 
 
+<details>
+<summary>Download Output & Report Viewing Walkthrough</summary>
+
 ![Download Output](https://d39bs20xyg7k53.cloudfront.net/services-screener/p4-outputzip.gif)
 
 Once downloaded, unzip the file and open 'index.html' in your browser. You should see a page like this:
@@ -104,8 +117,13 @@ Once downloaded, unzip the file and open 'index.html' in your browser. You shoul
 
 Ensure that you can see the service(s) run on listed on the left pane.
 You can navigate to the service(s) listed to see detailed findings on each service. 
+</details>
 
+<details>
+<summary>Sample Output Walkthrough</summary>
+
 ![Sample Output](https://d39bs20xyg7k53.cloudfront.net/services-screener/p5-sample.gif)
+</details>
 
 ## Using the report 
 The report provides you an easy-to-navigate dashboard of the various best-practice checks that were run. 

info.json

@@ -1 +1 @@
-{"cloudfront": 8, "cloudtrail": 17, "dynamodb": 24, "ec2": 49, "efs": 3, "eks": 7, "elasticache": 10, "guardduty": 4, "iam": 32, "kms": 2, "lambda": 14, "opensearch": 18, "rds": 65, "s3": 12}
+{"cloudfront": 8, "cloudtrail": 17, "dynamodb": 24, "ec2": 49, "efs": 3, "eks": 7, "elasticache": 10, "guardduty": 4, "iam": 32, "kms": 2, "lambda": 14, "opensearch": 18, "rds": 77, "s3": 12}

services/Service.py

@@ -38,21 +38,6 @@ def setRules(self, rules):
         rules = rules.lower().split('^')
         Config.set(self.RULESPREFIX, rules)
 
-    def setTags(self, tags):
-        rawTags = []
-        if not tags:
-            return
-
-        result = []
-        t = tags.split(self.TAGS_SEPARATOR)
-        for tag in t:
-            k, v = tag.split(self.KEYVALUE_SEPARATOR)
-            rawTags = {k: v.split(self.VALUES_SEPARATOR) for k, v in tag.items()}
-            result.append({"Name": "tag:" + k, "Values": v.split(self.VALUES_SEPARATOR)})
-
-        self._tags = rawTags
-        self.tags = result
-
     def __del__(self):
         timespent = round(time.time() - self.overallTimeStart, 3)
         print('\033[1;42mCOMPLETED\033[0m -- ' + self.__class__.__name__.upper() + '::'+self.region+' (' + str(timespent) + 's)')
@@ -67,6 +52,23 @@ def __del__(self):
 
         Config.set(self.RULESPREFIX, [])
 
+    def setTags(self, tags):
+        rawTags = {}
+        if not tags:
+            return
+
+        result = []
+        t = tags.split(self.TAGS_SEPARATOR)
+        for tag in t:
+            k, v = tag.split(self.KEYVALUE_SEPARATOR)
+            rawTags[k] = v.split(self.VALUES_SEPARATOR)
+            result.append({"Name": "tag:" + k, "Values": v.split(self.VALUES_SEPARATOR)})
+
+        self._tags = rawTags
+        self.tags = result
+
+        # print(self._tags, self.tags)
+
     def resourceHasTags(self, tags):
         if not self._tags:
             return True
@@ -94,6 +96,22 @@ def resourceHasTags(self, tags):
                 return False
 
         return True    
+
+    # convert normal keypair to tag format
+    # {env: prod, costcenter: hr} => [{'Key': 'env', 'Value': 'prod'}, {'Key': 'costcenter', 'Value': 'hr'}] 
+    def convertKeyPairTagToTagFormat(self, tags):
+        nTags = []
+        for k, v in tags.items():
+            nTags.append({'Key': k, 'Value': v})
+
+        return nTags
+
+    def convertTagKeyTagValueIntoKeyValue(self, tags):
+        nTags = []
+        for i in tags:
+            nTags.append({'Key': i['TagKey'], 'Value': i['TagValue']})
+
+        return nTags
 
 if __name__ == "__main__":
     Config.init()

services/cloudfront/Cloudfront.py

@@ -24,13 +24,22 @@ def getDistributions(self):
         while True:
             if "DistributionList" in response and "Items" in response["DistributionList"]:
                 for dist in response["DistributionList"]["Items"]:
-                    arr.append(dist["Id"])
+                    toAppend = True
+                    if self.tags:
+                        myTags = self.cloudfrontClient.list_tags_for_resource(Resource=dist['ARN'])
+                        if self.resourceHasTags(myTags.get('Tags')['Items']) == False:
+                            toAppend = False
+
+                    if toAppend:    
+                        arr.append(dist["Id"])
+
                 if "NextMarker" not in response["DistributionList"]:
                     break
 
                 response = self.cloudfrontClient.list_distributions(Marker=response["DistributionList"]["NextMarker"])
             else:
                 break
+
         return arr
 
 

services/cloudtrail/Cloudtrail.py

@@ -4,6 +4,7 @@
 import time
 
 from utils.Config import Config
+from botocore.config import Config as bConfig
 from services.Service import Service
 from services.cloudtrail.drivers.CloudtrailCommon import CloudtrailCommon
 from services.cloudtrail.drivers.CloudtrailAccount import CloudtrailAccount
@@ -30,7 +31,24 @@ def getTrails(self):
             resp = ctClient.list_trails(NextToken = resp.get('NextToken'))
             results += resp.get('Trails')
 
-        return results
+
+        if not self.tags:
+            return results    
+
+        finalArr = []
+        for i, detail in enumerate(results):
+            ctInfo = detail['TrailARN'].split(':')
+
+            ## despite cloudtrail seems like a "global api", for list_tags, need to call based on region tho.
+            ## need to create separate boto instance for that region
+            myTmpCtClient = self.ssBoto.client('cloudtrail', config=bConfig(region_name=ctInfo[3]))
+            tags = myTmpCtClient.list_tags(ResourceIdList=[detail['TrailARN']])
+
+            if self.resourceHasTags(tags.get('ResourceTagList')[0]['TagsList']):
+                finalArr.append(results[i])
+
+        return finalArr
+
 
     def advise(self):
         ## Will loop through all trail, and set to True if any has MultiRegion

services/dynamodb/Dynamodb.py

@@ -40,7 +40,17 @@ def list_tables(self):
                     tableDescription = self.dynamoDbClient.describe_table(TableName = tables)
                     tableArr.append(tableDescription)
 
-            return tableArr 
+            if not self.tags:
+                return tableArr 
+
+            finalArr = []
+            for i, detail in enumerate(tableArr):
+                tableArn = detail['Table']['TableArn']
+                tags = self.dynamoDbClient.list_tags_of_resource(ResourceArn=tableArn)
+                if self.resourceHasTags(tags.get('Tags')):
+                    finalArr.append(tableArr[i])
+
+            return finalArr
 
         except botocore.exceptions.ClientError as e:
             ecode = e.response['Error']['Code']

services/ec2/Ec2.py

@@ -112,7 +112,15 @@ def getEC2SecurityGroups(self,instance):
                 )
             arr = arr + results.get('SecurityGroups')
 
-        return arr
+        if not self.tags:
+            return arr
+
+        finalArr = []
+        for i, detail in enumerate(arr):
+            if 'Tags' in detail and self.resourceHasTags(detail['Tags']):
+                finalArr.append(arr[i])
+
+        return finalArr    
 
     def getEBSResources(self):
         filters = []
@@ -146,24 +154,22 @@ def getELB(self):
 
         ## TO DO: support tagging later
 
-        # if self.tags is None:
-        #     return arr
-
-        # filteredResults = []
-        # for lb in arr:
-        #     tagResults = self.elbClient.describe_tags(
-        #         ResourceArns = [lb['LoadBalancerArn']]
-        #     )
-        #     tagDesc = tagResults.get('TagDescriptions')
-        #     if len(tagDesc) > 0:
-        #         for desc in tagDesc:
-        #             if self.resourceHasTags(desc['Tags']):
-        #                 filteredResults.append(lb)
-        #                 break
-
-        # return filteredResults
+        if not self.tags:
+            return arr
 
-        return arr
+        filteredResults = []
+        for lb in arr:
+            tagResults = self.elbClient.describe_tags(
+                ResourceArns = [lb['LoadBalancerArn']]
+            )
+            tagDesc = tagResults.get('TagDescriptions')
+            if len(tagDesc) > 0:
+                for desc in tagDesc:
+                    if self.resourceHasTags(desc['Tags']):
+                        filteredResults.append(lb)
+                        break
+
+        return filteredResults
 
     def getELBClassic(self):
         results = self.elbClassicClient.describe_load_balancers()
@@ -208,6 +214,14 @@ def getELBSecurityGroup(self, elb):
                 NextToken = results.get('NextToken')
             )
             arr = arr + results.get('SecurityGroups')
+
+        if not self.tags:
+            return arr
+
+        finalArr = []
+        for i, detail in enumerate(arr):
+            if self.resourceHasTags(detail['Tags']):
+                finalArr.append(arr[i])
 
         return arr
 
@@ -240,7 +254,15 @@ def getEIPResources(self):
         )
         arr = result.get('Addresses')
 
-        return arr
+        if not self.tags:
+            return arr
+
+        finalArr = []
+        for i, detail in enumerate(arr):
+            if 'Tags' in detail and self.resourceHasTags(detail['Tags']):
+                finalArr.append(arr[i])
+
+        return finalArr    
 
     def getDefaultSG(self):
         defaultSGs = {}
@@ -257,7 +279,16 @@ def getDefaultSG(self):
                 if group.get('GroupName') == 'default':
                     defaultSGs[group.get('GroupId')] = group
 
-        return defaultSGs
+        if not self.tags:
+            return defaultSGs
+
+        finalArr = []
+
+        for i, detail in defaultSGs.items():
+            if 'Tags' in detail and self.resourceHasTags(detail['Tags']):
+                finalArr.append(defaultSGs[i])
+
+        return finalArr
 
     def advise(self):
         objs = {}
@@ -362,18 +393,20 @@ def advise(self):
             objs[f"ASG::{group['AutoScalingGroupName']}"] = obj.getInfo()
 
         defaultSGs = self.getDefaultSG()
-        for groupId in defaultSGs.keys():
-            if groupId not in secGroups:
-                secGroups[groupId] = defaultSGs[groupId]
-                secGroups[groupId]['inUsed'] = 'False'
+        if defaultSGs:
+            for groupId in defaultSGs.keys():
+                if groupId not in secGroups:
+                    secGroups[groupId] = defaultSGs[groupId]
+                    secGroups[groupId]['inUsed'] = 'False'
 
         # SG checks
-        for group in secGroups.values():
-            print(f"... (EC2::Security Group) inspecting {group['GroupId']}")
-            obj = Ec2SecGroup(group, self.ec2Client)
-            obj.run(self.__class__)
-
-            objs[f"SG::{group['GroupId']}"] = obj.getInfo()
+        if secGroups:
+            for group in secGroups.values():
+                print(f"... (EC2::Security Group) inspecting {group['GroupId']}")
+                obj = Ec2SecGroup(group, self.ec2Client)
+                obj.run(self.__class__)
+
+                objs[f"SG::{group['GroupId']}"] = obj.getInfo()
 
         # EIP checks    
         eips = self.getEIPResources()

services/efs/Efs.py

@@ -23,7 +23,7 @@ def get_resources(self):
 
         filtered_results = []
         for efs in results:
-            if self.resource_has_tags(efs['Tags']):
+            if self.resourceHasTags(efs['Tags']):
                 filtered_results.append(efs)
 
         return filtered_results

services/eks/Eks.py

@@ -44,9 +44,16 @@ def advise(self):
         for cluster in clusters:
             print('...(EKS:Cluster) inspecting ' + cluster)
             clusterInfo = self.describeCluster(cluster)
-            if clusterInfo.get('status') == 'CREATING':
-                print(cluster + " cluster is creating. Skipped")
-                continue
+
+            #if clusterInfo.get('status') == 'CREATING':
+            #    print(cluster + " cluster is creating. Skipped")
+            #    continue
+
+            if self.tags:
+                resp = self.eksClient.list_tags_for_resource(resourceArn=clusterInfo['arn'])
+                nTags = self.convertKeyPairTagToTagFormat(resp.get('tags'))
+                if self.resourceHasTags(nTags) == False:
+                    continue
 
             obj = EksCommon(cluster, clusterInfo, self.eksClient, self.ec2Client, self.iamClient)
             obj.run(self.__class__)