From 2dc7cd97a9686e322255ee636c5c9322215f29ac Mon Sep 17 00:00:00 2001 From: KuetTai Date: Tue, 22 Oct 2024 19:26:04 +0800 Subject: [PATCH 01/15] Fixed duplicated S3 output on findings --- services/s3/S3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/s3/S3.py b/services/s3/S3.py index 169e2ad..709c1b8 100644 --- a/services/s3/S3.py +++ b/services/s3/S3.py @@ -96,8 +96,8 @@ def advise(self): objs["Account::Control"] = obj.getInfo() - globalKey = 'GLOBALRESOURCES_s3' - Config.set(globalKey, objs) + # globalKey = 'GLOBALRESOURCES_s3' + # Config.set(globalKey, objs) Config.set('S3_HasAccountScanned', True) del obj From 3e135b89fbaa46f9e4ee33787980ac82618662de Mon Sep 17 00:00:00 2001 From: KuetTai Date: Tue, 22 Oct 2024 21:20:14 +0800 Subject: [PATCH 02/15] Improve logic on Concurreny Running --- main.py | 9 +++- services/Evaluator.py | 104 ++++++++++++++++++++++++++++++++---------- services/iam/Iam.py | 4 +- services/s3/S3.py | 4 -- 4 files changed, 90 insertions(+), 31 deletions(-) diff --git a/main.py b/main.py index eb2a382..266b87f 100644 --- a/main.py +++ b/main.py @@ -218,8 +218,15 @@ def number_format(num, places=2): pass input_ranges = [] + + ## Force IAM to run first, it takes the longest time + if 'IAM' in services: + input_ranges = [('IAM', regions, filters)] + for service in services: - input_ranges = [(service, regions, filters) for service in services] + otherInputs = [(service, regions, filters) for service in services] + + input_ranges.extend(otherInputs) pool = Pool(processes=int(workerCounts)) pool.starmap(Screener.scanByService, input_ranges) diff --git a/services/Evaluator.py b/services/Evaluator.py index b33246c..3afef3b 100644 --- a/services/Evaluator.py +++ b/services/Evaluator.py @@ -1,12 +1,43 @@ import traceback import botocore import time +import copy + +import concurrent.futures as cf from utils.Config import Config from utils.Tools import _warn, _info from utils.CustomPage.CustomPage import CustomPage import constants as _C +def runSingleCheck(tmp_obj, method_name): + debugFlag = Config.get('DEBUG') + obj = tmp_obj + try: + startTime = time.time() + if debugFlag: + print('--- --- fn: ' + method_name) + + getattr(obj, method_name)() + if debugFlag: + timeSpent = round(time.time() - startTime, 3) + if timeSpent >= 0.2: + _warn("Long running checks {}s".format(timeSpent)) + + getattr(obj, method_name)() + return 'OK' + except botocore.exceptions.ClientError as e: + code = e.response['Error']['Code'] + msg = e.response['Error']['Message'] + print(code, msg) + print(traceback.format_exc()) + traceback.format_exc() + except Exception: + print(traceback.format_exc()) + traceback.format_exc() + + return traceback.format_exc() + class Evaluator(): def __init__(self): self.init() @@ -25,42 +56,65 @@ def getII(self, k): else: _warn("{} is not found in drivers/{}.InventoryInfo".format(k, self.classname), forcePrint=False) return None - + def run(self, serviceName): servClass = self.classname rulePrefix = serviceName.__name__ + '::rules' + servMethods = servClass + '::methods' rules = Config.get(rulePrefix, []) debugFlag = Config.get('DEBUG') ecnt = cnt = 0 emsg = [] - methods = [method for method in dir(self) if method.startswith('__') is False and method.startswith('_check') is True] - for method in methods: - if not rules or str.lower(method[6:]) in rules: - try: - - startTime = time.time() - if debugFlag: - print('--- --- fn: ' + method) + + #Improve of methods scanning + methods = Config.get(servMethods, []) + if methods == []: + methods = [method for method in dir(self) if method.startswith('__') is False and method.startswith('_check') is True] + Config.set(servMethods, methods) + + filteredMethods = [method for method in methods if not rules or method[6:].lower() in rules] + + cnt = len(filteredMethods) + runUsingConcurrent = 1 + if runUsingConcurrent: + with cf.ThreadPoolExecutor(max_workers=4) as executor: + futures = [executor.submit(runSingleCheck, self, method) for method in filteredMethods] + + for future in cf.as_completed(futures): + for fr in future.result(): + if fr == 'OK': + continue + else: + emsg.append(fr) + ecnt += 1 + + else: + for method in methods: + if not rules or str.lower(method[6:]) in rules: + try: - getattr(self, method)() - if debugFlag: - timeSpent = round(time.time() - startTime, 3) - if timeSpent >= 0.2: - _warn("Long running checks {}s".format(timeSpent)) + startTime = time.time() + if debugFlag: + print('--- --- fn: ' + method) + + getattr(self, method)() + if debugFlag: + timeSpent = round(time.time() - startTime, 3) + if timeSpent >= 0.2: + _warn("Long running checks {}s".format(timeSpent)) - cnt += 1 - except botocore.exceptions.ClientError as e: - code = e.response['Error']['Code'] - msg = e.response['Error']['Message'] - print(code, msg) - print(traceback.format_exc()) - emsg.append(traceback.format_exc()) - except Exception: - ecnt += 1 - print(traceback.format_exc()) - emsg.append(traceback.format_exc()) + except botocore.exceptions.ClientError as e: + code = e.response['Error']['Code'] + msg = e.response['Error']['Message'] + print(code, msg) + print(traceback.format_exc()) + emsg.append(traceback.format_exc()) + except Exception: + ecnt += 1 + print(traceback.format_exc()) + emsg.append(traceback.format_exc()) if emsg: with open(_C.FORK_DIR + '/error.txt', 'a+') as f: diff --git a/services/iam/Iam.py b/services/iam/Iam.py index b1106e6..c9c8305 100644 --- a/services/iam/Iam.py +++ b/services/iam/Iam.py @@ -182,7 +182,9 @@ def _roleFilterByName(self, rn): 'GatedGarden', 'PVRE-SSMOnboarding', 'PVRE-Maintenance', - 'InternalAuditInternal' + 'InternalAuditInternal', + 'isengard-', + 'AWS-QuickSetup', ] for kw in keywords: diff --git a/services/s3/S3.py b/services/s3/S3.py index 709c1b8..6f3acd3 100644 --- a/services/s3/S3.py +++ b/services/s3/S3.py @@ -93,12 +93,8 @@ def advise(self): print('... (S3Account) inspecting ') obj = S3Control(self.s3Control) obj.run(self.__class__) - objs["Account::Control"] = obj.getInfo() - # globalKey = 'GLOBALRESOURCES_s3' - # Config.set(globalKey, objs) - Config.set('S3_HasAccountScanned', True) del obj From 3580bc1f1c6fb6852a770e774e9f53082fea361e Mon Sep 17 00:00:00 2001 From: KuetTai Date: Tue, 22 Oct 2024 21:26:47 +0800 Subject: [PATCH 03/15] Use default max_worker value for ThreadPoolExecutor --- services/Evaluator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/Evaluator.py b/services/Evaluator.py index 3afef3b..a1cf030 100644 --- a/services/Evaluator.py +++ b/services/Evaluator.py @@ -1,7 +1,8 @@ import traceback import botocore import time -import copy +import os +import math import concurrent.futures as cf @@ -79,7 +80,7 @@ def run(self, serviceName): cnt = len(filteredMethods) runUsingConcurrent = 1 if runUsingConcurrent: - with cf.ThreadPoolExecutor(max_workers=4) as executor: + with cf.ThreadPoolExecutor() as executor: futures = [executor.submit(runSingleCheck, self, method) for method in filteredMethods] for future in cf.as_completed(futures): From 058b5935c86e4a74032a48ef038920d8b2b5be93 Mon Sep 17 00:00:00 2001 From: KuetTai Date: Tue, 22 Oct 2024 21:35:02 +0800 Subject: [PATCH 04/15] tmp - WA update --- frameworks/Framework.py | 9 ++ frameworks/WAFS/WAFS.py | 80 +++++++++++-- frameworks/WAFS/map.json | 9 +- frameworks/helper/WATools.py | 223 +++++++++++++++++++++++++++++++++++ 4 files changed, 309 insertions(+), 12 deletions(-) create mode 100644 frameworks/helper/WATools.py diff --git a/frameworks/Framework.py b/frameworks/Framework.py index 58adc76..860a522 100644 --- a/frameworks/Framework.py +++ b/frameworks/Framework.py @@ -37,6 +37,12 @@ def getMetaData(self): # To be overwrite if needed def _hookGenerateMetaData(self): pass + + def _hookPostItemActivity(self, title, section, checks, comp): + return title, section, checks, comp + + def _hookPostItemsLoop(self): + pass # ['Main', 'ARC-003', 0, '[iam,rootMfaActive] Root ID, Admin
[iam.passwordPolicy] sss', 'Link 1
Link2'] def generateMappingInformation(self): @@ -70,6 +76,8 @@ def generateMappingInformation(self): pre.append(tmp) checks, links, comp = self.formatCheckAndLinks(pre) + + title, section, checks, comp = self._hookPostItemActivity(title, section, checks, comp) outp.append([title, section, comp, checks, links]) pos = comp @@ -78,6 +86,7 @@ def generateMappingInformation(self): summ[title][pos] += 1 + self._hookPostItemsLoop() self.stats = summ return outp diff --git a/frameworks/WAFS/WAFS.py b/frameworks/WAFS/WAFS.py index 2c8cd85..4b0026d 100644 --- a/frameworks/WAFS/WAFS.py +++ b/frameworks/WAFS/WAFS.py @@ -1,17 +1,81 @@ -import json +import json, re import constants as _C +from utils.Config import Config from frameworks.Framework import Framework +from frameworks.helper.WATools import WATools class WAFS(Framework): + WATools = None + ResultCache = {} def __init__(self, data): super().__init__(data) + + waTools = WATools('security') + cliParams = Config.get('_SS_PARAMS') + + tmpParams = {} + if 'others' in cliParams and not cliParams['others'] == None: + params = cliParams['others'] + cfg = json.loads(params) + + if 'WA' in cfg: + tmpParams = cfg['WA'] + + if waTools.preCheck(tmpParams): + self.WATools = waTools + self.WATools.init(tmpParams) + self.WATools.createReportIfNotExists() + self.WATools.listAnswers() + # print(self.WATools.answerSets) + + + def _hookPostItemActivity(self, title, section, checks, comp): + if self.WATools == None: + return title, section, checks, comp + + titleNum = self.extractNumber(title) + sectNum = self.extractNumber(section) + + paired = "{}::{}".format(titleNum, sectNum) + + newChecks = "

{}

{}".format(self.getDescription(titleNum, paired), checks) + + titleKey = self.WATools.answerSets.get(titleNum, [None])[0] + if not titleKey in self.ResultCache: + self.ResultCache[titleKey] = { + "0": [], + "1": [], + "-1": [] + } + + if not titleKey == None: + if comp == 1: + self.ResultCache[titleKey]["1"].append(self.WATools.answerSets.get(paired, [None])[0]) + elif comp == -1: + self.ResultCache[titleKey]["-1"].append(self.WATools.answerSets.get(paired, [None, None])[1]) + else: + self.ResultCache[titleKey]["0"].append(self.WATools.answerSets.get(paired, [None])[0]) + + return title, section, newChecks, comp + + def _hookPostItemsLoop(self): + for title, opts in self.ResultCache.items(): + if len(opts["1"]) == 0 and len(opts["-1"]) == 0: + continue + + ansStr = opts["1"] + unselectedNotes = "***Generated by SS\n\nHere are the items failed SS checks (if any):\n- {}".format("\n- ".join(opts["-1"])) + + self.WATools.updateAnswers(title, ansStr, unselectedNotes) + pass + + def extractNumber(self, s): + match = re.search(r'\d+', s) + return match.group() if match else None -if __name__ == "__main__": - data = json.loads(open(_C.FRAMEWORK_DIR + '/api.json').read()) - # print(data) - o = WARS(data) - o.readFile() - # o.obj() - o.generateMappingInformation() \ No newline at end of file + def getDescription(self, titleNum, paired): + titleStr = self.WATools.answerSets.get(titleNum, [None])[1] + sectStr = self.WATools.answerSets.get(paired, [None])[1] + return f"{titleStr} - {sectStr}" diff --git a/frameworks/WAFS/map.json b/frameworks/WAFS/map.json index 6c56831..208395e 100644 --- a/frameworks/WAFS/map.json +++ b/frameworks/WAFS/map.json @@ -10,10 +10,10 @@ "mapping": { "SEC01": { "BP01": ["iam.hasOrganization"], - "BP02": ["iam.rootMfaActive", "iam.hasAlternateContact", "iam.rootHasAccessKey", "iam.rootConsoleLogin30days", "iam.passwordPolicy", "iam.enableGuardDuty"], + "BP02": ["iam.rootMfaActive", "iam.hasAlternateContact", "iam.rootHasAccessKey", "iam.rootConsoleLogin30days", "iam.passwordPolicy", "iam.enableGuardDuty", "iam.rootConsoleLogin30days"], "BP03": ["iam.mfaActive", "iam.passwordPolicyWeak", "iam.passwordLastChange90", "iam.hasAccessKeyNoRotate30days"], - "BP04": ["iam.enableGuardDuty"], - "BP05": [], + "BP04": ["iam.enableGuardDuty""iam.enableGuardDuty"], + "BP05": ["lambda.$length", "rds.$length", "ecs.$length", "eks.$length", "dynamodb.$length", "elasticache.$length"], "BP06": [], "BP07": [], "BP08": [] @@ -48,11 +48,12 @@ "BP02": ["ec.SGSensitivePortOpenToAll", "ec2.SGAllTCPOpen", "ec2.SGAllUDPOpen", "ec2.SGDefaultInUsed", "ec2.SGEncryptionInTransit", "ec2.ELBListenerInsecure", "rds.PubliclyAccessible"], "BP03": [], "BP04": [] + "BP04": [] }, "SEC06":{ "BP01": [], "BP02": [], - "BP03": ["lambda.$length", "rds.$length", "ecs.$length", "eks.$length", "dynamodb.$length", "elasticache.$length"], + "BP03": [], "BP04": [], "BP05": [], "BP06": [] diff --git a/frameworks/helper/WATools.py b/frameworks/helper/WATools.py new file mode 100644 index 0000000..190d8f7 --- /dev/null +++ b/frameworks/helper/WATools.py @@ -0,0 +1,223 @@ +import boto3, json +from botocore.exceptions import BotoCoreError +from botocore.config import Config as bConfig +from utils.Config import Config +from datetime import datetime + +## --others '{"WA": {"region": "ap-southeast-1", "reportName":"SS_Report", "newMileStone":0}}' + +class WATools(): + DEFAULT_REPORTNAME = 'SS_Report' + DEFAULT_NEWMILESTONE = 0 + waInfo = { + 'isExists': False, + 'WorkloadId': None, + 'LensesAlias': 'wellarchitected' + } + + def __init__(self, pillarId): + self.pillarId = pillarId + pass + + def preCheck(self, params): + if not 'reportName' in params: + params['reportName'] = self.DEFAULT_REPORTNAME + + if not 'newMileStone' in params: + params['newMileStone'] = 0 + + if not 'region' in params: + params['region'] = Config.get('REGIONS_SELECTED')[0] + + return True + + def init(self, cfg): + self.cfg = cfg + self.stsInfo = Config.get('stsInfo') + + boto3Config = bConfig(region_name = cfg['region']) + ssBoto = Config.get('ssBoto', None) + + self.waClient = ssBoto.client('wellarchitected', config=boto3Config) + + + def checkIfReportExists(self): + workload_name = self.cfg['reportName'] + try: + # List workloads with the given name prefix + response = self.waClient.list_workloads( + WorkloadNamePrefix=workload_name, + MaxResults=50 # Adjust this value as needed + ) + + # Check if any workload matches the exact name + for workload in response.get('WorkloadSummaries', []): + if workload['WorkloadName'] == workload_name: + self.waInfo['isExists'] = True + self.waInfo['WorkloadId'] = workload['WorkloadId'] + + except Exception as e: + print(f"An error occurred: {str(e)}") + return False, None + + def createReportIfNotExists(self): + workload_name = self.cfg['reportName'] + self.checkIfReportExists() + + if self.waInfo['isExists'] == True: + return True + + wLargs = { + 'WorkloadName': workload_name, + 'Description': 'Auto generated by ServiceScreener', + 'Environment': 'PRODUCTION', + 'AccountIds': [self.stsInfo['Account']], + 'AwsRegions': Config.get('REGIONS_SELECTED'), + 'ReviewOwner': self.stsInfo['Arn'], + 'Lenses': [self.waInfo['LensesAlias']] + } + + try: + response = self.waClient.create_workload(**wLargs) + self.waInfo['WorkloadId'] = response['WorkloadId'] + return True + except Exception as e: + print(f"An error occurred while creating the workload: {str(e)}") + return False + + def createMilestoneIfNotExists(self): + if self.cfg['newMileStone'] == 1: + self.createMilestone() + return + + all_milestones = [] + next_token = None + + try: + while True: + params = { + 'WorkloadId': self.waInfo['WorkloadId'], + 'MaxResults': 20 + } + if next_token: + params['NextToken'] = next_token + + response = self.waClient.list_milestones(**params) + all_milestones.extend(response['MilestoneSummaries']) + + next_token = response.get('NextToken') + if not next_token: + break # No more pages, exit the loop + + if not all_milestones: + print(f"No milestones found for workload {workload_id}... creating milestone...") + self.createMilestone() + return None + + # Sort milestones by date (most recent first) + sorted_milestones = sorted( + all_milestones, + key=lambda x: x['RecordedAt'], + reverse=True + ) + + # Get the latest milestone + latest_milestone = sorted_milestones[0] + self.waInfo['MilestoneName'] = latest_milestone['MilestoneName'] + self.waInfo['MilestoneNumber'] = latest_milestone['MilestoneNumber'] + + except BotoCoreError as e: + print(f"An error occurred: {str(e)}") + return None + + def createMilestone(self): + cdate = datetime.now().strftime('%Y%m%d%H%M%S') + milestoneName = 'SS-{}'.format(cdate) + + try: + resp = self.waClient.create_milestone( + WorkloadId=self.waInfo['WorkloadId'], + MilestoneName=milestoneName + ) + + print(f"Milestone Number: {resp['MilestoneNumber']}") + + self.waInfo['MilestoneName'] = milestoneName + self.waInfo['MilestoneNumber'] = resp['MilestoneNumber'] + + return True + except BotoCoreError as e: + print(f"An error occurred while creating the milestone: {str(e)}") + return None + + def listAnswers(self): + next_token = None + ansArgs = { + 'WorkloadId': self.waInfo['WorkloadId'], + 'LensAlias': self.waInfo['LensesAlias'], + 'PillarId': self.pillarId, + # 'MilestoneNumber': self.waInfo['MilestoneNumber'], + 'MaxResults': 50 + } + + answers = [] + while True: + if next_token: + ansArgs['nextToken'] = next_token + + resp = self.waClient.list_answers(**ansArgs) + # print(resp['AnswerSummaries']) + answers.extend(resp['AnswerSummaries']) + + next_token = resp.get('NextTOken') + if not next_token: + break + + i = 1 + j = 1 + answerSets = {} + for ans in answers: + j = 1 + answerSets[f'{i:02}'] = [ans['QuestionId'], ans['QuestionTitle']] + # print(ans['QuestionId']) + for choice in ans['Choices']: + # print(choice) + fkey = f'{i:02}::{j:02}' + answerSets[fkey] = [choice['ChoiceId'], choice['Title']] + j = j+1 + + i = i+1 + + self.answerSets = answerSets + + def updateAnswers(self, questionId, selectedChoices, unselectedNotes): + ansArgs = { + 'WorkloadId': self.waInfo['WorkloadId'], + 'LensAlias': self.waInfo['LensesAlias'], + 'QuestionId': questionId, + 'SelectedChoices': selectedChoices, + 'Notes': unselectedNotes + } + + resp = self.waClient.update_answer(**ansArgs) + pass +''' +stsInfo = {'UserId': 'AIDA55JZ3XKTBZPEJU5K7', 'Account': '956288449190', 'Arn': 'arn:aws:iam::956288449190:user/macbook-ss'} +Config.set('stsInfo', stsInfo) + +Config.set('REGIONS_SELECTED', ['ap-southeast-1', 'us-east-1']) + +boto3args = {'region_name': 'ap-southeast-1'} +Config.set('ssBoto', boto3.Session(**boto3args)) + +myCfg = '{"WA": {"region": "ap-southeast-1", "reportName":"SS_Report", "newMileStone":0}}' +cfg = json.loads(myCfg) + +o = WATools() +o.init(cfg['WA']) +o.createReportIfNotExists() +resp = o.listAnswers('security') +# resp = o.createMilestoneIfNotExists() +print(o.waInfo) +print(resp) +''' \ No newline at end of file From 6963782e860618ca10877e0ea8e618e863afcdb8 Mon Sep 17 00:00:00 2001 From: KuetTai Date: Tue, 22 Oct 2024 21:38:51 +0800 Subject: [PATCH 05/15] update mapping --- frameworks/WAFS/map.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frameworks/WAFS/map.json b/frameworks/WAFS/map.json index 208395e..f5f79aa 100644 --- a/frameworks/WAFS/map.json +++ b/frameworks/WAFS/map.json @@ -12,7 +12,7 @@ "BP01": ["iam.hasOrganization"], "BP02": ["iam.rootMfaActive", "iam.hasAlternateContact", "iam.rootHasAccessKey", "iam.rootConsoleLogin30days", "iam.passwordPolicy", "iam.enableGuardDuty", "iam.rootConsoleLogin30days"], "BP03": ["iam.mfaActive", "iam.passwordPolicyWeak", "iam.passwordLastChange90", "iam.hasAccessKeyNoRotate30days"], - "BP04": ["iam.enableGuardDuty""iam.enableGuardDuty"], + "BP04": ["iam.enableGuardDuty"], "BP05": ["lambda.$length", "rds.$length", "ecs.$length", "eks.$length", "dynamodb.$length", "elasticache.$length"], "BP06": [], "BP07": [], @@ -48,7 +48,6 @@ "BP02": ["ec.SGSensitivePortOpenToAll", "ec2.SGAllTCPOpen", "ec2.SGAllUDPOpen", "ec2.SGDefaultInUsed", "ec2.SGEncryptionInTransit", "ec2.ELBListenerInsecure", "rds.PubliclyAccessible"], "BP03": [], "BP04": [] - "BP04": [] }, "SEC06":{ "BP01": [], From 8e55393cc656484f878b384c5c1e95c254a2c25c Mon Sep 17 00:00:00 2001 From: KuetTai Date: Tue, 22 Oct 2024 22:00:09 +0800 Subject: [PATCH 06/15] [Beta Features] Auto create WA reviews --- frameworks/WAFS/WAFS.py | 5 +++- frameworks/helper/WATools.py | 50 +++++++++++++++++++++++++++--------- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/frameworks/WAFS/WAFS.py b/frameworks/WAFS/WAFS.py index 4b0026d..06a7f16 100644 --- a/frameworks/WAFS/WAFS.py +++ b/frameworks/WAFS/WAFS.py @@ -31,7 +31,7 @@ def __init__(self, data): def _hookPostItemActivity(self, title, section, checks, comp): - if self.WATools == None: + if self.WATools == None or self.WATools.HASPERMISSION == False: return title, section, checks, comp titleNum = self.extractNumber(title) @@ -60,6 +60,9 @@ def _hookPostItemActivity(self, title, section, checks, comp): return title, section, newChecks, comp def _hookPostItemsLoop(self): + if self.WATools == None or self.WATools.HASPERMISSION == False: + return + for title, opts in self.ResultCache.items(): if len(opts["1"]) == 0 and len(opts["-1"]) == 0: continue diff --git a/frameworks/helper/WATools.py b/frameworks/helper/WATools.py index 190d8f7..5c8b9c6 100644 --- a/frameworks/helper/WATools.py +++ b/frameworks/helper/WATools.py @@ -3,6 +3,7 @@ from botocore.config import Config as bConfig from utils.Config import Config from datetime import datetime +from utils.Tools import _warn ## --others '{"WA": {"region": "ap-southeast-1", "reportName":"SS_Report", "newMileStone":0}}' @@ -15,6 +16,8 @@ class WATools(): 'LensesAlias': 'wellarchitected' } + HASPERMISSION = True + def __init__(self, pillarId): self.pillarId = pillarId pass @@ -57,13 +60,17 @@ def checkIfReportExists(self): self.waInfo['WorkloadId'] = workload['WorkloadId'] except Exception as e: - print(f"An error occurred: {str(e)}") + _warn(f"Error checking if workload exists: {str(e)}") + self.HASPERMISSION = False return False, None def createReportIfNotExists(self): workload_name = self.cfg['reportName'] self.checkIfReportExists() + if self.HASPERMISSION == False: + return False + if self.waInfo['isExists'] == True: return True @@ -82,6 +89,7 @@ def createReportIfNotExists(self): self.waInfo['WorkloadId'] = response['WorkloadId'] return True except Exception as e: + self.HASPERMISSION = False print(f"An error occurred while creating the workload: {str(e)}") return False @@ -147,10 +155,14 @@ def createMilestone(self): return True except BotoCoreError as e: - print(f"An error occurred while creating the milestone: {str(e)}") + self.HASPERMISSION = False + _warn(f"An error occurred while creating the milestone: {str(e)}") return None def listAnswers(self): + if self.HASPERMISSION == False: + return None + next_token = None ansArgs = { 'WorkloadId': self.waInfo['WorkloadId'], @@ -161,17 +173,22 @@ def listAnswers(self): } answers = [] - while True: - if next_token: - ansArgs['nextToken'] = next_token + try: + while True: + if next_token: + ansArgs['nextToken'] = next_token - resp = self.waClient.list_answers(**ansArgs) - # print(resp['AnswerSummaries']) - answers.extend(resp['AnswerSummaries']) + resp = self.waClient.list_answers(**ansArgs) + # print(resp['AnswerSummaries']) + answers.extend(resp['AnswerSummaries']) - next_token = resp.get('NextTOken') - if not next_token: - break + next_token = resp.get('NextTOken') + if not next_token: + break + except BotoCoreError as e: + _warn(f"[ERROR - WATOOLS]: {str(e)}") + self.HASPERMISSION = False + return None i = 1 j = 1 @@ -191,6 +208,9 @@ def listAnswers(self): self.answerSets = answerSets def updateAnswers(self, questionId, selectedChoices, unselectedNotes): + if self.HASPERMISSION == False: + return None + ansArgs = { 'WorkloadId': self.waInfo['WorkloadId'], 'LensAlias': self.waInfo['LensesAlias'], @@ -199,7 +219,13 @@ def updateAnswers(self, questionId, selectedChoices, unselectedNotes): 'Notes': unselectedNotes } - resp = self.waClient.update_answer(**ansArgs) + try: + resp = self.waClient.update_answer(**ansArgs) + except BotoCoreError as e: + _warn(f"[ERROR - WATOOLS]: {str(e)}") + self.HASPERMISSION = False + return None + pass ''' stsInfo = {'UserId': 'AIDA55JZ3XKTBZPEJU5K7', 'Account': '956288449190', 'Arn': 'arn:aws:iam::956288449190:user/macbook-ss'} From 2ac4efc6dd2e383c31f86bff3247b512627b4bb6 Mon Sep 17 00:00:00 2001 From: KuetTai Date: Wed, 23 Oct 2024 17:12:37 +0800 Subject: [PATCH 07/15] Disable argument abbrev --- Screener.py | 2 +- main.py | 7 +------ utils/ArguParser.py | 10 ++++++---- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/Screener.py b/Screener.py index b006301..339e774 100644 --- a/Screener.py +++ b/Screener.py @@ -170,7 +170,7 @@ def getServicePagebuilderDynamically(service): @staticmethod - def generateScreenerOutput(runmode, contexts, hasGlobal, regions, uploadToS3, bucket): + def generateScreenerOutput(runmode, contexts, hasGlobal, regions, uploadToS3): htmlFolder = Config.get('HTML_ACCOUNT_FOLDER_FULLPATH') if not os.path.exists(htmlFolder): os.makedirs(htmlFolder) diff --git a/main.py b/main.py index 266b87f..a17b28b 100644 --- a/main.py +++ b/main.py @@ -31,7 +31,6 @@ def number_format(num, places=2): # feedbackFlag = _cli_options['feedback'] # testmode = _cli_options['dev'] testmode = _cli_options['ztestmode'] -bucket = _cli_options['bucket'] runmode = _cli_options['mode'] filters = _cli_options['tags'] crossAccounts = _cli_options['crossAccounts'] @@ -45,10 +44,6 @@ def number_format(num, places=2): runmode = runmode if runmode in ['api-raw', 'api-full', 'report'] else 'report' -# , yet to convert to python -# S3 upload specific variables -# uploadToS3 = Uploader.getConfirmationToUploadToS3(bucket) - # analyse the impact profile switching _AWS_OPTIONS = { 'signature_version': Config.AWS_SDK['signature_version'] @@ -309,7 +304,7 @@ def number_format(num, places=2): Config.set('cli_regions', regions) Config.set('cli_frameworks', frameworks) - Screener.generateScreenerOutput(runmode, contexts, hasGlobal, regions, uploadToS3, bucket) + Screener.generateScreenerOutput(runmode, contexts, hasGlobal, regions, uploadToS3) # os.chdir(_C.FORK_DIR) filetodel = _C.FORK_DIR + '/tail.txt' diff --git a/utils/ArguParser.py b/utils/ArguParser.py index 5bdd53a..0d438d6 100644 --- a/utils/ArguParser.py +++ b/utils/ArguParser.py @@ -58,10 +58,6 @@ class ArguParser: "required": False, "default": False }, - "bucket": { - "required": False, - "default": False - }, "tags": { "required": False, "default": False @@ -84,6 +80,11 @@ class ArguParser: "required": False, "default": 4, "help": "Number of parallel threads, recommend 4 for Cloudshell" + }, + 'beta': { + "required": False, + "default": False, + "help": "Enable Beta features" } } @@ -94,6 +95,7 @@ def Load(): for k, v in ArguParser.CLI_ARGUMENT_RULES.items(): parser.add_argument('-' + k[:1], '--' + k, required=v['required'], default=v['default'], help=v.get('help', None)) + parser.allow_abbrev = False args = vars(parser.parse_args()) return args From f9c4ef12a8e1e436e91327a75be0aca6de7c6a57 Mon Sep 17 00:00:00 2001 From: KuetTai Date: Wed, 23 Oct 2024 17:28:40 +0800 Subject: [PATCH 08/15] Added beta flag support --- main.py | 3 +++ services/Evaluator.py | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index a17b28b..aa1834d 100644 --- a/main.py +++ b/main.py @@ -35,11 +35,13 @@ def number_format(num, places=2): filters = _cli_options['tags'] crossAccounts = _cli_options['crossAccounts'] workerCounts = _cli_options['workerCounts'] +beta = _cli_options['beta'] # print(crossAccounts) DEBUG = True if debugFlag in _C.CLI_TRUE_KEYWORD_ARRAY or debugFlag is True else False testmode = True if testmode in _C.CLI_TRUE_KEYWORD_ARRAY or testmode is True else False crossAccounts = True if crossAccounts in _C.CLI_TRUE_KEYWORD_ARRAY or crossAccounts is True else False +beta = True if beta in _C.CLI_TRUE_KEYWORD_ARRAY or beta is True else False _cli_options['crossAccounts'] = crossAccounts runmode = runmode if runmode in ['api-raw', 'api-full', 'report'] else 'report' @@ -52,6 +54,7 @@ def number_format(num, places=2): Config.init() Config.set('_AWS_OPTIONS', _AWS_OPTIONS) Config.set('DEBUG', DEBUG) +Config.set('beta', beta) _AWS_OPTIONS = { 'signature_version': Config.AWS_SDK['signature_version'] diff --git a/services/Evaluator.py b/services/Evaluator.py index a1cf030..0278312 100644 --- a/services/Evaluator.py +++ b/services/Evaluator.py @@ -78,8 +78,10 @@ def run(self, serviceName): filteredMethods = [method for method in methods if not rules or method[6:].lower() in rules] cnt = len(filteredMethods) - runUsingConcurrent = 1 - if runUsingConcurrent: + + isBeta = Config.get('beta', False) + if isBeta: + _info('[Beta Features] Concurrent Thread on _checks...', True) with cf.ThreadPoolExecutor() as executor: futures = [executor.submit(runSingleCheck, self, method) for method in filteredMethods] From 9bfca331aaeb4163bdef096f36cb4b535eb60fad Mon Sep 17 00:00:00 2001 From: KuetTai Date: Wed, 23 Oct 2024 17:47:00 +0800 Subject: [PATCH 09/15] WAFS to support Beta Flag --- frameworks/WAFS/WAFS.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/frameworks/WAFS/WAFS.py b/frameworks/WAFS/WAFS.py index 06a7f16..2106376 100644 --- a/frameworks/WAFS/WAFS.py +++ b/frameworks/WAFS/WAFS.py @@ -2,15 +2,22 @@ import constants as _C from utils.Config import Config +from utils.Tools import _warn, _info from frameworks.Framework import Framework from frameworks.helper.WATools import WATools class WAFS(Framework): WATools = None ResultCache = {} + isBeta = False def __init__(self, data): super().__init__(data) + self.isBeta = Config.get('beta', False) + if self.isBeta == False: + return + + _info('[Beta Features] WA Tools integration...', True) waTools = WATools('security') cliParams = Config.get('_SS_PARAMS') From 4099d8dc783f1e85a205307076876185b3b83675 Mon Sep 17 00:00:00 2001 From: KuetTai Date: Wed, 23 Oct 2024 18:26:34 +0800 Subject: [PATCH 10/15] Beta supported display --- frameworks/WAFS/WAFS.py | 1 - main.py | 18 +++- services/Evaluator.py | 1 - services/PageBuilder.py | 97 ++++++++++++++++++- .../Pages/Findings/FindingsPageBuilder.py | 4 + 5 files changed, 112 insertions(+), 9 deletions(-) diff --git a/frameworks/WAFS/WAFS.py b/frameworks/WAFS/WAFS.py index 2106376..ac29045 100644 --- a/frameworks/WAFS/WAFS.py +++ b/frameworks/WAFS/WAFS.py @@ -17,7 +17,6 @@ def __init__(self, data): if self.isBeta == False: return - _info('[Beta Features] WA Tools integration...', True) waTools = WATools('security') cliParams = Config.get('_SS_PARAMS') diff --git a/main.py b/main.py index aa1834d..f6f940c 100644 --- a/main.py +++ b/main.py @@ -197,8 +197,8 @@ def number_format(num, places=2): cfnAdditionalStr = None if mpeid is not None: cfnAdditionalStr = " --mpeid:{}".format(mpeid) - CfnTrailObj.boto3init(cfnAdditionalStr) - CfnTrailObj.createStack() + # CfnTrailObj.boto3init(cfnAdditionalStr) + # CfnTrailObj.createStack() overallTimeStart = time.time() # os.chdir('__fork') @@ -231,7 +231,8 @@ def number_format(num, places=2): pool.close() if testmode == False: - CfnTrailObj.deleteStack() + ppp = 1 + # CfnTrailObj.deleteStack() ## ## parallel logic to be implement in Python @@ -343,4 +344,13 @@ def number_format(num, places=2): print("CloudShell user, you may use this path: \033[1;42m =====> \033[0m /tmp/service-screener-v2/output.zip \033[1;42m <===== \033[0m") scriptTimeSpent = round(time.time() - scriptStartTime, 3) -print("@ Thank you for using {}, script spent {}s to complete @".format(Config.ADVISOR['TITLE'], scriptTimeSpent)) \ No newline at end of file +print("@ Thank you for using {}, script spent {}s to complete @".format(Config.ADVISOR['TITLE'], scriptTimeSpent)) + +if beta: + print("") + print("\033[93m[-- ..... --] BETA MODE ENABLED [-- ..... --] \033[0m") + print("Current Beta Features:") + print("\033[96m 01/ Concurrent Mode on Evaluator \033[0m") + print("\033[96m 02/ WA Frameworks Integration \033[0m") + print("\033[96m 03/ GenAI Api Caller Button \033[0m") + print("\033[93m[-- ..... --] THANK YOU FOR TESTING BETA FEATURES [-- ..... --] \033[0m") \ No newline at end of file diff --git a/services/Evaluator.py b/services/Evaluator.py index 0278312..dfbd0ec 100644 --- a/services/Evaluator.py +++ b/services/Evaluator.py @@ -81,7 +81,6 @@ def run(self, serviceName): isBeta = Config.get('beta', False) if isBeta: - _info('[Beta Features] Concurrent Thread on _checks...', True) with cf.ThreadPoolExecutor() as executor: futures = [executor.submit(runSingleCheck, self, method) for method in filteredMethods] diff --git a/services/PageBuilder.py b/services/PageBuilder.py index fa32e3e..83c76ba 100644 --- a/services/PageBuilder.py +++ b/services/PageBuilder.py @@ -41,6 +41,7 @@ class PageBuilder: } isHome = False + isBeta = False colorCustomHex = None colorCustomRGB = None @@ -108,11 +109,10 @@ def buildContentDetail(self): else: cls = self.__class__.__name__ print("[{}] Template for ContentDetail not found: {}".format(cls, method)) - + def generateRowWithCol(self, size=12, items=[], rowHtmlAttr=''): output = [] output.append("
".format(rowHtmlAttr)) - _size = size for ind, item in enumerate(items): if isinstance(size, list): @@ -142,9 +142,13 @@ def generateCard(self, pid, html, cardClass='warning', title='', titleBadge='', defaultCollapseIcon = "plus" if collapse == 9 else "minus" output.append("
".format(pid, lteCardClass, defaultCollapseClass)) + + genAiButton = '' + if self.isBeta: + genAiButton = ' ' if title: - output.append("

{}

".format(title)) + output.append("

{}{}

".format(genAiButton, title)) if collapse: output.append("
".format(defaultCollapseIcon)) @@ -645,9 +649,96 @@ def _buildIndividualKpiCard(self, stat, cat): return s + def genaiModalHtml(self): + genAIJS = """$('.beta-genai').click(function(){ + t = $(this) + currentInfo = {'activeAcct': activeAcct, 'service': serv,'title': t.parent().text().trim(),'resources': {}, 'href': []} + t.parent().parent().parent().find('.card-body dd').each(function(index, el){ + _t = $(this) + cls = _t.attr('class') + tmpText = _t.text() + + if(cls == 'detail-desc'){ + currentInfo['desc'] = tmpText.trim() + } + + if(cls == 'detail-regions'){ + let colonIndex = tmpText.indexOf(':') + region = tmpText.substring(0, colonIndex) + resources = tmpText.substring(colonIndex+2) //include : and space + + currentInfo['resources'][region] = resources.split(' | ') + } + + if(cls == 'detail-href'){ + _t.find('a').each(function(){ + __t = $(this) + currentInfo['href'].push(__t.attr('href')) + }) + } + }) +}) + +genaiResp = $('#genai-modal-response') +$('#genai-savequery').click(function(){ + genaikeys = $('#genai-key').val().split('|') + + if((genaikeys.length < 2) || (genaikeys.length > 2)){ + alert('invalid keys') + return + } + + a_url = genaikeys[0] + a_key = genaikeys[1] + + $.ajax({ + url: a_url, + headers: {'x-api-key': a_key}, + type: 'POST', + data: JSON.stringify(currentInfo), + contentType: 'application/json', + success: function(response) { + genaiResp.text(response['createdAt']) + }, + error: function(xhr, status, error) { + genaiResp.text("Error..., check console.log") + console.error('Error:', error); + } + }); +})""" + + self.addJS(genAIJS) + + return '''''' + def buildContentSummary_default(self): output = [] + self.isBeta = Config.get('beta', False) + if self.isBeta == True: + output.append(self.genaiModalHtml()) + ## KPI Building, 2023-10-16 items = [] kpiCards = self.buildKpiCard() diff --git a/utils/CustomPage/Pages/Findings/FindingsPageBuilder.py b/utils/CustomPage/Pages/Findings/FindingsPageBuilder.py index 0592a95..b3c6843 100644 --- a/utils/CustomPage/Pages/Findings/FindingsPageBuilder.py +++ b/utils/CustomPage/Pages/Findings/FindingsPageBuilder.py @@ -61,6 +61,10 @@ def genTableHTML(self): "", "" ] + + if not columnTitles: + return '' + for title in columnTitles: tableHTMLList.append("") tableHTMLList.append("") From fb732f2c547c5a03735f43de38e08576551aa0c9 Mon Sep 17 00:00:00 2001 From: KuetTai Date: Thu, 24 Oct 2024 15:25:03 +0800 Subject: [PATCH 11/15] Improve visual on scan --- main.py | 20 +++++++------- services/Evaluator.py | 4 +-- services/PageBuilder.py | 9 ++++--- services/Service.py | 4 +-- services/apigateway/Apigateway.py | 6 +++-- services/cloudfront/Cloudfront.py | 4 ++- services/cloudtrail/Cloudtrail.py | 8 +++--- .../cloudtrail/drivers/CloudtrailCommon.py | 1 - services/cloudwatch/Cloudwatch.py | 18 +++---------- .../cloudwatch/drivers/CloudwatchTrails.py | 6 ++--- services/dynamodb/Dynamodb.py | 6 +++-- services/ec2/Ec2.py | 26 ++++++++++--------- services/efs/Efs.py | 4 ++- services/eks/Eks.py | 4 ++- services/elasticache/Elasticache.py | 5 ++-- services/guardduty/Guardduty.py | 4 ++- services/iam/Iam.py | 10 ++++--- services/kms/Kms.py | 4 ++- services/lambda_/Lambda.py | 4 ++- services/opensearch/Opensearch.py | 4 ++- services/rds/Rds.py | 8 +++--- services/redshift/Redshift.py | 4 ++- services/s3/S3.py | 7 +++-- utils/Tools.py | 6 +++++ 24 files changed, 102 insertions(+), 74 deletions(-) diff --git a/main.py b/main.py index f6f940c..473f2b9 100644 --- a/main.py +++ b/main.py @@ -215,17 +215,19 @@ def number_format(num, places=2): with open(directory + '/tail.txt', 'w') as fp: pass - input_ranges = [] + special_services = {'iam', 's3'} + input_ranges = {} - ## Force IAM to run first, it takes the longest time - if 'IAM' in services: - input_ranges = [('IAM', regions, filters)] + ## Make IAM and S3 to be separate pool + if 'iam' in services: + input_ranges['iam'] = ('iam', regions, filters) + + input_ranges.update({service: (service, regions, filters) for service in services if service not in special_services}) + input_ranges = list(input_ranges.values()) + + if 's3' in services: + input_ranges['s3'] = ('s3', regions, filters) - for service in services: - otherInputs = [(service, regions, filters) for service in services] - - input_ranges.extend(otherInputs) - pool = Pool(processes=int(workerCounts)) pool.starmap(Screener.scanByService, input_ranges) pool.close() diff --git a/services/Evaluator.py b/services/Evaluator.py index dfbd0ec..82fb18b 100644 --- a/services/Evaluator.py +++ b/services/Evaluator.py @@ -16,12 +16,10 @@ def runSingleCheck(tmp_obj, method_name): obj = tmp_obj try: startTime = time.time() - if debugFlag: - print('--- --- fn: ' + method_name) - getattr(obj, method_name)() if debugFlag: timeSpent = round(time.time() - startTime, 3) + print('--- --- fn: ' + method_name) if timeSpent >= 0.2: _warn("Long running checks {}s".format(timeSpent)) diff --git a/services/PageBuilder.py b/services/PageBuilder.py index 83c76ba..0cbcd5e 100644 --- a/services/PageBuilder.py +++ b/services/PageBuilder.py @@ -144,7 +144,7 @@ def generateCard(self, pid, html, cardClass='warning', title='', titleBadge='', output.append("
".format(pid, lteCardClass, defaultCollapseClass)) genAiButton = '' - if self.isBeta: + if self.isBeta and pid[:8]=="SUMMARY_": genAiButton = ' ' if title: @@ -650,7 +650,10 @@ def _buildIndividualKpiCard(self, stat, cat): return s def genaiModalHtml(self): - genAIJS = """$('.beta-genai').click(function(){ + genAIJS = """serv = $('h1').text() +activeAcct = $('#changeAcctId').val() + +$('.beta-genai').click(function(){ t = $(this) currentInfo = {'activeAcct': activeAcct, 'service': serv,'title': t.parent().text().trim(),'resources': {}, 'href': []} t.parent().parent().parent().find('.card-body dd').each(function(index, el){ @@ -781,7 +784,7 @@ def buildContentSummary_default(self): body = self.generateSummaryCardContent(attrs) badge = self.generatePriorityPrefix(attrs['criticality'], "style='float:right'") + ' ' + self.generateCategoryBadge(attrs['__categoryMain'], "style='float:right'") - card = self.generateCard(pid=self.getHtmlId(label), html=body, cardClass='', title=label, titleBadge=badge, collapse=9, noPadding=False) + card = self.generateCard(pid="SUMMARY_"+self.getHtmlId(label), html=body, cardClass='', title=label, titleBadge=badge, collapse=9, noPadding=False) divHtmlAttr = "data-category='" + attrs['__categoryMain'] + "' data-criticality='" + attrs['criticality'] + "'" if self.checkIsLowHangingFruit(attrs): diff --git a/services/Service.py b/services/Service.py index 3f866d4..a97afd6 100644 --- a/services/Service.py +++ b/services/Service.py @@ -31,7 +31,7 @@ def __init__(self, region): if self.ssBoto == None: print('BOTO3 SESSION IS MISSING') - print('PREPARING -- ' + classname.upper()+ '::'+region) + print('\x1b[1;37;43mPREPARING\x1b[0m -- \x1b[1;31;43m' + classname.upper()+ '::'+region + '\x1b[0m') def setRules(self, rules): ## Class method is case insensitive, lower to improve accessibilities @@ -40,7 +40,7 @@ def setRules(self, rules): def __del__(self): timespent = round(time.time() - self.overallTimeStart, 3) - print('\033[1;42mCOMPLETED\033[0m -- ' + self.__class__.__name__.upper() + '::'+self.region+' (' + str(timespent) + 's)') + print('\033[1;42mCOMPLETED\033[0m -- \x1b[4;30;47m' + self.__class__.__name__.upper() + '::'+self.region+'\x1b[0m (' + str(timespent) + 's)') items = Config.retrieveAllCache() key = [k for k in items.keys() if 'AllScannedResources' in k] diff --git a/services/apigateway/Apigateway.py b/services/apigateway/Apigateway.py index 64fdc0d..5c3c0fb 100644 --- a/services/apigateway/Apigateway.py +++ b/services/apigateway/Apigateway.py @@ -7,6 +7,8 @@ from services.apigateway.drivers.ApiGatewayCommon import ApiGatewayCommon from services.apigateway.drivers.ApiGatewayRest import ApiGatewayRest +from utils.Tools import _pi + class Apigateway(Service): @@ -54,7 +56,7 @@ def advise(self): self.getApis() for api in self.apisv2: objName = api['ProtocolType'] + '::' + api['Name'] - print('... (APIGateway) inspecting ' + objName) + _pi('APIGateway', objName) obj = ApiGatewayCommon(api, self.apiv2Client) obj.run(self.__class__) objs[objName] = obj.getInfo() @@ -63,7 +65,7 @@ def advise(self): self.getRestApis() for api in self.apis: objName = 'REST' + '::' + api['name'] - print('... (APIGateway) inspecting ' + objName) + _pi('APIGateway', objName) obj = ApiGatewayRest(api, self.apiClient) obj.run(self.__class__) objs[objName] = obj.getInfo() diff --git a/services/cloudfront/Cloudfront.py b/services/cloudfront/Cloudfront.py index d97195a..ecdacf1 100644 --- a/services/cloudfront/Cloudfront.py +++ b/services/cloudfront/Cloudfront.py @@ -8,6 +8,8 @@ from services.Service import Service from services.cloudfront.drivers.cloudfrontDist import cloudfrontDist +from utils.Tools import _pi + class Cloudfront(Service): def __init__(self, region): @@ -48,7 +50,7 @@ def advise(self): dists = self.getDistributions() for dist in dists: - print('... (CloudFront::Distribution) inspecting ' + dist) + _pi('CloudFront::Distribution', dist) obj = cloudfrontDist(dist, self.cloudfrontClient) obj.run(self.__class__) diff --git a/services/cloudtrail/Cloudtrail.py b/services/cloudtrail/Cloudtrail.py index a48b874..8ead331 100644 --- a/services/cloudtrail/Cloudtrail.py +++ b/services/cloudtrail/Cloudtrail.py @@ -9,6 +9,8 @@ from services.cloudtrail.drivers.CloudtrailCommon import CloudtrailCommon from services.cloudtrail.drivers.CloudtrailAccount import CloudtrailAccount +from utils.Tools import _pi + class Cloudtrail(Service): def __init__(self, region): super().__init__(region) @@ -65,10 +67,10 @@ def advise(self): for trail in trails: if trail['TrailARN'] in ctRanList: - print('... [Cloudtrail::SKIPPED] ' + trail['Name'] + ', executed in other regions') + print('[Cloudtrail::SKIPPED] {} executed in other regions'.format(trail['Name'])) continue - print("... [Cloudtrail] inspecting " + trail['Name']) + _pi('Cloudtrail', trail['Name']) ctRanList.append(trail['TrailARN']) obj = CloudtrailCommon(trail, self.ctClient, self.snsClient, self.s3Client) @@ -78,7 +80,7 @@ def advise(self): Config.set('CloudTrail_ranList', ctRanList) - print('... (CloudTrail:Common) inspecting') + _pi('CloudTrail:Common') obj = CloudtrailAccount(self.ctClient, len(trails)) objs['Cloudtrail::General'] = obj.getInfo() del obj diff --git a/services/cloudtrail/drivers/CloudtrailCommon.py b/services/cloudtrail/drivers/CloudtrailCommon.py index 15fdba8..0063f8f 100644 --- a/services/cloudtrail/drivers/CloudtrailCommon.py +++ b/services/cloudtrail/drivers/CloudtrailCommon.py @@ -94,7 +94,6 @@ def _checkS3BucketSettings(self): ## For safety purpose, though all trails must have bucket if 'S3BucketName' in self.trailInfo and len(self.trailInfo['S3BucketName']) > 0: s3Bucket = self.trailInfo['S3BucketName'] - print(s3Bucket) # help me retrieve s3 bucket public try: resp = self.s3Client.get_public_access_block( diff --git a/services/cloudwatch/Cloudwatch.py b/services/cloudwatch/Cloudwatch.py index 4b414db..b207871 100644 --- a/services/cloudwatch/Cloudwatch.py +++ b/services/cloudwatch/Cloudwatch.py @@ -11,6 +11,7 @@ from services.cloudwatch.drivers.CloudwatchCommon import CloudwatchCommon from services.cloudwatch.drivers.CloudwatchTrails import CloudwatchTrails +from utils.Tools import _pi ###### TO DO ##### ## Replace ServiceName with @@ -75,7 +76,7 @@ def advise(self): self.loopTrail() for log in self.ctLogs: - print("... (Cloudwatch Logs) inspecting CloudTrail's related LogGroup [{}]".format(log[0])) + _pi("CloudTrail's CloudWatch Logs", log[0]) obj = CloudwatchTrails(log, log[2], self.cwLogClient) obj.run(self.__class__) @@ -84,24 +85,11 @@ def advise(self): self.getAllLogs() for log in self.logGroups: - print("... (Cloudwatch Logs inspecting LogGroup [{}]".format(log['logGroupName'])) + _pi('Cloudwatch Logs', log['logGroupName']) obj = CloudwatchCommon(log, self.cwLogClient) obj.run(self.__class__) objs[f"Log::{log['logGroupName']}"] = obj.getInfo() del obj - ###### TO DO ##### - ## call getResources method - ## loop through the resources and run the checks in drivers - ## Example - # instances = self.getResources() - # for instance in instances: - # instanceData = instance['Instances'][0] - # print('... (EC2) inspecting ' + instanceData['InstanceId']) - # obj = Ec2Instance(instanceData,self.ec2Client, self.cwClient) - # obj.run(self.__class__) - # objs[f"EC2::{instanceData['InstanceId']}"] = obj.getInfo() - #. del obj - return objs \ No newline at end of file diff --git a/services/cloudwatch/drivers/CloudwatchTrails.py b/services/cloudwatch/drivers/CloudwatchTrails.py index 8d71969..6cfbb05 100644 --- a/services/cloudwatch/drivers/CloudwatchTrails.py +++ b/services/cloudwatch/drivers/CloudwatchTrails.py @@ -24,8 +24,8 @@ class CloudwatchTrails(Evaluator): ] }, {'trailWOMAunauthAPI2': [ - ["$.errorCode", "=", "\*UnauthorizedOperation"], - ["$.errorCode", "=", "AccessDenied\*"] + ["$.errorCode", "=", r"\*UnauthorizedOperation"], + ["$.errorCode", "=", r"AccessDenied\*"] ] }, {'trailWOMAnoMFA3': [ @@ -175,7 +175,7 @@ def __init__(self, log, logname, logClient): def regexBuilder(self, rules): regexPatterns = [] for rule in rules: - regexPattern = "\\" + rule[0] + "\s*\\" + rule[1] + "\s*[\\'\\\"]*" + rule[2] + "[\\'\\\"]*" + regexPattern = r"\\" + rule[0] + r"\s*\\" + rule[1] + r"\s*[\'\"]*" + rule[2] + r"[\'\"]*" regexPatterns.append(regexPattern) return regexPatterns diff --git a/services/dynamodb/Dynamodb.py b/services/dynamodb/Dynamodb.py index 095605b..1737965 100644 --- a/services/dynamodb/Dynamodb.py +++ b/services/dynamodb/Dynamodb.py @@ -8,6 +8,8 @@ from services.dynamodb.drivers.DynamoDbCommon import DynamoDbCommon from services.dynamodb.drivers.DynamoDbGeneric import DynamoDbGeneric +from utils.Tools import _pi + class Dynamodb(Service): @@ -66,7 +68,7 @@ def advise(self): try: #Run generic checks - print('... (Dynamodb::Generic) inspecting') + _pi('Dynamodb::Generic') obj = DynamoDbGeneric(listOfTables, self.dynamoDbClient, self.cloudWatchClient, self.serviceQuotaClient, self.appScalingPolicyClient, self.backupClient, self.cloudTrailClient) obj.run(self.__class__) objs['DynamoDb::Generic'] = obj.getInfo() @@ -75,7 +77,7 @@ def advise(self): #Run table specific checks for eachTable in listOfTables: objName = 'Dynamodb::' + eachTable['Table']['TableName'] - print('... ({}) inspecting'.format(objName)) + _pi('Dynamodb::Table', objName) obj = DynamoDbCommon(eachTable, self.dynamoDbClient, self.cloudWatchClient, self.serviceQuotaClient, self.appScalingPolicyClient, self.backupClient, self.cloudTrailClient) obj.run(self.__class__) objs[objName] = obj.getInfo() diff --git a/services/ec2/Ec2.py b/services/ec2/Ec2.py index 570d3c6..11dc466 100644 --- a/services/ec2/Ec2.py +++ b/services/ec2/Ec2.py @@ -8,6 +8,8 @@ import json import time +from utils.Tools import _pi + from utils.Config import Config from services.Service import Service from services.ec2.drivers.Ec2Instance import Ec2Instance @@ -381,7 +383,7 @@ def advise(self): ) if 'Parameters' in compOptCheck and len(compOptCheck['Parameters']) > 0: - print('... (Compute Optimizer Recommendations) inspecting') + _pi('Compute Optimizer Recommendations') obj = Ec2CompOpt(self.compOptClient) obj.run(self.__class__) objs['ComputeOptimizer'] = obj.getInfo() @@ -399,7 +401,7 @@ def advise(self): #EC2 Cost Explorer checks hasRunRISP = Config.get('EC2_HasRunRISP', False) if hasRunRISP == False: - print('... (Cost Explorer Recommendations) inspecting') + _pi('Cost Explorer Recommendations') obj = Ec2CostExplorerRecs(self.ceClient) obj.run(self.__class__) @@ -410,7 +412,7 @@ def advise(self): instances = self.getResources() for instanceArr in instances: for instanceData in instanceArr['Instances']: - print('... (EC2) inspecting ' + instanceData['InstanceId']) + _pi('EC2', instanceData['InstanceId']) obj = Ec2Instance(instanceData,self.ec2Client, self.cwClient) obj.run(self.__class__) @@ -424,13 +426,13 @@ def advise(self): #EBS checks volumes = self.getEBSResources() for volume in volumes: - print('... (EBS) inspecting ' + volume['VolumeId']) + _pi('EBS', volume['VolumeId']) obj = Ec2EbsVolume(volume,self.ec2Client, self.cwClient) obj.run(self.__class__) objs[f"EBS::{volume['VolumeId']}"] = obj.getInfo() #EBS Snapshots - print('... (EBS::Snapshots) inspecting') + _pi('EBS::Snapshots') obj = Ec2EbsSnapshot(self.ec2Client) obj.run(self.__class__) objs["EBS::Snapshots"] = obj.getInfo() @@ -443,7 +445,7 @@ def advise(self): for group in elbSGList: secGroups[group['GroupId']] = group - print(f"... (ELB::Load Balancer) inspecting {lb['LoadBalancerName']}") + _pi('ELB::Load Balancer', lb['LoadBalancerName']) obj = Ec2ElbCommon(lb, elbSGList, self.elbClient, self.wafv2Client) obj.run(self.__class__) objs[f"ELB::{lb['LoadBalancerName']}"] = obj.getInfo() @@ -452,7 +454,7 @@ def advise(self): # ELB classic checks lbClassic = self.getELBClassic() for lb in lbClassic: - print(f"... (ELB::Load Balancer Classic) inspecting {lb['LoadBalancerName']}") + _pi('ELB::Load Balancer Classic', lb['LoadBalancerName']) obj = Ec2ElbClassic(lb, self.elbClassicClient) obj.run(self.__class__) objs[f"ELB Classic::{lb['LoadBalancerName']}"] = obj.getInfo() @@ -464,7 +466,7 @@ def advise(self): # ASG checks autoScalingGroups = self.getASGResources() for group in autoScalingGroups: - print(f"... (ASG::Auto Scaling Group) inspecting {group['AutoScalingGroupName']}"); + _pi('ASG::Auto Scaling Group', group['AutoScalingGroupName']); obj = Ec2AutoScaling(group, self.asgClient, self.elbClient, self.elbClassicClient, self.ec2Client) obj.run(self.__class__) objs[f"ASG::{group['AutoScalingGroupName']}"] = obj.getInfo() @@ -479,7 +481,7 @@ def advise(self): # SG checks if secGroups: for group in secGroups.values(): - print(f"... (EC2::Security Group) inspecting {group['GroupId']}") + _pi('EC2::Security Group', group['GroupId']) obj = Ec2SecGroup(group, self.ec2Client) obj.run(self.__class__) @@ -488,7 +490,7 @@ def advise(self): # EIP checks eips = self.getEIPResources() for eip in eips: - print('... (Elastic IP Recommendations) inspecting {}'.format(eip['PublicIp'])) + _pi('Elastic IP Recommendations', eip['PublicIp']) obj = Ec2EIP(eip) obj.run(self.__class__) objs[f"ElasticIP::{eip['AllocationId']}"] = obj.getInfo() @@ -497,7 +499,7 @@ def advise(self): vpcs = self.getVpcs() flowLogs = self.getFlowLogs() for vpc in vpcs: - print(f"... (VPC::Virtual Private Cloud) inspecting {vpc['VpcId']}") + _pi('VPC::Virtual Private Cloud', vpc['VpcId']) obj = Ec2Vpc(vpc, flowLogs, self.ec2Client) obj.run(self.__class__) objs[f"VPC::{vpc['VpcId']}"] = obj.getInfo() @@ -505,7 +507,7 @@ def advise(self): # NACL Checks nacls = self.getNetworkACLs() for nacl in nacls: - print(f"... (NACL::Network ACL) inspecting {nacl['NetworkAclId']}") + _pi('NACL::Network ACL', nacl['NetworkAclId']) obj = Ec2NACL(nacl, self.ec2Client) obj.run(self.__class__) objs[f"NACL::{nacl['NetworkAclId']}"] = obj.getInfo() diff --git a/services/efs/Efs.py b/services/efs/Efs.py index 185c95c..c69e68b 100644 --- a/services/efs/Efs.py +++ b/services/efs/Efs.py @@ -7,6 +7,8 @@ from services.efs.drivers.EfsDriver import EfsDriver +from utils.Tools import _pi + class Efs(Service): def __init__(self, region): super().__init__(region) @@ -35,7 +37,7 @@ def advise(self): driver = 'EfsDriver' if globals().get(driver): for efs in efs_list: - print('... (EFS) inspecting ' + efs['FileSystemId']) + _pi('EFS', efs['FileSystemId']) obj = globals()[driver](efs, self.efs_client) obj.run(self.__class__) diff --git a/services/eks/Eks.py b/services/eks/Eks.py index e3c8fa0..4077f15 100644 --- a/services/eks/Eks.py +++ b/services/eks/Eks.py @@ -8,6 +8,8 @@ from services.Service import Service from services.eks.drivers.EksCommon import EksCommon +from utils.Tools import _pi + class Eks(Service): def __init__(self, region): super().__init__(region) @@ -42,7 +44,7 @@ def advise(self): clusters = self.getClusters() for cluster in clusters: - print('...(EKS:Cluster) inspecting ' + cluster) + _pi('EKS:Cluster', cluster) clusterInfo = self.describeCluster(cluster) #if clusterInfo.get('status') == 'CREATING': diff --git a/services/elasticache/Elasticache.py b/services/elasticache/Elasticache.py index 08183f7..ed24644 100644 --- a/services/elasticache/Elasticache.py +++ b/services/elasticache/Elasticache.py @@ -10,6 +10,7 @@ from services.elasticache.drivers.ElasticacheReplicationGroup import ElasticacheReplicationGroup from typing import Dict, List, Set +from utils.Tools import _pi class Elasticache(Service): def __init__(self, region) -> None: @@ -171,7 +172,7 @@ def advise(self): repGroups = self.getReplicationGroupInfo() for group in repGroups: - print(f"... (ElastiCache::ReplicationGroup) inspecting {group.get('ReplicationGroupId')}") + _pi("ElastiCache::ReplicationGroup", group.get('ReplicationGroupId')) obj = ElasticacheReplicationGroup(group, self.elasticacheClient) obj.run(self.__class__) objs[f"ElastiCache::{group.get('ReplicationGroupId')}"] = obj.getInfo() @@ -197,7 +198,7 @@ def advise(self): if obj is not None: objName = cluster.get('Engine') + f"{cluster.get('ARN')}" - print("... (ElastiCache:" + cluster.get('Engine') + ') ' + f"{cluster.get('ARN')}") + _pi("ElastiCache:" + cluster.get('Engine'), cluster.get('ARN')) obj.run(self.__class__) objs[objName] = obj.getInfo() del obj diff --git a/services/guardduty/Guardduty.py b/services/guardduty/Guardduty.py index 49ba13f..3e211fe 100644 --- a/services/guardduty/Guardduty.py +++ b/services/guardduty/Guardduty.py @@ -5,6 +5,8 @@ from services.Service import Service from services.guardduty.drivers.GuarddutyDriver import GuarddutyDriver +from utils.Tools import _pi + class Guardduty(Service): def __init__(self, region): super().__init__(region) @@ -25,7 +27,7 @@ def advise(self): objs = {} detectors = self.get_resources() for detector in detectors: - print(f"... (GuardDuty) inspecting {detector}") + _pi("GuardDuty", detector) obj = GuarddutyDriver(detector, self.guardduty_client, self.region) obj.run(self.__class__) objs[f"Detector::{detector}"] = obj.getInfo() diff --git a/services/iam/Iam.py b/services/iam/Iam.py index c9c8305..0ae7403 100644 --- a/services/iam/Iam.py +++ b/services/iam/Iam.py @@ -11,6 +11,8 @@ from services.iam.drivers.IamUser import IamUser from services.iam.drivers.IamAccount import IamAccount +from utils.Tools import _pi + class Iam(Service): def __init__(self, region): super().__init__(region) @@ -139,7 +141,7 @@ def advise(self): return objs for user in users: - print('... (IAM::User) inspecting ' + user['user']) + _pi('IAM::User', user['user']) obj = IamUser(user, self.iamClient) obj.run(self.__class__) @@ -149,7 +151,7 @@ def advise(self): roles = self.getRoles() for role in roles: - print('... (IAM::Role) inspecting ' + role['RoleName']) + _pi('IAM::Role', role['RoleName']) obj = IamRole(role, self.iamClient) obj.run(self.__class__) @@ -158,14 +160,14 @@ def advise(self): groups = self.getGroups() for group in groups: - print('... (IAM::Group) inspecting ' + group['GroupName']) + _pi('IAM::Group', group['GroupName']) obj = IamGroup(group, self.iamClient) obj.run(self.__class__) objs['Group::' + group['GroupName']] = obj.getInfo() del obj - print('... (IAM:Account) inspecting') + _pi('IAM:Account') obj = IamAccount(None, self.awsClients, users, roles, self.ssBoto) obj.run(self.__class__) objs['Account::Config'] = obj.getInfo() diff --git a/services/kms/Kms.py b/services/kms/Kms.py index d69a75f..44a80b1 100644 --- a/services/kms/Kms.py +++ b/services/kms/Kms.py @@ -6,6 +6,8 @@ ##import drivers here from services.kms.drivers.KmsCommon import KmsCommon +from utils.Tools import _pi + class Kms(Service): def __init__(self, region): super().__init__(region) @@ -47,7 +49,7 @@ def advise(self): self.getResources() for key in self.kmsCustomerManagedKeys: - print('... (KMS) inspecting ' + key['KeyId'] + ' (' + key['Arn'] +')') + _pi('KMS', key['KeyId'] + ' (' + key['Arn'] +')') obj = KmsCommon(key, self.kmsClient) obj.run(self.__class__) diff --git a/services/lambda_/Lambda.py b/services/lambda_/Lambda.py index 83844e0..cd547f8 100644 --- a/services/lambda_/Lambda.py +++ b/services/lambda_/Lambda.py @@ -7,6 +7,8 @@ from services.Service import Service from utils.Config import Config +from utils.Tools import _pi + class Lambda(Service): def __init__(self, region): super().__init__(region) @@ -70,7 +72,7 @@ def advise(self): try: # module = importlib.import_module(f"drivers.{driver}") # cls = getattr(module, driver) - print(f"... (Lambda) inspecting {lambda_function['FunctionName']}") + _pi('Lambda', lambda_function['FunctionName']) # obj = cls(lambda_function, self.lambda_client, self.iam_client, role_count) obj = LambdaCommon(lambda_function, self.lambda_client, self.iam_client, role_count) obj.run(self.__class__) diff --git a/services/opensearch/Opensearch.py b/services/opensearch/Opensearch.py index f264de7..996e5ba 100644 --- a/services/opensearch/Opensearch.py +++ b/services/opensearch/Opensearch.py @@ -6,6 +6,8 @@ ##import drivers here from services.opensearch.drivers.OpensearchCommon import OpensearchCommon +from utils.Tools import _pi + class Opensearch(Service): def __init__(self, region): super().__init__(region) @@ -46,7 +48,7 @@ def advise(self): for domain in domains: domain_name = domain["DomainName"] - print("... (OpenSearch) inspecting " + domain_name) + _pi("OpenSearch", domain_name) obj = OpensearchCommon(self.bConfig, domain_name, domain['info'], self.osClient, self.cwClient) obj.run(self.__class__) diff --git a/services/rds/Rds.py b/services/rds/Rds.py index 02939ab..b2a527c 100644 --- a/services/rds/Rds.py +++ b/services/rds/Rds.py @@ -15,6 +15,8 @@ from services.rds.drivers.RdsSecretsVsDB import RdsSecretsVsDB from services.rds.drivers.RdsSecurityGroup import RdsSecurityGroup +from utils.Tools import _pi + class Rds(Service): def __init__(self, region): super().__init__(region) @@ -126,7 +128,7 @@ def advise(self): dbInfo = 'Instance' dbKey = 'DBInstanceIdentifier' - print('... (RDS) inspecting {}::{}'.format(dbInfo, instance[dbKey])) + _pi('RDS', '{}::{}'.format(dbInfo, instance[dbKey])) if 'VpcSecurityGroups' in instance: for sg in instance['VpcSecurityGroups']: @@ -157,7 +159,7 @@ def advise(self): del obj for sg, rdsList in securityGroupArr.items(): - print('... (RDS-SG) inspecting ' + sg) + _pi('RDS-SG', sg) obj = RdsSecurityGroup(sg, self.ec2Client, rdsList) obj.run(self.__class__) objs['RDS_SG::' + sg] = obj.getInfo() @@ -165,7 +167,7 @@ def advise(self): self.getSecrets() for secret in self.secrets: - print('... (SecretsManager) inspecting ' + secret['Name']) + _pi('SecretsManager', secret['Name']) obj = RdsSecretsManager(secret, self.smClient, self.ctClient) obj.run(self.__class__) diff --git a/services/redshift/Redshift.py b/services/redshift/Redshift.py index 1388be7..e9d2735 100644 --- a/services/redshift/Redshift.py +++ b/services/redshift/Redshift.py @@ -6,6 +6,8 @@ from services.Service import Service from services.redshift.drivers.RedshiftCluster import RedshiftCluster +from utils.Tools import _pi + class Redshift(Service): def __init__(self, region): super().__init__(region) @@ -48,7 +50,7 @@ def advise(self): self.getClusterResources() for cluster in self.redshifts: - print('... (Redshift) inspecting ' + cluster['ClusterIdentifier']) + _pi('Redshift', cluster['ClusterIdentifier']) obj = RedshiftCluster(cluster, self.rsClient) obj.run(self.__class__) objs[f"Redshift::{cluster['ClusterIdentifier']}"] = obj.getInfo() diff --git a/services/s3/S3.py b/services/s3/S3.py index 6f3acd3..19002ad 100644 --- a/services/s3/S3.py +++ b/services/s3/S3.py @@ -14,6 +14,8 @@ from services.s3.drivers.S3Control import S3Control from services.s3.drivers.S3Macie import S3Macie +from utils.Tools import _pi + class S3(Service): def __init__(self, region): super().__init__(region) @@ -90,7 +92,7 @@ def advise(self): objs = {} accountScanned = Config.get('S3_HasAccountScanned', False) if accountScanned == False: - print('... (S3Account) inspecting ') + _pi('S3Account') obj = S3Control(self.s3Control) obj.run(self.__class__) objs["Account::Control"] = obj.getInfo() @@ -101,13 +103,14 @@ def advise(self): objs = {} buckets = self.getResources() for bucket in buckets: - print('... (S3Bucket) inspecting ' + bucket['Name']) + _pi('S3Bucket', bucket['Name']) obj = S3Bucket(bucket['Name'], self.s3Client) obj.run(self.__class__) objs["Bucket::" + bucket['Name']] = obj.getInfo() del obj + _pi('S3Macie') obj = S3Macie(self.macieV2Client) obj.run(self.__class__) objs["Macie"] = obj.getInfo() diff --git a/utils/Tools.py b/utils/Tools.py index 6daf881..86e02ba 100644 --- a/utils/Tools.py +++ b/utils/Tools.py @@ -7,6 +7,12 @@ from typing import Set, Dict, Union from netaddr import IPAddress +## from utils.Tools import _pi +def _pi(group, res=''): + det = '' + if res: + det = '- ' + print("... \x1b[1;37;44m({})\x1b[0m {}\x1b[1;37;45m{}\x1b[0m".format(group, det, res)) def _pr(s, forcePrint = False): DEBUG = Config.get('DEBUG') From ce774017dae4cf6b1c11de7a28f763d777d6b75b Mon Sep 17 00:00:00 2001 From: KuetTai Date: Thu, 24 Oct 2024 15:36:10 +0800 Subject: [PATCH 12/15] Improve visual on scan --- main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/main.py b/main.py index 473f2b9..9f6b82a 100644 --- a/main.py +++ b/main.py @@ -197,8 +197,8 @@ def number_format(num, places=2): cfnAdditionalStr = None if mpeid is not None: cfnAdditionalStr = " --mpeid:{}".format(mpeid) - # CfnTrailObj.boto3init(cfnAdditionalStr) - # CfnTrailObj.createStack() + CfnTrailObj.boto3init(cfnAdditionalStr) + CfnTrailObj.createStack() overallTimeStart = time.time() # os.chdir('__fork') @@ -223,18 +223,18 @@ def number_format(num, places=2): input_ranges['iam'] = ('iam', regions, filters) input_ranges.update({service: (service, regions, filters) for service in services if service not in special_services}) - input_ranges = list(input_ranges.values()) if 's3' in services: input_ranges['s3'] = ('s3', regions, filters) + input_ranges = list(input_ranges.values()) + pool = Pool(processes=int(workerCounts)) pool.starmap(Screener.scanByService, input_ranges) pool.close() if testmode == False: - ppp = 1 - # CfnTrailObj.deleteStack() + CfnTrailObj.deleteStack() ## ## parallel logic to be implement in Python From 551f4521db046efb85695f8813c1fa84c20b0342 Mon Sep 17 00:00:00 2001 From: KuetTai Date: Thu, 24 Oct 2024 16:02:09 +0800 Subject: [PATCH 13/15] to support 0.0.0.0 range check --- utils/Tools.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/Tools.py b/utils/Tools.py index 86e02ba..02af0fa 100644 --- a/utils/Tools.py +++ b/utils/Tools.py @@ -31,6 +31,9 @@ def _printStatus(status, s, forcePrint = False): def checkIsPrivateIp(ipaddr): ip = ipaddr.split('/') + if ip[0] == '0.0.0.0': + return False + return IPAddress(ip[0]).is_private() def aws_parseInstanceFamily(instanceFamily: str, region=None) -> Dict[str, str]: From d17e8f9b54bb6922b731bf8e0cabd6a761478782 Mon Sep 17 00:00:00 2001 From: KuetTai Date: Thu, 24 Oct 2024 16:02:23 +0800 Subject: [PATCH 14/15] Improve error message captured --- services/Evaluator.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/services/Evaluator.py b/services/Evaluator.py index 82fb18b..a4af512 100644 --- a/services/Evaluator.py +++ b/services/Evaluator.py @@ -29,13 +29,12 @@ def runSingleCheck(tmp_obj, method_name): code = e.response['Error']['Code'] msg = e.response['Error']['Message'] print(code, msg) - print(traceback.format_exc()) - traceback.format_exc() + emsg = traceback.format_exc() except Exception: - print(traceback.format_exc()) - traceback.format_exc() + emsg = traceback.format_exc() - return traceback.format_exc() + print(emsg) + return emsg class Evaluator(): def __init__(self): @@ -83,13 +82,11 @@ def run(self, serviceName): futures = [executor.submit(runSingleCheck, self, method) for method in filteredMethods] for future in cf.as_completed(futures): - for fr in future.result(): - if fr == 'OK': - continue - else: - emsg.append(fr) - ecnt += 1 - + if future.result() == 'OK': + continue + else: + emsg.append(future.result()) + ecnt += 1 else: for method in methods: if not rules or str.lower(method[6:]) in rules: From ba84bb3f54e4e25f8376237b9e63c7f2f7391c5e Mon Sep 17 00:00:00 2001 From: KuetTai Date: Thu, 24 Oct 2024 18:11:47 +0800 Subject: [PATCH 15/15] added new instructions to README, and improve error handling on WA Integration --- README.md | 20 +++++++++- frameworks/WAFS/WAFS.py | 2 +- frameworks/helper/WATools.py | 25 +++++++++++- main.py | 2 +- services/Evaluator.py | 3 +- services/lambda_/drivers/LambdaCommon.py | 48 ++++++++++++------------ 6 files changed, 68 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index d2e7185..a4a8390 100755 --- a/README.md +++ b/README.md @@ -59,9 +59,14 @@ When running Service Screener, you will need to specify the regions and services We recommend running it in all regions where you have deployed workloads in. Adjust the code samples below to suit your needs then copy and paste it into Cloudshell to run Service Screener. -**Example 1: Run in the Singapore region, check all services** +**Example 1: (Recommended) Run in the Singapore region, check all services with beta features enabled** ``` -screener --regions ap-southeast-1 +screener --regions ap-southeast-1 --beta 1 +``` + +**Example 1a: Run in the Singapore region, check all services on stable releases** +``` +screener --regions ap-southeast-1 ``` **Example 2: Run in the Singapore region, check only Amazon S3** @@ -89,6 +94,7 @@ screener --regions ap-southeast-1 --tags env=prod%department=hr,coe screener --regions ALL ``` + ### Other parameters ```bash ##mode @@ -97,6 +103,16 @@ screener --regions ALL # api-full: give full results in JSON format # api-raw: raw findings # report: generate default web html + +##others +# AWS Partner used, migration evaluation id +--others '{"mpe": {"id": "aaaa-1111-cccc"}}' + +# To override default Well Architected Tools integration parameter +--others '{"WA": {"region": "ap-southeast-1", "reportName":"SS_Report", "newMileStone":0}}' + +# you can combine both +--others '{"WA": {"region": "ap-southeast-1", "reportName":"SS_Report", "newMileStone":0}, "mpe": {"id": "aaaa-1111-cccc"}}' ```
Get Report Walkthrough diff --git a/frameworks/WAFS/WAFS.py b/frameworks/WAFS/WAFS.py index ac29045..38e7821 100644 --- a/frameworks/WAFS/WAFS.py +++ b/frameworks/WAFS/WAFS.py @@ -27,7 +27,7 @@ def __init__(self, data): if 'WA' in cfg: tmpParams = cfg['WA'] - + if waTools.preCheck(tmpParams): self.WATools = waTools self.WATools.init(tmpParams) diff --git a/frameworks/helper/WATools.py b/frameworks/helper/WATools.py index 5c8b9c6..6dbedde 100644 --- a/frameworks/helper/WATools.py +++ b/frameworks/helper/WATools.py @@ -1,9 +1,10 @@ -import boto3, json +import boto3, json, botocore from botocore.exceptions import BotoCoreError from botocore.config import Config as bConfig from utils.Config import Config from datetime import datetime from utils.Tools import _warn +import time ## --others '{"WA": {"region": "ap-southeast-1", "reportName":"SS_Report", "newMileStone":0}}' @@ -31,6 +32,8 @@ def preCheck(self, params): if not 'region' in params: params['region'] = Config.get('REGIONS_SELECTED')[0] + + print("*** [WATool] Attempting to deploy WA Tools in this region: {}".format(params['region'])) return True @@ -172,6 +175,26 @@ def listAnswers(self): 'MaxResults': 50 } + isSuccess = False + maxRetry = 3 + currAttempt = 0 + while True: + currAttempt = currAttempt + 1 + try: + resp = self.waClient.list_answers(**ansArgs) + isSuccess = True + break + except botocore.errorfactory.ResourceNotFoundException: + # wait for 3 seconds before retrying + print("*** [WATools] ListAnswer failed, waiting workload to be generated, retry in 3 seconds") + if currAttempt >= maxRetry: + break + time.sleep(3) + + if isSuccess == False: + print("*** [WATools] Unable to retrieve list of checklists, skipped WATool integration") + return None + answers = [] try: while True: diff --git a/main.py b/main.py index 9f6b82a..5dd9ee4 100644 --- a/main.py +++ b/main.py @@ -266,7 +266,7 @@ def number_format(num, places=2): hasGlobal = True if testmode == True: - exit("Test mode enable, script halted") + exit("Test mode enable, script halted") timespent = round(time.time() - overallTimeStart, 3) scanned['timespent'] = timespent diff --git a/services/Evaluator.py b/services/Evaluator.py index a4af512..1bc19e8 100644 --- a/services/Evaluator.py +++ b/services/Evaluator.py @@ -23,7 +23,6 @@ def runSingleCheck(tmp_obj, method_name): if timeSpent >= 0.2: _warn("Long running checks {}s".format(timeSpent)) - getattr(obj, method_name)() return 'OK' except botocore.exceptions.ClientError as e: code = e.response['Error']['Code'] @@ -184,7 +183,7 @@ def __del__(self): if name == None: return - scanned.append(';'.join([Config.get(classPrefix), driver, name, hasError])) + scanned.append(';'.join([Config.get(classPrefix, ""), driver, name, hasError])) Config.set(ConfigKey, scanned) diff --git a/services/lambda_/drivers/LambdaCommon.py b/services/lambda_/drivers/LambdaCommon.py index ab5d449..cd24650 100644 --- a/services/lambda_/drivers/LambdaCommon.py +++ b/services/lambda_/drivers/LambdaCommon.py @@ -77,12 +77,21 @@ def _check_architectures_is_arm(self): self.results['UseArmArchitecture'] = [-1, ', '.join(self.lambda_['Architectures'])] - def _check_function_url_in_used(self): - url_config = self.lambda_client.list_function_url_configs( - FunctionName=self.function_name - ) - if url_config['FunctionUrlConfigs']: - self.results['lambdaURLInUsed'] = [-1, "Enabled"] + def _check_function_url_in_used_and_auth(self): + try: + url_config = self.lambda_client.list_function_url_configs( + FunctionName=self.function_name + ) + if url_config['FunctionUrlConfigs']: + self.results['lambdaURLInUsed'] = [-1, "Enabled"] + + for config in url_config['FunctionUrlConfigs']: + if config['AuthType'] == 'NONE': + self.results['lambdaURLWithoutAuth'] = [-1, config['AuthType']] + return + + except botocore.exceptions.ClientError as e: + print("No permission to access lambda:list_function_url_configs") return def _check_missing_role(self): @@ -100,28 +109,17 @@ def _check_missing_role(self): raise e return - def _check_url_without_auth(self): - url_configs = self.lambda_client.list_function_url_configs( - FunctionName=self.function_name - ) - - if url_configs['FunctionUrlConfigs']: - for config in url_configs['FunctionUrlConfigs']: - if config['AuthType'] == 'NONE': - self.results['lambdaURLWithoutAuth'] = [-1, config['AuthType']] - return - - return - def _check_code_signing_disabled(self): if self.lambda_['PackageType'] != 'Zip': return - - code_sign = self.lambda_client.get_function_code_signing_config( - FunctionName=self.function_name - ) - if not code_sign.get('CodeSigningConfigArn'): - self.results['lambdaCodeSigningDisabled'] = [-1, 'Disabled'] + try: + code_sign = self.lambda_client.get_function_code_signing_config( + FunctionName=self.function_name + ) + if not code_sign.get('CodeSigningConfigArn'): + self.results['lambdaCodeSigningDisabled'] = [-1, 'Disabled'] + except botocore.exceptions.ClientError as e: + print("No permission to access get_function_code_signing_config") return
" + title + "