From 9630e9b0340f3001dd30377e40a5c916a7c026c4 Mon Sep 17 00:00:00 2001 From: Akshay Bhansali Date: Fri, 29 Oct 2021 11:23:18 +0530 Subject: [PATCH 1/3] ignore vulnerabilities for VA --- bayesian/api/api_v2.py | 5 +++- bayesian/utility/v2/component_analyses.py | 28 ++++++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/bayesian/api/api_v2.py b/bayesian/api/api_v2.py index 58856fe3..884a03ef 100644 --- a/bayesian/api/api_v2.py +++ b/bayesian/api/api_v2.py @@ -91,6 +91,7 @@ def vulnerability_analysis_post(): """ input_json: Dict = request.get_json() ecosystem: str = input_json.get('ecosystem') + ignore_vulnerabilities: Dict = input_json.get('ignore', {}) try: # Step1: Gather and clean Request @@ -98,7 +99,7 @@ def vulnerability_analysis_post(): # Step2: Get aggregated CA data from Query GraphDB, graph_response = get_vulnerability_data(ecosystem, packages_list) # Step3: Build Unknown packages and Generates Stack Recommendation. - stack_recommendation = get_known_pkgs(graph_response, packages_list) + stack_recommendation = get_known_pkgs(graph_response, packages_list, ignore_vulnerabilities) except BadRequest as br: logger.error(br) raise HTTPError(400, str(br)) from br @@ -175,7 +176,9 @@ def component_analyses_post(): return jsonify({"message": "disabled"}), 404 try: # Step1: Gather and clean Request + print("Result Coming") packages_list, normalised_input_pkgs = ca_validate_input(input_json, ecosystem) + print(packages_list, normalised_input_pkgs) # Step2: Get aggregated CA data from Query GraphDB, graph_response = get_batch_ca_data(ecosystem, packages_list) # Step3: Build Unknown packages and Generates Stack Recommendation. diff --git a/bayesian/utility/v2/component_analyses.py b/bayesian/utility/v2/component_analyses.py index 7044b197..d596d68a 100644 --- a/bayesian/utility/v2/component_analyses.py +++ b/bayesian/utility/v2/component_analyses.py @@ -78,6 +78,12 @@ def validate_input(input_json: Dict, ecosystem: str) -> List[Dict]: error_msg: str = "package_versions is missing" raise BadRequest(error_msg) + ignore_package_vulns = input_json.get('ignore', {}) + if not isinstance(ignore_package_vulns, dict): + error_msg = "Expected a dictionary 'ignore' consisting of package names as key" \ + ", and list of vulnerabilities to ignore for that package as value" + raise BadRequest(error_msg) + packages_list = [] for pkg in input_json.get('package_versions'): package = pkg.get("package") @@ -171,7 +177,9 @@ def _fetcher_in_batches(func: Callable, packages: List, def get_batch_ca_data(ecosystem: str, packages: List) -> dict: + """Fetch package details for component analyses.""" + print("Ex get_batch_ca_data") logger.debug('Executing get_batch_ca_data') started_at = time.time() @@ -188,16 +196,19 @@ def get_batch_ca_data(ecosystem: str, packages: List) -> dict: semver_packages.append(p) else: semver_packages = packages + print("semver packages", packages) graph_data_fetcher = [] if len(semver_packages) > 0: get_semver_data = functools.partial(GraphAnalyses.get_batch_ca_data, ecosystem) graph_data_fetcher = list(_fetcher_in_batches(get_semver_data, semver_packages)) + print("graph_data_fetcher", graph_data_fetcher) if len(pseudo_version_packages) > 0: get_pseudo_data = functools.partial(GraphAnalyses.get_batch_ca_data_for_pseudo_version, ecosystem) graph_data_fetcher += list(_fetcher_in_batches(get_pseudo_data, pseudo_version_packages)) + print("graph_data_fetcher", graph_data_fetcher) response = { "result": { @@ -210,6 +221,7 @@ def get_batch_ca_data(ecosystem: str, packages: List) -> dict: elapsed_time = time.time() - started_at logger.info("concurrent batch exec took %s sec", elapsed_time) + print("response is", response) return response @@ -305,14 +317,28 @@ def clean_package_list(package_details_dict: Dict): return packages_list -def get_known_pkgs(graph_response: Dict, packages_list: Dict) -> List[Dict]: +def get_known_pkgs(graph_response: Dict, packages_list: Dict, ignore_vulnerabilities: Dict) -> List[Dict]: """Analyse Known Packages.""" package_details_dict = {} + print("package list", packages_list) + print("graph_response", graph_response) + for temp in packages_list: temp["vulnerabilities"] = [] package_details_dict[temp["name"]] = temp for vulnerability in graph_response.get('result', {}).get('data'): package_details = package_details_dict.get(vulnerability["package_name"][0]) + vulns_to_ignore = ignore_vulnerabilities.get(vulnerability["package_name"][0], []) + if not isinstance(vulns_to_ignore, list): + error_msg = "Expected list of vulnerabilities to ignore in the 'ignore' dictionary as values, where the package name is the key" + raise BadRequest(error_msg) + # if package is in the 'ignore' dict and list of vulnerabilities to ignore is empty, ignore the vulnerability + if vulnerability["package_name"][0] in ignore_vulnerabilities and len(vulns_to_ignore) == 0: + continue + # if package is in the 'ignore' dict and list of vulnerabilities to ignore is not empty, ignore vulnerability + # if the vulnerability is in the list of vulnerabilities to ignore for that package + if vulnerability["package_name"][0] in ignore_vulnerabilities and vulnerability["snyk_vuln_id"][0] in vulns_to_ignore: + continue if(check_vulnerable_package(package_details["version"], vulnerability['vulnerable_versions'][0])): package_details["vulnerabilities"].append( From 51c12ff9486325832855eb93fb1b3eb4fac81b62 Mon Sep 17 00:00:00 2001 From: Akshay Bhansali Date: Thu, 11 Nov 2021 13:43:11 +0530 Subject: [PATCH 2/3] Revert "ignore vulnerabilities for VA" This reverts commit 9630e9b0340f3001dd30377e40a5c916a7c026c4. --- bayesian/api/api_v2.py | 5 +--- bayesian/utility/v2/component_analyses.py | 28 +---------------------- 2 files changed, 2 insertions(+), 31 deletions(-) diff --git a/bayesian/api/api_v2.py b/bayesian/api/api_v2.py index 884a03ef..58856fe3 100644 --- a/bayesian/api/api_v2.py +++ b/bayesian/api/api_v2.py @@ -91,7 +91,6 @@ def vulnerability_analysis_post(): """ input_json: Dict = request.get_json() ecosystem: str = input_json.get('ecosystem') - ignore_vulnerabilities: Dict = input_json.get('ignore', {}) try: # Step1: Gather and clean Request @@ -99,7 +98,7 @@ def vulnerability_analysis_post(): # Step2: Get aggregated CA data from Query GraphDB, graph_response = get_vulnerability_data(ecosystem, packages_list) # Step3: Build Unknown packages and Generates Stack Recommendation. - stack_recommendation = get_known_pkgs(graph_response, packages_list, ignore_vulnerabilities) + stack_recommendation = get_known_pkgs(graph_response, packages_list) except BadRequest as br: logger.error(br) raise HTTPError(400, str(br)) from br @@ -176,9 +175,7 @@ def component_analyses_post(): return jsonify({"message": "disabled"}), 404 try: # Step1: Gather and clean Request - print("Result Coming") packages_list, normalised_input_pkgs = ca_validate_input(input_json, ecosystem) - print(packages_list, normalised_input_pkgs) # Step2: Get aggregated CA data from Query GraphDB, graph_response = get_batch_ca_data(ecosystem, packages_list) # Step3: Build Unknown packages and Generates Stack Recommendation. diff --git a/bayesian/utility/v2/component_analyses.py b/bayesian/utility/v2/component_analyses.py index d596d68a..7044b197 100644 --- a/bayesian/utility/v2/component_analyses.py +++ b/bayesian/utility/v2/component_analyses.py @@ -78,12 +78,6 @@ def validate_input(input_json: Dict, ecosystem: str) -> List[Dict]: error_msg: str = "package_versions is missing" raise BadRequest(error_msg) - ignore_package_vulns = input_json.get('ignore', {}) - if not isinstance(ignore_package_vulns, dict): - error_msg = "Expected a dictionary 'ignore' consisting of package names as key" \ - ", and list of vulnerabilities to ignore for that package as value" - raise BadRequest(error_msg) - packages_list = [] for pkg in input_json.get('package_versions'): package = pkg.get("package") @@ -177,9 +171,7 @@ def _fetcher_in_batches(func: Callable, packages: List, def get_batch_ca_data(ecosystem: str, packages: List) -> dict: - """Fetch package details for component analyses.""" - print("Ex get_batch_ca_data") logger.debug('Executing get_batch_ca_data') started_at = time.time() @@ -196,19 +188,16 @@ def get_batch_ca_data(ecosystem: str, packages: List) -> dict: semver_packages.append(p) else: semver_packages = packages - print("semver packages", packages) graph_data_fetcher = [] if len(semver_packages) > 0: get_semver_data = functools.partial(GraphAnalyses.get_batch_ca_data, ecosystem) graph_data_fetcher = list(_fetcher_in_batches(get_semver_data, semver_packages)) - print("graph_data_fetcher", graph_data_fetcher) if len(pseudo_version_packages) > 0: get_pseudo_data = functools.partial(GraphAnalyses.get_batch_ca_data_for_pseudo_version, ecosystem) graph_data_fetcher += list(_fetcher_in_batches(get_pseudo_data, pseudo_version_packages)) - print("graph_data_fetcher", graph_data_fetcher) response = { "result": { @@ -221,7 +210,6 @@ def get_batch_ca_data(ecosystem: str, packages: List) -> dict: elapsed_time = time.time() - started_at logger.info("concurrent batch exec took %s sec", elapsed_time) - print("response is", response) return response @@ -317,28 +305,14 @@ def clean_package_list(package_details_dict: Dict): return packages_list -def get_known_pkgs(graph_response: Dict, packages_list: Dict, ignore_vulnerabilities: Dict) -> List[Dict]: +def get_known_pkgs(graph_response: Dict, packages_list: Dict) -> List[Dict]: """Analyse Known Packages.""" package_details_dict = {} - print("package list", packages_list) - print("graph_response", graph_response) - for temp in packages_list: temp["vulnerabilities"] = [] package_details_dict[temp["name"]] = temp for vulnerability in graph_response.get('result', {}).get('data'): package_details = package_details_dict.get(vulnerability["package_name"][0]) - vulns_to_ignore = ignore_vulnerabilities.get(vulnerability["package_name"][0], []) - if not isinstance(vulns_to_ignore, list): - error_msg = "Expected list of vulnerabilities to ignore in the 'ignore' dictionary as values, where the package name is the key" - raise BadRequest(error_msg) - # if package is in the 'ignore' dict and list of vulnerabilities to ignore is empty, ignore the vulnerability - if vulnerability["package_name"][0] in ignore_vulnerabilities and len(vulns_to_ignore) == 0: - continue - # if package is in the 'ignore' dict and list of vulnerabilities to ignore is not empty, ignore vulnerability - # if the vulnerability is in the list of vulnerabilities to ignore for that package - if vulnerability["package_name"][0] in ignore_vulnerabilities and vulnerability["snyk_vuln_id"][0] in vulns_to_ignore: - continue if(check_vulnerable_package(package_details["version"], vulnerability['vulnerable_versions'][0])): package_details["vulnerabilities"].append( From a9ccc341c4d08b44863cf947e11c540477074eab Mon Sep 17 00:00:00 2001 From: Akshay Bhansali Date: Thu, 11 Nov 2021 13:47:22 +0530 Subject: [PATCH 3/3] validate ignore vulnerability --- bayesian/utility/v2/sa_models.py | 1 + bayesian/utility/v2/stack_analyses.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bayesian/utility/v2/sa_models.py b/bayesian/utility/v2/sa_models.py index ad1064ed..d64ef348 100644 --- a/bayesian/utility/v2/sa_models.py +++ b/bayesian/utility/v2/sa_models.py @@ -48,6 +48,7 @@ class StackAnalysesPostRequest(BaseModel): True, description='This is required to enable or disable the transitive support', ) + ignore: Optional[Dict[str, List[str]]] class Config: """Validation configuration for model.""" diff --git a/bayesian/utility/v2/stack_analyses.py b/bayesian/utility/v2/stack_analyses.py index aef9a269..325c294b 100644 --- a/bayesian/utility/v2/stack_analyses.py +++ b/bayesian/utility/v2/stack_analyses.py @@ -45,7 +45,7 @@ def post_request(self): logger.info('SA Post request with ecosystem: %s manifest: %s path: %s ' 'show_transitive: %s', self.params.ecosystem, self.params.manifest.filename, self.params.file_path, - self.params.show_transitive) + self.params.show_transitive, self.params.ignore) # Build manifest file info. self._manifest_file_info = { 'filename': self.params.manifest.filename, @@ -147,7 +147,8 @@ def _make_backbone_request(self): 'packages': data['packages'], 'manifest_name': self._manifest_file_info['filename'], 'manifest_file_path': self._manifest_file_info['filepath'], - 'show_transitive': self.params.show_transitive + 'show_transitive': self.params.show_transitive, + 'ignore': self.params.ignore } request_params = { 'persist': 'true',