[aws][fix] Collect and connect Inspector resources properly (#2253)

Co-authored-by: Matthias Veit <[email protected]>

fixlib/fixlib/baseresources.py

@@ -259,7 +259,7 @@ class Finding:
     severity: Severity = Severity.medium
     description: Optional[str] = None
     remediation: Optional[str] = None
-    created_at: Optional[datetime] = None
+    updated_at: Optional[datetime] = None
     details: Optional[Json] = None
 
 
@@ -409,6 +409,13 @@ def log(self, msg: str, data: Optional[Any] = None, exception: Optional[Exceptio
         self.__log.append(log_entry)
         self._changes.add("log")
 
+    def add_finding(self, provider: str, finding: Finding) -> None:
+        for assessment in self._assessments:
+            if assessment.provider == provider:
+                assessment.findings.append(finding)
+                return
+        self._assessments.append(Assessment(provider=provider, findings=[finding]))
+
     def add_change(self, change: str) -> None:
         self._changes.add(change)
 

plugins/aws/Makefile

@@ -29,7 +29,7 @@ clean-test: ## remove test and coverage artifacts
 
 lint: ## static code analysis
 	black --line-length 120 --check fix_plugin_aws test
-	flake8 fix_plugin_aws
+	flake8 fix_plugin_aws test
 	mypy --python-version 3.12 --strict --install-types fix_plugin_aws test
 
 test: ## run tests quickly with the default Python

plugins/aws/fix_plugin_aws/collector.py

@@ -50,6 +50,7 @@
     backup,
     bedrock,
     scp,
+    inspector,
 )
 from fix_plugin_aws.resource.base import (
     AwsAccount,
@@ -118,6 +119,7 @@
     + backup.resources
     + amazonq.resources
     + bedrock.resources
+    + inspector.resources
 )
 all_resources: List[Type[AwsResource]] = global_resources + regional_resources
 
@@ -244,6 +246,10 @@ def get_last_run() -> Optional[datetime]:
                     )
             shared_queue.wait_for_submitted_work()
 
+            # call all registered after collect hooks
+            for after_collect in global_builder.after_collect_actions:
+                after_collect()
+
             # connect nodes
             log.info(f"[Aws:{self.account.id}] Connect resources and create edges.")
             for node, data in list(self.graph.nodes(data=True)):

plugins/aws/fix_plugin_aws/resource/backup.py

@@ -175,7 +175,7 @@ class AwsBackupProtectedResource(AwsResource):
     }
     api_spec: ClassVar[AwsApiSpec] = AwsApiSpec("backup", "list-protected-resources", "Results")
     mapping: ClassVar[Dict[str, Bender]] = {
-        "id": S("ResourceArn") >> F(lambda arn: arn.rsplit("/")[1]),
+        "id": S("ResourceArn") >> F(AwsResource.id_from_arn),
         "name": S("ResourceName"),
         "resource_arn": S("ResourceArn"),
         "resource_type": S("ResourceType"),

plugins/aws/fix_plugin_aws/resource/base.py

@@ -453,6 +453,7 @@ def __init__(
         graph_nodes_access: Optional[RWLock] = None,
         graph_edges_access: Optional[RWLock] = None,
         last_run_started_at: Optional[datetime] = None,
+        after_collect_actions: Optional[List[Callable[[], Any]]] = None,
     ) -> None:
         self.graph = graph
         self.cloud = cloud
@@ -469,6 +470,7 @@ def __init__(
         self.last_run_started_at = last_run_started_at
         self.created_at = utc()
         self.__builder_cache = {region.safe_name: self}
+        self.after_collect_actions = after_collect_actions if after_collect_actions is not None else []
 
         if last_run_started_at:
             now = utc()
@@ -705,6 +707,7 @@ def for_region(self, region: AwsRegion) -> GraphBuilder:
             self.graph_nodes_access,
             self.graph_edges_access,
             self.last_run_started_at,
+            self.after_collect_actions,
         )
         self.__builder_cache[region.safe_name] = builder
         return builder

plugins/aws/fix_plugin_aws/resource/ec2.py

@@ -1,14 +1,14 @@
 import base64
-from functools import partial
+import copy
 import logging
 from contextlib import suppress
 from datetime import datetime, timedelta
+from functools import partial
 from typing import ClassVar, Dict, Optional, List, Type, Any
-import copy
 
 from attrs import define, field
-from fix_plugin_aws.aws_client import AwsClient
 
+from fix_plugin_aws.aws_client import AwsClient
 from fix_plugin_aws.resource.base import AwsResource, GraphBuilder, AwsApiSpec, get_client
 from fix_plugin_aws.resource.cloudwatch import (
     AwsCloudwatchQuery,
@@ -18,9 +18,9 @@
     operations_to_iops,
     normalizer_factory,
 )
+from fix_plugin_aws.resource.iam import AwsIamInstanceProfile
 from fix_plugin_aws.resource.kms import AwsKmsKey
 from fix_plugin_aws.resource.s3 import AwsS3Bucket
-from fix_plugin_aws.resource.iam import AwsIamInstanceProfile
 from fix_plugin_aws.utils import ToDict, TagsValue
 from fixlib.baseresources import (
     BaseInstance,
@@ -49,7 +49,6 @@
 from fixlib.json_bender import Bender, S, Bend, ForallBend, bend, MapEnum, F, K, StripNones
 from fixlib.types import Json
 
-
 # region InstanceType
 from fixlib.utils import utc
 

plugins/aws/fix_plugin_aws/resource/ecr.py

@@ -1,14 +1,14 @@
 import json
 import logging
-from typing import ClassVar, Dict, Optional, List, Tuple, Type, Any
 from json import loads as json_loads
+from typing import ClassVar, Dict, Optional, List, Tuple, Type, Any
 
 from attrs import define, field
 from boto3.exceptions import Boto3Error
 
 from fix_plugin_aws.resource.base import AwsResource, AwsApiSpec, GraphBuilder
 from fix_plugin_aws.utils import ToDict
-from fixlib.baseresources import HasResourcePolicy, PolicySource, PolicySourceKind
+from fixlib.baseresources import HasResourcePolicy, ModelReference, PolicySource, PolicySourceKind
 from fixlib.json import sort_json
 from fixlib.json_bender import Bender, S, Bend
 from fixlib.types import Json
@@ -34,6 +34,7 @@ class AwsEcrRepository(AwsResource, HasResourcePolicy):
     _kind_service: ClassVar[Optional[str]] = service_name
     _metadata: ClassVar[Dict[str, Any]] = {"icon": "repository", "group": "compute"}
     _aws_metadata: ClassVar[Dict[str, Any]] = {"provider_link_tpl": "https://{region_id}.console.aws.amazon.com/ecr/repositories/{name}?region={region}", "arn_tpl": "arn:{partition}:ecr:{region}:{account}:repository/{name}"}  # fmt: skip
+    _reference_kinds: ClassVar[ModelReference] = {}
     api_spec: ClassVar[AwsApiSpec] = AwsApiSpec("ecr", "describe-repositories", "repositories")
     public_spec: ClassVar[AwsApiSpec] = AwsApiSpec("ecr-public", "describe-repositories", "repositories")
     mapping: ClassVar[Dict[str, Bender]] = {

plugins/aws/fix_plugin_aws/resource/inspector.py

@@ -0,0 +1,187 @@
+import logging
+from datetime import datetime
+from functools import partial
+from typing import ClassVar, Dict, Optional, List, Tuple, Type, Any
+
+from attrs import define, field
+from boto3.exceptions import Boto3Error
+
+from fix_plugin_aws.resource.base import AwsResource, AwsApiSpec, GraphBuilder
+from fix_plugin_aws.resource.ec2 import AwsEc2Instance
+from fix_plugin_aws.resource.ecr import AwsEcrRepository
+from fix_plugin_aws.resource.lambda_ import AwsLambdaFunction
+from fixlib.baseresources import PhantomBaseResource, Severity, Finding
+from fixlib.json_bender import Bender, S, ForallBend, Bend, F
+from fixlib.types import Json
+
+log = logging.getLogger("fix.plugins.aws")
+service_name = "inspector2"
+
+amazon_inspector = "amazon_inspector"
+
+
+@define(eq=False, slots=False)
+class AwsInspectorRecommendation:
+    kind: ClassVar[str] = "aws_inspector_recommendation"
+    mapping: ClassVar[Dict[str, Bender]] = {"url": S("Url"), "text": S("text")}
+    url: Optional[str] = field(default=None, metadata={"description": "The URL address to the CVE remediation recommendations."})  # fmt: skip
+    text: Optional[str] = field(default=None, metadata={"description": "The recommended course of action to remediate the finding."})  # fmt: skip
+
+
+@define(eq=False, slots=False)
+class AwsInspectorRemediation:
+    kind: ClassVar[str] = "aws_inspector_remediation"
+    mapping: ClassVar[Dict[str, Bender]] = {
+        "recommendation": S("recommendation") >> Bend(AwsInspectorRecommendation.mapping)
+    }
+    recommendation: Optional[AwsInspectorRecommendation] = field(default=None, metadata={"description": "An object that contains information about the recommended course of action to remediate the finding."})  # fmt: skip
+
+
+@define(eq=False, slots=False)
+class AwsInspectorResource:
+    kind: ClassVar[str] = "aws_inspector_resource"
+    mapping: ClassVar[Dict[str, Bender]] = {
+        # "details": S("details") # not used
+        "id": S("id"),
+        "partition": S("partition"),
+        "region": S("region"),
+        "type": S("type"),
+    }
+    id: Optional[str] = field(default=None, metadata={"description": "The ID of the resource."})  # fmt: skip
+    partition: Optional[str] = field(default=None, metadata={"description": "The partition of the resource."})  # fmt: skip
+    region: Optional[str] = field(default=None, metadata={"description": "The Amazon Web Services Region the impacted resource is located in."})  # fmt: skip
+    type: Optional[str] = field(default=None, metadata={"description": "The type of resource."})  # fmt: skip
+
+
+@define(eq=False, slots=False)
+class AwsInspectorFinding(AwsResource, PhantomBaseResource):
+    kind: ClassVar[str] = "aws_inspector_finding"
+    api_spec: ClassVar[AwsApiSpec] = AwsApiSpec(service_name, "list-findings")
+    _model_export: ClassVar[bool] = False  # do not export this class, since there will be no instances of it
+    mapping: ClassVar[Dict[str, Bender]] = {
+        "id": S("findingArn") >> F(AwsResource.id_from_arn),
+        "name": S("title"),
+        "mtime": S("updatedAt"),
+        "arn": S("findingArn"),
+        "aws_account_id": S("awsAccountId"),
+        "description": S("description"),
+        "epss": S("epss", "score"),
+        "exploit_available": S("exploitAvailable"),
+        "exploitability_details": S("exploitabilityDetails", "lastKnownExploitAt"),
+        "finding_arn": S("findingArn"),
+        "first_observed_at": S("firstObservedAt"),
+        "fix_available": S("fixAvailable"),
+        "inspector_score": S("inspectorScore"),
+        "last_observed_at": S("lastObservedAt"),
+        "remediation": S("remediation") >> Bend(AwsInspectorRemediation.mapping),
+        "finding_resources": S("resources", default=[]) >> ForallBend(AwsInspectorResource.mapping),
+        "finding_severity": S("severity"),
+        "status": S("status"),
+        "title": S("title"),
+        "type": S("type"),
+        "updated_at": S("updatedAt"),
+        # available but not used properties:
+        # "inspector_score_details": S("inspectorScoreDetails")
+        # "code_vulnerability_details": S("codeVulnerabilityDetails")
+        # "network_reachability_details": S("networkReachabilityDetails")
+        # "package_vulnerability_details": S("packageVulnerabilityDetails")
+    }
+    aws_account_id: Optional[str] = field(default=None, metadata={"description": "The Amazon Web Services account ID associated with the finding."})  # fmt: skip
+    description: Optional[str] = field(default=None, metadata={"description": "The description of the finding."})  # fmt: skip
+    epss: Optional[float] = field(default=None, metadata={"description": "The finding's EPSS score."})  # fmt: skip
+    exploit_available: Optional[str] = field(default=None, metadata={"description": "If a finding discovered in your environment has an exploit available."})  # fmt: skip
+    exploitability_details: Optional[datetime] = field(default=None, metadata={"description": "The details of an exploit available for a finding discovered in your environment."})  # fmt: skip
+    finding_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Number (ARN) of the finding."})  # fmt: skip
+    first_observed_at: Optional[datetime] = field(default=None, metadata={"description": "The date and time that the finding was first observed."})  # fmt: skip
+    fix_available: Optional[str] = field(default=None, metadata={"description": "Details on whether a fix is available through a version update. This value can be YES, NO, or PARTIAL. A PARTIAL fix means that some, but not all, of the packages identified in the finding have fixes available through updated versions."})  # fmt: skip
+    inspector_score: Optional[float] = field(default=None, metadata={"description": "The Amazon Inspector score given to the finding."})  # fmt: skip
+    last_observed_at: Optional[datetime] = field(default=None, metadata={"description": "The date and time the finding was last observed. This timestamp for this field remains unchanged until a finding is updated."})  # fmt: skip
+    remediation: Optional[AwsInspectorRemediation] = field(default=None, metadata={"description": "An object that contains the details about how to remediate a finding."})  # fmt: skip
+    finding_resources: Optional[List[AwsInspectorResource]] = field(factory=list, metadata={"description": "Contains information on the resources involved in a finding. The resource value determines the valid values for type in your request. For more information, see Finding types in the Amazon Inspector user guide."})  # fmt: skip
+    finding_severity: Optional[str] = field(default=None, metadata={"description": "The severity of the finding. UNTRIAGED applies to PACKAGE_VULNERABILITY type findings that the vendor has not assigned a severity yet. For more information, see Severity levels for findings in the Amazon Inspector user guide."})  # fmt: skip
+    status: Optional[str] = field(default=None, metadata={"description": "The status of the finding."})  # fmt: skip
+    title: Optional[str] = field(default=None, metadata={"description": "The title of the finding."})  # fmt: skip
+    type: Optional[str] = field(default=None, metadata={"description": "The type of the finding. The type value determines the valid values for resource in your request. For more information, see Finding types in the Amazon Inspector user guide."})  # fmt: skip
+    updated_at: Optional[datetime] = field(default=None, metadata={"description": "The date and time the finding was last updated at."})  # fmt: skip
+
+    def parse_finding(self, source: Json) -> Finding:
+        severity_mapping = {
+            "INFORMATIONAL": Severity.info,
+            "LOW": Severity.low,
+            "MEDIUM": Severity.medium,
+            "HIGH": Severity.high,
+            "CRITICAL": Severity.critical,
+        }
+        finding_title = self.safe_name
+        if not self.finding_severity:
+            finding_severity = Severity.medium
+        else:
+            finding_severity = severity_mapping.get(self.finding_severity, Severity.medium)
+        description = self.description
+        remediation = ""
+        if self.remediation and self.remediation.recommendation:
+            remediation = self.remediation.recommendation.text or ""
+        updated_at = self.updated_at
+        details = source.get("packageVulnerabilityDetails", {}) | source.get("codeVulnerabilityDetails", {})
+        return Finding(finding_title, finding_severity, description, remediation, updated_at, details)
+
+    @classmethod
+    def collect_resources(cls, builder: GraphBuilder) -> None:
+        def check_type_and_adjust_id(
+            class_type: Optional[str], resource_id: Optional[str]
+        ) -> Tuple[Optional[Type[Any]], Optional[Dict[str, Any]]]:
+            if not resource_id or not class_type:
+                return None, None
+            match class_type:
+                case "AWS_LAMBDA_FUNCTION":
+                    # remove lambda's version from arn
+                    lambda_arn = resource_id.rsplit(":", 1)[0]
+                    return AwsLambdaFunction, {"arn": lambda_arn}
+                case "AWS_EC2_INSTANCE":
+                    return AwsEc2Instance, {"id": resource_id}
+                case "AWS_ECR_REPOSITORY":
+                    return AwsEcrRepository, {"id": resource_id, "_region": builder.region}
+                case _:
+                    return None, None
+
+        def add_finding(
+            provider: str, finding: Finding, clazz: Optional[Type[AwsResource]] = None, **node: Any
+        ) -> None:
+            if resource := builder.node(clazz=clazz, **node):
+                resource.add_finding(provider, finding)
+
+        # Default behavior: in case the class has an ApiSpec, call the api and call collect.
+        log.debug(f"Collecting {cls.__name__} in region {builder.region.name}")
+        try:
+            for item in builder.client.list(
+                aws_service=service_name,
+                action="list-findings",
+                result_name="findings",
+                expected_errors=["AccessDeniedException"],
+                filterCriteria={"awsAccountId": [{"comparison": "EQUALS", "value": f"{builder.account.id}"}]},
+            ):
+                if finding := AwsInspectorFinding.from_api(item, builder):
+                    for fr in finding.finding_resources or []:
+                        clazz, res_filter = check_type_and_adjust_id(fr.type, fr.id)
+                        if clazz and res_filter:
+                            # append the finding when all resources have been collected
+                            builder.after_collect_actions.append(
+                                partial(
+                                    add_finding,
+                                    amazon_inspector,
+                                    finding.parse_finding(item),
+                                    clazz,
+                                    **res_filter,
+                                )
+                            )
+        except Boto3Error as e:
+            msg = f"Error while collecting {cls.__name__} in region {builder.region.name}: {e}"
+            builder.core_feedback.error(msg, log)
+            raise
+        except Exception as e:
+            msg = f"Error while collecting {cls.__name__} in region {builder.region.name}: {e}"
+            builder.core_feedback.info(msg, log)
+            raise
+
+
+resources: List[Type[AwsResource]] = [AwsInspectorFinding]

plugins/aws/fix_plugin_aws/resource/lambda_.py

@@ -1,10 +1,11 @@
-from datetime import timedelta
 import json as json_p
 import logging
 import re
+from datetime import timedelta
 from typing import ClassVar, Dict, Optional, List, Tuple, Type, Any
 
 from attrs import define, field
+
 from fix_plugin_aws.aws_client import AwsClient
 from fix_plugin_aws.resource.base import AwsResource, GraphBuilder, AwsApiSpec, parse_json
 from fix_plugin_aws.resource.cloudwatch import AwsCloudwatchQuery, normalizer_factory
@@ -19,9 +20,9 @@
     PolicySourceKind,
 )
 from fixlib.graph import Graph
+from fixlib.json import sort_json
 from fixlib.json_bender import Bender, S, Bend, ForallBend, F, bend
 from fixlib.types import Json
-from fixlib.json import sort_json
 
 log = logging.getLogger("fix.plugins.aws")
 

plugins/aws/test/collector_test.py

@@ -29,6 +29,9 @@ def count_kind(clazz: Type[AwsResource]) -> int:
         return count
 
     for resource in all_resources:
+        # there will be no instances of resources that are not exported
+        if not resource._model_export:
+            continue
         assert count_kind(resource) > 0, f"No instances of {resource.__name__} found"
 
     # make sure all threads have been joined
@@ -106,6 +109,8 @@ def all_base_classes(cls: Type[Any]) -> Set[Type[Any]]:
     expected_declared_properties = ["kind", "_kind_display"]
     expected_props_in_hierarchy = ["_kind_service", "_metadata"]
     for rc in all_resources:
+        if not rc._model_export:
+            continue
         for prop in expected_declared_properties:
             assert prop in rc.__dict__, f"{rc.__name__} missing {prop}"
         with_bases = (all_base_classes(rc) | {rc}) - {AwsResource, BaseResource}

plugins/aws/test/resources/__init__.py

@@ -158,6 +158,8 @@ def build_graph(
         for cls in clazz if isinstance(clazz, list) else [clazz]:
             cls.collect_resources(builder)
         builder.executor.wait_for_submitted_work()
+        for after_collect in builder.after_collect_actions:
+            after_collect()
         return builder