Merge branch 'master' into dependabot/pip/black-25.1.0

CHANGELOG.md

@@ -31,6 +31,7 @@
 - strings: add type hints and fix uncovered bugs @williballenthin #2555
 - elffile: handle symbols without a name @williballenthin #2553
 - project: remove pytest-cov that wasn't used @williballenthin @2491
+- rules: scopes can now have subscope blocks with the same scope @williballenthin #2584
 
 ### capa Explorer Web
 

capa/features/extractors/cape/extractor.py

@@ -22,8 +22,8 @@
 import capa.features.extractors.cape.global_
 import capa.features.extractors.cape.process
 from capa.exceptions import EmptyReportError, UnsupportedFormatError
-from capa.features.common import Feature, Characteristic
-from capa.features.address import NO_ADDRESS, Address, AbsoluteVirtualAddress, _NoAddress
+from capa.features.common import Feature
+from capa.features.address import Address, AbsoluteVirtualAddress, _NoAddress
 from capa.features.extractors.cape.models import Call, Static, Process, CapeReport
 from capa.features.extractors.base_extractor import (
     CallHandle,
@@ -77,11 +77,7 @@ def get_threads(self, ph: ProcessHandle) -> Iterator[ThreadHandle]:
         yield from capa.features.extractors.cape.process.get_threads(ph)
 
     def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[tuple[Feature, Address]]:
-        if False:
-            # force this routine to be a generator,
-            # but we don't actually have any elements to generate.
-            yield Characteristic("never"), NO_ADDRESS
-        return
+        yield from []
 
     def get_calls(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[CallHandle]:
         yield from capa.features.extractors.cape.thread.get_calls(ph, th)

capa/features/extractors/drakvuf/extractor.py

@@ -21,7 +21,7 @@
 import capa.features.extractors.drakvuf.thread
 import capa.features.extractors.drakvuf.global_
 import capa.features.extractors.drakvuf.process
-from capa.features.common import Feature, Characteristic
+from capa.features.common import Feature
 from capa.features.address import NO_ADDRESS, Address, ThreadAddress, ProcessAddress, AbsoluteVirtualAddress, _NoAddress
 from capa.features.extractors.base_extractor import (
     CallHandle,
@@ -74,11 +74,7 @@ def get_threads(self, ph: ProcessHandle) -> Iterator[ThreadHandle]:
         yield from capa.features.extractors.drakvuf.process.get_threads(self.sorted_calls, ph)
 
     def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[tuple[Feature, Address]]:
-        if False:
-            # force this routine to be a generator,
-            # but we don't actually have any elements to generate.
-            yield Characteristic("never"), NO_ADDRESS
-        return
+        yield from []
 
     def get_calls(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[CallHandle]:
         yield from capa.features.extractors.drakvuf.thread.get_calls(self.sorted_calls, ph, th)

capa/features/extractors/elf.py

@@ -1088,7 +1088,7 @@ def guess_os_from_go_buildinfo(elf: ELF) -> Optional[OS]:
     #  and the 32-byte header is followed by varint-prefixed string data
     #  for the two string values we care about.
     # https://github.com/mandiant/GoReSym/blob/0860a1b1b4f3495e9fb7e71eb4386bf3e0a7c500/buildinfo/buildinfo.go#L185-L193
-    BUILDINFO_MAGIC = b"\xFF Go buildinf:"
+    BUILDINFO_MAGIC = b"\xff Go buildinf:"
 
     try:
         index = buf.index(BUILDINFO_MAGIC)

capa/features/extractors/pefile.py

@@ -115,10 +115,7 @@ def extract_file_function_names(**kwargs):
     """
     extract the names of statically-linked library functions.
     """
-    if False:
-        # using a `yield` here to force this to be a generator, not function.
-        yield NotImplementedError("pefile doesn't have library matching")
-    return
+    yield from []
 
 
 def extract_file_os(**kwargs):

capa/rules/__init__.py

@@ -597,6 +597,43 @@ def unique(sequence):
     return [x for x in sequence if not (x in seen or seen.add(x))]  # type: ignore [func-returns-value]
 
 
+STATIC_SCOPE_ORDER = [
+    Scope.FILE,
+    Scope.FUNCTION,
+    Scope.BASIC_BLOCK,
+    Scope.INSTRUCTION,
+]
+
+
+DYNAMIC_SCOPE_ORDER = [
+    Scope.FILE,
+    Scope.PROCESS,
+    Scope.THREAD,
+    Scope.SPAN_OF_CALLS,
+    Scope.CALL,
+]
+
+
+def is_subscope_compatible(scope: Scope | None, subscope: Scope) -> bool:
+    if not scope:
+        return False
+
+    if subscope in STATIC_SCOPE_ORDER:
+        try:
+            return STATIC_SCOPE_ORDER.index(subscope) >= STATIC_SCOPE_ORDER.index(scope)
+        except ValueError:
+            return False
+
+    elif subscope in DYNAMIC_SCOPE_ORDER:
+        try:
+            return DYNAMIC_SCOPE_ORDER.index(subscope) >= DYNAMIC_SCOPE_ORDER.index(scope)
+        except ValueError:
+            return False
+
+    else:
+        raise ValueError("unexpected scope")
+
+
 def build_statements(d, scopes: Scopes):
     if len(d.keys()) > 2:
         raise InvalidRule("too many statements")
@@ -621,7 +658,7 @@ def build_statements(d, scopes: Scopes):
         return ceng.Some(0, unique(build_statements(dd, scopes) for dd in d[key]), description=description)
 
     elif key == "process":
-        if Scope.FILE not in scopes:
+        if not is_subscope_compatible(scopes.dynamic, Scope.PROCESS):
             raise InvalidRule("`process` subscope supported only for `file` scope")
 
         if len(d[key]) != 1:
@@ -632,7 +669,7 @@ def build_statements(d, scopes: Scopes):
         )
 
     elif key == "thread":
-        if all(s not in scopes for s in (Scope.FILE, Scope.PROCESS)):
+        if not is_subscope_compatible(scopes.dynamic, Scope.THREAD):
             raise InvalidRule("`thread` subscope supported only for the `process` scope")
 
         if len(d[key]) != 1:
@@ -643,7 +680,7 @@ def build_statements(d, scopes: Scopes):
         )
 
     elif key == "span of calls":
-        if all(s not in scopes for s in (Scope.FILE, Scope.PROCESS, Scope.THREAD, Scope.SPAN_OF_CALLS)):
+        if not is_subscope_compatible(scopes.dynamic, Scope.SPAN_OF_CALLS):
             raise InvalidRule("`span of calls` subscope supported only for the `process` and `thread` scopes")
 
         if len(d[key]) != 1:
@@ -656,7 +693,7 @@ def build_statements(d, scopes: Scopes):
         )
 
     elif key == "call":
-        if all(s not in scopes for s in (Scope.FILE, Scope.PROCESS, Scope.THREAD, Scope.SPAN_OF_CALLS, Scope.CALL)):
+        if not is_subscope_compatible(scopes.dynamic, Scope.CALL):
             raise InvalidRule("`call` subscope supported only for the `process`, `thread`, and `call` scopes")
 
         if len(d[key]) != 1:
@@ -667,7 +704,7 @@ def build_statements(d, scopes: Scopes):
         )
 
     elif key == "function":
-        if Scope.FILE not in scopes:
+        if not is_subscope_compatible(scopes.static, Scope.FUNCTION):
             raise InvalidRule("`function` subscope supported only for `file` scope")
 
         if len(d[key]) != 1:
@@ -678,7 +715,7 @@ def build_statements(d, scopes: Scopes):
         )
 
     elif key == "basic block":
-        if Scope.FUNCTION not in scopes:
+        if not is_subscope_compatible(scopes.static, Scope.BASIC_BLOCK):
             raise InvalidRule("`basic block` subscope supported only for `function` scope")
 
         if len(d[key]) != 1:
@@ -689,7 +726,7 @@ def build_statements(d, scopes: Scopes):
         )
 
     elif key == "instruction":
-        if all(s not in scopes for s in (Scope.FUNCTION, Scope.BASIC_BLOCK)):
+        if not is_subscope_compatible(scopes.static, Scope.INSTRUCTION):
             raise InvalidRule("`instruction` subscope supported only for `function` and `basic block` scope")
 
         if len(d[key]) == 1:

pyproject.toml

@@ -138,7 +138,7 @@ dev = [
     "flake8-copyright==0.2.4",
     "ruff==0.9.2",
     "black==25.1.0",
-    "isort==5.13.2",
+    "isort==6.0.0",
     "mypy==1.14.1",
     "mypy-protobuf==3.6.0",
     "PyGithub==2.5.0",

scripts/lint.py

@@ -48,7 +48,7 @@
 import capa.helpers
 import capa.features.insn
 import capa.capabilities.common
-from capa.rules import Rule, Scope, RuleSet
+from capa.rules import Rule, RuleSet
 from capa.features.common import OS_AUTO, String, Feature, Substring
 from capa.render.result_document import RuleMetadata
 
@@ -536,15 +536,8 @@ def _is_static_scope_compatible(parent: Rule, child: Rule) -> bool:
             # Assume for now it is not.
             return True
 
-        static_scope_order = [
-            None,
-            Scope.FILE,
-            Scope.FUNCTION,
-            Scope.BASIC_BLOCK,
-            Scope.INSTRUCTION,
-        ]
-
-        return static_scope_order.index(child.scopes.static) >= static_scope_order.index(parent.scopes.static)
+        assert child.scopes.static is not None
+        return capa.rules.is_subscope_compatible(parent.scopes.static, child.scopes.static)
 
     @staticmethod
     def _is_dynamic_scope_compatible(parent: Rule, child: Rule) -> bool:
@@ -563,16 +556,8 @@ def _is_dynamic_scope_compatible(parent: Rule, child: Rule) -> bool:
             # Assume for now it is not.
             return True
 
-        dynamic_scope_order = [
-            None,
-            Scope.FILE,
-            Scope.PROCESS,
-            Scope.THREAD,
-            Scope.SPAN_OF_CALLS,
-            Scope.CALL,
-        ]
-
-        return dynamic_scope_order.index(child.scopes.dynamic) >= dynamic_scope_order.index(parent.scopes.dynamic)
+        assert child.scopes.dynamic is not None
+        return capa.rules.is_subscope_compatible(parent.scopes.dynamic, child.scopes.dynamic)
 
 
 class OptionalNotUnderAnd(Lint):