Merge branch 'main' into feature/mula/create-schedule-overrides

.github/workflows/pre_commit_checks.yml

@@ -24,7 +24,7 @@ jobs:
           cache: pip
 
       - name: Install pre-commit
-        run: pip install pre-commit==3.8.0
+        run: pip install pre-commit==4.0.1
 
       - uses: actions/cache@v4
         with:

.gitignore

@@ -449,6 +449,8 @@ docs/source/_static/mermaid.min.js
 # rpki cache
 /boefjes/boefjes/plugins/kat_rpki/rpki.json
 /boefjes/boefjes/plugins/kat_rpki/rpki-meta.json
+/boefjes/boefjes/plugins/kat_rpki/bgp.jsonl
+/boefjes/boefjes/plugins/kat_rpki/bgp-meta.json
 
 *.pstat
 **/.cache*

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: mixed-line-ending
@@ -29,13 +29,13 @@ repos:
         args: ["--autofix", "--no-ensure-ascii", "--no-sort-keys"]
 
   - repo: https://github.com/abravalheri/validate-pyproject
-    rev: v0.16
+    rev: v0.23
     hooks:
       - id: validate-pyproject
         files: pyproject.toml$
 
   - repo: https://github.com/rstcheck/rstcheck
-    rev: v6.2.1
+    rev: v6.2.4
     hooks:
       - id: rstcheck
         # https://github.com/rstcheck/rstcheck-core/issues/4
@@ -49,37 +49,37 @@ repos:
         additional_dependencies: ["rstcheck[sphinx]", "autodoc-pydantic==2.1.0"]
 
   - repo: https://github.com/MarketSquare/robotframework-tidy
-    rev: "4.11.0"
+    rev: "4.14.0"
     hooks:
       - id: robotidy
 
   - repo: https://github.com/jendrikseipp/vulture
-    rev: v2.11
+    rev: v2.13
     hooks:
       - id: vulture
         exclude: |
           /tests/
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.3.5"
+    rev: "v0.8.1"
     hooks:
       - id: ruff
       - id: ruff-format
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.2
+    rev: v3.19.0
     hooks:
       - id: pyupgrade
         args: [--py310-plus]
 
   - repo: https://github.com/adamchainz/django-upgrade
-    rev: 1.16.0
+    rev: 1.22.1
     hooks:
       - id: django-upgrade
         args: [--target-version, "5.0"]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.9.0
+    rev: v1.13.0
     hooks:
       - id: mypy
         additional_dependencies:
@@ -106,7 +106,7 @@ repos:
           )
 
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.6
+    rev: v2.3.0
     hooks:
       - id: codespell
         additional_dependencies: ["tomli"]
@@ -136,7 +136,7 @@ repos:
           )
 
   - repo: https://github.com/Riverside-Healthcare/djLint
-    rev: v1.34.1
+    rev: v1.36.3
     hooks:
       - id: djlint-reformat-django
         files: |
@@ -155,7 +155,7 @@ repos:
         exclude: '^rocky/rocky/templates/admin/.*\.html$'
 
   - repo: https://github.com/thibaudcolas/pre-commit-stylelint
-    rev: v16.3.1
+    rev: v16.10.0
     hooks:
       - id: stylelint
         args: [--fix]
@@ -171,13 +171,13 @@ repos:
         args: ["-e", "SC1091"]
 
   - repo: https://github.com/scop/pre-commit-shfmt
-    rev: v3.8.0-1
+    rev: v3.10.0-1
     hooks:
       - id: shfmt
         args: ["-w", "-s", "-i", "4", "-sr"]
 
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v3.1.0
+    rev: v4.0.0-alpha.8
     hooks:
       - id: prettier
         additional_dependencies:

boefjes/boefjes/plugins/helpers.py

@@ -54,18 +54,22 @@ def get_file_from_container(container: docker.models.containers.Container, path:
         logging.warning("%s not found in container %s %s", path, container.short_id, container.image.tags)
         return None
 
-    f = tarfile.open(mode="r|", fileobj=TarStream(stream).reader())
-    tarobject = f.next()
-    if not tarobject or tarobject.name != os.path.basename(path):
-        logging.warning("%s not found in tarfile from container %s %s", path, container.short_id, container.image.tags)
-        return None
+    with tarfile.open(mode="r|", fileobj=TarStream(stream).reader()) as f:
+        tarobject = f.next()
+        if not tarobject or tarobject.name != os.path.basename(path):
+            logging.warning(
+                "%s not found in tarfile from container %s %s", path, container.short_id, container.image.tags
+            )
+            return None
 
-    extracted_file = f.extractfile(tarobject)
-    if not extracted_file:
-        logging.warning("%s not found in tarfile from container %s %s", path, container.short_id, container.image.tags)
-        return None
+        extracted_file = f.extractfile(tarobject)
+        if not extracted_file:
+            logging.warning(
+                "%s not found in tarfile from container %s %s", path, container.short_id, container.image.tags
+            )
+            return None
 
-    return extracted_file.read()
+        return extracted_file.read()
 
 
 def cpe_to_name_version(cpe: str) -> tuple[str | None, str | None]:

boefjes/boefjes/plugins/kat_kat_finding_types/kat_finding_types.json

@@ -509,5 +509,12 @@
     "source": "Check the version of the host manually.",
     "impact": "Unknown. The server may or may not be vulnerable. OpenKAT is not able to determine the version.",
     "recommendation": "Verify manually if the software is up to date as OpenKAT is not able to determine the software version ."
+  },
+  "KAT-LAME-DELEGATION": {
+    "description": "The nameservers for this object are configured but cannot be reached or are non existent.",
+    "risk": "recommendation",
+    "source": "Check the nameservers of the host or iprange manually.",
+    "impact": "No resolving can be done for this IP or host. This might cause problems for mailservers or slow down connections.",
+    "recommendation": "Verify that the listed nameservers are reachable and have valid hostnames."
   }
 }

boefjes/boefjes/plugins/kat_rdns/main.py

@@ -1,4 +1,5 @@
 import dns
+from dns.edns import EDEOption
 from dns.resolver import Answer
 
 from boefjes.config import settings
@@ -9,14 +10,29 @@ def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]:
     """return results to normalizer."""
     ip = boefje_meta.arguments["input"]["address"]
 
+    resolver = dns.resolver.Resolver()
+    # https://dnspython.readthedocs.io/en/stable/_modules/dns/edns.html
+    # enable EDE to get the ServFail return values if the server supports it # codespell-ignore
+    resolver.use_edns(options=[EDEOption(15)])
+    resolver.nameservers = [str(settings.remote_ns)]
+    reverse_ip = dns.reversename.from_address(ip)
     try:
-        resolver = dns.resolver.Resolver()
-        resolver.nameservers = [str(settings.remote_ns)]
-        reverse_ip = dns.reversename.from_address(ip)
         answer: Answer = resolver.resolve(reverse_ip, "PTR")
         result = f"RESOLVER: {answer.nameserver}\n{answer.response}"
         return [(set(), result)]
     except dns.resolver.NXDOMAIN:
         return [(set(), "NXDOMAIN")]
+    except dns.resolver.NoNameservers as error:
+        # no servers responded happily, we'll check the response from the first
+        # https://dnspython.readthedocs.io/en/latest/_modules/dns/rcode.html
+        # https://www.rfc-editor.org/rfc/rfc8914#name-extended-dns-error-code-6-d
+        first_error = error.kwargs["errors"][0]
+        if first_error[3] == "SERVFAIL":
+            for ede_error in first_error[4].options:
+                if int(ede_error.code) == 22:
+                    # Auth nameserver for ip could not be reached, error codes defined in RFC 8914
+                    return [(set(), "NoAuthServersReachable:" + ede_error.text)]
+                    # returned when the resolver indicates a Lame delegation.
+        return [(set(), "NoAnswer")]
     except (dns.resolver.Timeout, dns.resolver.NoAnswer):
         return [(set(), "NoAnswer")]

boefjes/boefjes/plugins/kat_rdns/normalize.py

@@ -7,22 +7,32 @@
 from octopoes.models import Reference
 from octopoes.models.ooi.dns.records import DNSPTRRecord
 from octopoes.models.ooi.dns.zone import Hostname
+from octopoes.models.ooi.findings import Finding, KATFindingType
 from octopoes.models.ooi.network import Network
 
 
 def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]:
-    ooi = Reference.from_str(input_ooi["primary_key"])
+    reference = Reference.from_str(input_ooi["primary_key"])
+
     answers = raw.decode()
     if answers == "NXDOMAIN" or answers == "NoAnswer":
         return
-    lines = [line for line in answers.split("\n") if not line.startswith("option")]
-    for rrset in from_text("\n".join(lines[1:])).answer:
-        for rr in rrset:
-            if isinstance(rr, PTR):
-                value = rrset.to_text()
-                hostname = Hostname(
-                    name=rr.to_text().rstrip("."), network=Network(name=input_ooi["network"]["name"]).reference
-                )
-                yield hostname
-                ptr_record = DNSPTRRecord(address=ooi, hostname=hostname.reference, value=value, ttl=rrset.ttl)
-                yield ptr_record
+    if answers.startswith("NoAuthServersReachable:"):
+        ft = KATFindingType(id="KAT-LAME-DELEGATION")
+        f = Finding(finding_type=ft.reference, ooi=reference, description=answers.split(":", 1)[1])
+        yield ft
+        yield f
+    else:
+        lines = [line for line in answers.split("\n") if not line.startswith("option")]
+        for rrset in from_text("\n".join(lines[1:])).answer:
+            for rr in rrset:
+                if isinstance(rr, PTR):
+                    value = rrset.to_text()
+                    hostname = Hostname(
+                        name=rr.to_text().rstrip("."), network=Network(name=input_ooi["network"]["name"]).reference
+                    )
+                    yield hostname
+                    ptr_record = DNSPTRRecord(
+                        address=reference, hostname=hostname.reference, value=value, ttl=rrset.ttl
+                    )
+                    yield ptr_record

boefjes/boefjes/plugins/kat_webpage_analysis/check_images/normalize.py

@@ -54,5 +54,5 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]:
         yield Finding(
             finding_type=kat_ooi.reference,
             ooi=resource,
-            description="Image ended up bigger than %d Pixels, possible decompression Bomb" % image.MAX_IMAGE_PIXELS,
+            description=f"Image ended up bigger than {image.MAX_IMAGE_PIXELS} Pixels, possible decompression Bomb",
         )

boefjes/tests/integration/test_sql_repositories.py

@@ -53,7 +53,7 @@ def test_settings_storage(plugin_storage, organisation_storage, config_storage):
     with pytest.raises(ConfigNotFound):
         config_storage.delete("no organisation!", plugin_id)
 
-    assert {"TEST_SETTING": "123.9", "TEST_SETTING2": 13} == settings_storage.get_all_settings(org.id, plugin_id)
+    assert settings_storage.get_all_settings(org.id, plugin_id) == {"TEST_SETTING": "123.9", "TEST_SETTING2": 13}
     assert config_storage.get_all_settings(org.id, "wrong") == {}
     assert config_storage.get_all_settings("wrong", plugin_id) == {}
 
@@ -91,14 +91,14 @@ def test_settings_storage_values_field_limits(plugin_storage, organisation_stora
             },
         )
 
-    assert {
+    assert settings_storage.get_all_settings(org.id, plugin_id) == {
         "TEST_SETTING": 12 * "123.9",
         "TEST_SETTING2": 12000,
         "TEST_SETTING3": 30 * "b",
         "TEST_SETTING4": 30 * "b",
         "TEST_SETTING5": 10 * "b",
         "TEST_SETTING6": 123456789,
-    } == settings_storage.get_all_settings(org.id, plugin_id)
+    }
 
 
 def test_plugin_enabled_storage(organisation_storage, plugin_storage, config_storage):

boefjes/tests/katalogus/test_plugin_service.py

@@ -47,12 +47,12 @@ def test_update_by_id_bad_schema(mock_plugin_service, test_organisation):
 
 def test_get_schema(mock_plugin_service):
     schema = mock_plugin_service.schema("kat_test")
-    assert {
+    assert schema == {
         "title": "Arguments",
         "type": "object",
         "properties": {"api_key": {"title": "Api Key", "maxLength": 128, "type": "string"}},
         "required": ["api_key"],
-    } == schema
+    }
 
     schema = mock_plugin_service.schema("kat_test_normalize")
     assert schema is None

boefjes/tests/plugins/test_bodyimage.py

@@ -48,7 +48,7 @@ def test_body_image_normalizer(normalizer_runner):
     output = normalizer_runner.run(meta, get_dummy_data("cat_image")).observations[0].results
 
     assert len(output) == 1
-    assert {
+    assert output[0].dict() == {
         "object_type": "ImageMetadata",
         "primary_key": "ImageMetadata|internet|134.209.85.72|tcp|443|https|internet"
         "|mispo.es|https|internet|mispo.es|443|/",
@@ -65,7 +65,7 @@ def test_body_image_normalizer(normalizer_runner):
             "size": (600, 600),
             "width": 600,
         },
-    } == output[0].dict()
+    }
 
 
 def test_body_normalizer(normalizer_runner):

boefjes/tests/plugins/test_calvin.py

@@ -8,7 +8,7 @@ def test_parse_user_changed(normalizer_runner):
     output = normalizer_runner.run(meta, get_dummy_data("user-changed.json"))
 
     assert len(output.declarations) == 8
-    assert {
+    assert output.declarations[1].ooi.dict() == {
         "application": Application(name="organisation/env/app").reference,
         "event_id": '{"client_environment_app":"organisation/env/app","log_user_user_id":1234}-1655979300000',
         "event_title": "UC: User privilege monitoring",
@@ -31,9 +31,9 @@ def test_parse_user_changed(normalizer_runner):
         "scan_profile": None,
         "user_id": None,
         "severity": "MEDIUM",
-    } == output.declarations[1].ooi.dict()
+    }
 
-    assert {
+    assert output.declarations[-1].ooi.dict() == {
         "application": Application(name="organisation/env/app").reference,
         "event_id": '{"client_environment_app":"organisation/env/app","log_user_user_id":1234}-1658825100000',
         "event_title": "UC: User privilege monitoring",
@@ -56,15 +56,15 @@ def test_parse_user_changed(normalizer_runner):
         "scan_profile": None,
         "user_id": None,
         "severity": "MEDIUM",
-    } == output.declarations[-1].ooi.dict()
+    }
 
 
 def test_parse_admin_login_failure(normalizer_runner):
     meta = NormalizerMeta.model_validate_json(get_dummy_data("calvin-normalizer.json"))
     output = normalizer_runner.run(meta, get_dummy_data("user-login-admin-failure.json"))
 
     assert len(output.declarations) == 8
-    assert {
+    assert output.declarations[1].ooi.dict() == {
         "application": Application(name="organisation/env/app").reference,
         "event_id": '{"client_environment_app":"organisation/env/app","log_user_user_id":1234}-1659618600000',
         "event_title": "UC: Detect brute force login attempts for an admin account",
@@ -89,15 +89,15 @@ def test_parse_admin_login_failure(normalizer_runner):
         "scan_profile": None,
         "user_id": None,
         "severity": "MEDIUM",
-    } == output.declarations[1].ooi.dict()
+    }
 
 
 def test_parse_user_login_failure(normalizer_runner):
     meta = NormalizerMeta.model_validate_json(get_dummy_data("calvin-normalizer.json"))
     output = normalizer_runner.run(meta, get_dummy_data("user-login-failure.json"))
 
     assert len(output.declarations) == 8
-    assert {
+    assert output.declarations[1].ooi.dict() == {
         "application": Application(name="organisation/env/app").reference,
         "event_id": '{"client_environment_app":"organisation/env/app","log_user_user_id":1234}-1658998200000',
         "event_title": "UC: Detects attempts to guess passwords",
@@ -123,4 +123,4 @@ def test_parse_user_login_failure(normalizer_runner):
         "scan_profile": None,
         "user_id": None,
         "severity": "MEDIUM",
-    } == output.declarations[1].ooi.dict()
+    }