Add export http boefje (#3901)

Co-authored-by: ammar92 <[email protected]>
Co-authored-by: Jan Klopper <[email protected]>
Co-authored-by: Jeroen Dekkers <[email protected]>

boefjes/Makefile

@@ -40,6 +40,7 @@ images:  # Build the images for the containerized boefjes
 	# docker build -f images/base.Dockerfile -t ghcr.io/minvws/openkat/dns-records --build-arg BOEFJE_PATH=./boefjes/plugins/kat_dns .
 	docker build -f ./boefjes/plugins/kat_dnssec/boefje.Dockerfile -t ghcr.io/minvws/openkat/dns-sec:latest .
 	docker build -f ./boefjes/plugins/kat_nmap_tcp/boefje.Dockerfile -t ghcr.io/minvws/openkat/nmap:latest  .
+	docker build -f ./boefjes/plugins/kat_export_http/boefje.Dockerfile -t ghcr.io/minvws/openkat/export-http:latest .
 
 
 ##

boefjes/boefjes/plugins/kat_export_http/boefje.Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+RUN apt-get update && pip install httpx && pip install requests
+
+ARG BOEFJE_PATH=./boefjes/plugins/kat_export_http
+ENV PYTHONPATH=/app:$BOEFJE_PATH
+
+COPY ./images/oci_adapter.py ./
+COPY $BOEFJE_PATH $BOEFJE_PATH
+
+ENTRYPOINT ["/usr/local/bin/python", "-m", "oci_adapter"]

boefjes/boefjes/plugins/kat_export_http/boefje.json

@@ -0,0 +1,12 @@
+{
+  "id": "export-to-http-api",
+  "name": "Export To HTTP API",
+  "description": "Exports the Input OOI to the configured HTTP api, configure by copying this Boefje, and provinding Input OOIs of your choice. Limit Exported Objects by selecting an appropriate Scan Level.",
+  "consumes": [],
+  "scan_level": 4,
+  "oci_image": "ghcr.io/minvws/openkat/export-http:latest",
+  "run_on": [
+    "create",
+    "update"
+  ]
+}

boefjes/boefjes/plugins/kat_export_http/description.md

@@ -0,0 +1,19 @@
+# Export to HTTP api
+
+This Boefje can be confgred to un on selected OOI's with a selected clearance level, it then exports these OOI's to the configured Http endpoint.
+Configure by copying this Boefje, and provinding Input OOI's of your choice. Limit Exported Objects by selecting an appropriate Scan Level.
+
+### Input OOIs
+
+Select your own desired input's by creating a variant. By doing so the user can also select the Scan level, this limits the exported OOI's to only those who have received a high enough scan level, possibly ignoring objects outside of the scope of your organization.
+
+### Configurables """
+
+EXPORT_HTTP_ENDPOINT an http(s) url possibly containing Basic Auth credentials
+EXPORT_REQUEST_HEADERS, an enter separated list of headers to be send with the request. Useful for injecting api-tokens.
+EXPORT_HTTP_VERB, GET, POST, DEL, PUT, PATCH, defaults to POST
+EXPORT_REQUEST_PARAMETER, optional named url/post parameter. If none is given the data will be posted as json body.
+EXPORT_HTTP_ORGANIZATION_IDENTIFIER, optional overwritable organization name, defaults to the organization identiefier to which the OOI belongs. Will be added to the params / post data as 'organizaton'.
+TIMEOUT defaults to 15
+USERAGENT defaults to OpenKAT
+REQUESTS_CA_BUNDLE optional local CA bundle file path when dealing with internal / self-signed servers.

boefjes/boefjes/plugins/kat_export_http/main.py

@@ -0,0 +1,39 @@
+"""Boefje script for exporting OOI's to an external http api"""
+
+import json
+from os import getenv
+from typing import Any
+
+import requests
+
+
+def run(boefje_meta: dict) -> list[tuple[set, bytes | str]]:
+    input_ooi = boefje_meta["arguments"]["input"]
+
+    timeout = getenv("TIMEOUT", default=15)
+    endpoint_uri = getenv("EXPORT_HTTP_ENDPOINT", "")
+    request_headers = getenv("EXPORT_REQUEST_HEADERS", "")
+    request_parameter = getenv("EXPORT_REQUEST_PARAMETER", "")
+    request_verb = getenv("EXPORT_HTTP_VERB", default="POST").lower()
+    useragent = getenv("USERAGENT", default="OpenKAT")
+    organization = getenv("", boefje_meta["organization"])
+
+    headers = {"User-Agent": useragent}
+
+    request_header_list = request_headers.split("\n")
+    headers.update({header.split(":")[0]: header.split(":")[1] for header in request_header_list if ":" in header})
+
+    kwargs: dict[str, Any] = {"headers": headers, "timeout": float(timeout)}
+
+    if request_verb == "get":
+        kwargs.update({"params": {request_parameter: json.dumps(input_ooi), "organization": organization}})
+    else:
+        if request_parameter:
+            kwargs.update({"data": {request_parameter: json.dumps(input_ooi), "organization": organization}})
+        else:
+            kwargs.update({"json": input_ooi})
+
+    response = requests.request(request_verb, endpoint_uri, **kwargs)  # noqa: S113
+    response.raise_for_status()
+
+    return [(set(), response.content)]

boefjes/boefjes/plugins/kat_export_http/schema.json

@@ -0,0 +1,68 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Arguments",
+  "type": "object",
+  "properties": {
+    "EXPORT_HTTP_ENDPOINT": {
+      "title": "EXPORT_HTTP_ENDPOINT",
+      "type": "string",
+      "maxLength": 1024,
+      "description": "URL to call."
+    },
+    "TIMEOUT": {
+      "title": "TIMEOUT",
+      "type": "integer",
+      "default": 15,
+      "description": "Optional HTTP timeout in seconds."
+    },
+    "USERAGENT": {
+      "title": "USERAGENT",
+      "type": "string",
+      "default": "OpenKAT",
+      "maxLength": 1024,
+      "description": "Optional HTTP user-agent."
+    },
+    "EXPORT_REQUEST_HEADERS": {
+      "title": "EXPORT_REQUEST_HEADERS",
+      "type": "string",
+      "default": "OpenKAT",
+      "maxLength": 1024,
+      "description": "Optional extra HTTP request headers, newline-separated pairs of header:value. Useful for API tokens, etc."
+    },
+    "EXPORT_HTTP_VERB": {
+      "title": "EXPORT_HTTP_VERB",
+      "type": "string",
+      "enum": [
+        "GET",
+        "POST",
+        "DELETE",
+        "PUT",
+        "PATCH"
+      ],
+      "default": "POST",
+      "description": "Optional HTTP verb."
+    },
+    "EXPORT_REQUEST_PARAMETER": {
+      "title": "EXPORT_REQUEST_PARAMETER",
+      "type": "string",
+      "default": "ooi",
+      "maxLength": 1024,
+      "description": "Optional URL parameter to use when sending data, required for GET. If not given, other HTTP verbs will send OOI as a JSON body."
+    },
+    "EXPORT_HTTP_ORGANIZATION_IDENTIFIER": {
+      "title": "EXPORT_HTTP_ORGANIZATION_IDENTIFIER",
+      "type": "string",
+      "maxLength": 1024,
+      "description": "Optional organization identifier, defaults to the organization identifier present in the OOI."
+    },
+    "REQUESTS_CA_BUNDLE": {
+      "title": "REQUESTS_CA_BUNDLE",
+      "type": "string",
+      "maxLength": 1024,
+      "description": "Optional local file path for a CA bundle."
+    }
+  },
+  "required": [
+    "EXPORT_HTTP_ENDPOINT"
+  ]
+}

boefjes/tests/integration/test_api.py

@@ -17,11 +17,11 @@ def test_get_local_plugin(test_client, organisation):
 
 def test_filter_plugins(test_client, organisation):
     response = test_client.get(f"/v1/organisations/{organisation.id}/plugins/")
-    assert len(response.json()) == 101
+    assert len(response.json()) > 10
     response = test_client.get(f"/v1/organisations/{organisation.id}/plugins?plugin_type=boefje")
-    assert len(response.json()) == 45
+    assert len(response.json()) > 10
     response = test_client.get(f"/v1/organisations/{organisation.id}/plugins?state=true")
-    assert len(response.json()) == 62
+    assert len(response.json()) > 10
     response = test_client.get(f"/v1/organisations/{organisation.id}/plugins?limit=10")
     assert len(response.json()) == 10
 
@@ -63,7 +63,7 @@ def test_add_boefje(test_client, organisation):
     assert response.status_code == 422
 
     response = test_client.get(f"/v1/organisations/{organisation.id}/plugins/?plugin_type=boefje")
-    assert len(response.json()) == 46
+    assert len(response.json()) > 10
 
     boefje_dict = boefje.model_dump()
     boefje_dict["consumes"] = list(boefje_dict["consumes"])