feat(api): support for graylog v4.x (#2452)

Signed-off-by: 35C4n0r <[email protected]>
Co-authored-by: Shahar Glazner <[email protected]>

Author: 35C4n0r

docs/providers/documentation/graylog-provider.mdx

@@ -46,11 +46,13 @@ Example:
 ```
 - You can modify this to fetch either alerts, events or both.
 
----
-
-**Note**: Ensure that the product of `page` and `per_page` does not exceed 10,000.
+<Note>
+Ensure that the product of `page` and `per_page` does not exceed 10,000.
+</Note>
 
----
+<Note>
+The notification URL for Graylog v4.x has the api_key as a query param, this is the default behaviour.
+</Note>
 
 ## Useful Links
 

keep/providers/graylog_provider/README.md

@@ -1,6 +1,6 @@
 # Instructions for a quick setup
 
-## Setting up Graylog
+## Setting up Graylog (v6)
 
 ### Installation
 

keep/providers/graylog_provider/docker-compose-v4.yml

@@ -0,0 +1,55 @@
+version: '3'
+services:
+  # MongoDB: https://hub.docker.com/_/mongo/
+  mongo:
+    image: mongo:4.2
+    networks:
+      - graylog
+
+  # Elasticsearch: https://www.elastic.co/guide/en/elasticsearch/reference/7.10/docker.html
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2
+    environment:
+      - http.host=0.0.0.0
+      - transport.host=localhost
+      - network.host=0.0.0.0
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    deploy:
+      resources:
+        limits:
+          memory: 1g
+    networks:
+      - graylog
+    ports:
+      - "9200:9200"
+      - "9300:9300"
+
+  # Graylog: https://hub.docker.com/r/graylog/graylog/
+  graylog:
+    image: graylog/graylog:4.0
+    environment:
+      - GRAYLOG_PASSWORD_SECRET=somepasswordpepper
+      - GRAYLOG_ROOT_PASSWORD_SHA2=8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918
+      - GRAYLOG_HTTP_EXTERNAL_URI=http://127.0.0.1:9000/
+      - GRAYLOG_ELASTICSEARCH_HOSTS=http://elasticsearch:9200
+    entrypoint: /usr/bin/tini -- wait-for-it elasticsearch:9200 -t 60 -- /docker-entrypoint.sh
+    networks:
+      - graylog
+    restart: always
+    depends_on:
+      - mongo
+      - elasticsearch
+    ports:
+      - "9000:9000"
+      - "1514:1514"
+      - "1514:1514/udp"
+      - "12201:12201"
+      - "12201:12201/udp"
+
+networks:
+  graylog:
+    driver: bridge

keep/providers/graylog_provider/graylog_provider.py

@@ -1,6 +1,7 @@
 """
 Graylog Provider is a class that allows to install webhooks in Graylog.
 """
+# Documentation for older versions of graylog: https://github.com/Graylog2/documentation
 
 import dataclasses
 import math
@@ -67,6 +68,8 @@ class GraylogProvider(BaseProvider):
 2. Click "Create Notification".
 3. In the New Notification form, configure:
 
+**Note**: For Graylog v4.x please set the **URL** to `{keep_webhook_api_url}?api_key={api_key}`.
+
 - **Display Name**: keep-graylog-webhook-integration
 - **Title**: keep-graylog-webhook-integration
 - **Notification Type**: Custom HTTP Notification
@@ -105,6 +108,7 @@ def __init__(
     ):
         super().__init__(context_manager, provider_id, config)
         self._host = None
+        self.is_v4 = self.__get_graylog_version().startswith("4")
 
     def dispose(self):
         """
@@ -224,6 +228,22 @@ def validate_scopes(self) -> dict[str, bool | str]:
             "authorized": authorized,
         }
 
+    def __get_graylog_version(self) -> str:
+        self.logger.info("Getting graylog version info")
+        try:
+            version_response = requests.get(
+                url=self.__get_url(),
+                headers=self._headers
+            )
+            if version_response.status_code != 200:
+                raise Exception(version_response.text)
+            version = version_response.json()["version"].strip()
+            self.logger.info(f"We are working with Graylog version: {version}")
+            return version
+        except Exception as e:
+            self.logger.error("Error while getting Graylog Version", extra={"exception": str(e)})
+
+
     def __get_url_whitelist(self):
         try:
             self.logger.info("Fetching URL Whitelist")
@@ -410,6 +430,16 @@ def setup_webhook(
         self, tenant_id: str, keep_api_url: str, api_key: str, setup_alerts: bool = True
     ):
         self.logger.info("Setting up webhook in Graylog")
+
+        # Extracting provider_id from the keep_api_url
+        parsed_url = urlparse(keep_api_url)
+        query_params = parsed_url.query
+        provider_id = query_params.split("provider_id=")[-1]
+        notification_name = f"Keep-{provider_id}"
+
+        if self.is_v4:
+            keep_api_url = f"{keep_api_url}&api_key={api_key}"
+
         try:
             event_definitions = []
             events_1 = self.__get_events(page=1, per_page=100)
@@ -422,12 +452,6 @@ def setup_webhook(
                     self.__get_events(page=page, per_page=100)["event_definitions"]
                 )
 
-            # Extracting provider_id from the keep_api_url
-            parsed_url = urlparse(keep_api_url)
-            query_params = parsed_url.query
-            provider_id = query_params.split("provider_id=")[-1]
-            notification_name = f"Keep-{provider_id}"
-
             # Whitelist URL
             url_whitelist = self.__get_url_whitelist()
             url_found = False
@@ -452,17 +476,27 @@ def setup_webhook(
             notification = self.__get_notification(
                 page=1, per_page=1, notification_name=notification_name
             )
+
+            existing_notification_id = None
+
             if int(notification["count"]) > 0:
                 self.logger.info("Notification already exists, deleting it")
+
+                # We need to clean up the previously installed notification
+                existing_notification_id = notification["notifications"][0]["id"]
+
                 self.__delete_notification(
-                    notification_id=notification["notifications"][0]["id"]
+                    notification_id=existing_notification_id
                 )
 
             self.logger.info("Creating new notification")
-            notification_body = {
-                "title": notification_name,
-                "description": "Hello, this Notification is created by Keep, please do not change the title.",
-                "config": {
+            if self.is_v4:
+                config = {
+                    "type": "http-notification-v1",
+                    "url": keep_api_url
+                }
+            else:
+                config = {
                     "type": "http-notification-v2",
                     "basic_auth": None,
                     "api_key_as_header": False,
@@ -475,17 +509,28 @@ def setup_webhook(
                     "content_type": "JSON",
                     "headers": f"X-API-KEY:{api_key}",
                     "body_template": "",
-                },
+                }
+            notification_body = {
+                "title": notification_name,
+                "description": "Hello, this Notification is created by Keep, please do not change the title.",
+                "config": config,
             }
             new_notification = self.__create_notification(
                 notification_name=notification_name, notification_body=notification_body
             )
 
             for event_definition in event_definitions:
-                if event_definition["_scope"] == "SYSTEM_NOTIFICATION_EVENT":
+                if not self.is_v4 and event_definition["_scope"] == "SYSTEM_NOTIFICATION_EVENT":
                     self.logger.info("Skipping SYSTEM_NOTIFICATION_EVENT")
                     continue
                 self.logger.info(f"Updating event with ID: {event_definition['id']}")
+
+                # Attempting to clean up the deleted notification from the event, it is not handled well in Graylog v4.
+                for ind, notification in enumerate(event_definition["notifications"]):
+                    if notification["notification_id"] == existing_notification_id:
+                        event_definition["notifications"].pop(ind)
+                        break
+
                 event_definition["notifications"].append(
                     {"notification_id": new_notification["id"]}
                 )