docs: Add example of ELK integration using filebeat (#2838)

Co-authored-by: Shahar Glazner <[email protected]>
Co-authored-by: Tal <[email protected]>

elk/README.md

@@ -0,0 +1,102 @@
+# ELK-stack integration
+
+This directory contains the configuration files and Docker services needed to run Keep with a filebeat container. Useful if you want to test integration of Keep backend logs with Logstash and Kibana.
+
+## Directory Structure
+
+```
+proxy/
+├── docker-compose-elk.yml   # Docker Compose configuration for elk integtation
+├── filebeat.yaml            # Filebeat configuration file
+├── logstash.conf            # Logstash configuration example to save keep-backend logs
+└── README.md                # This files
+```
+
+## Components
+
+The setup consists of several services:
+
+- **Filebeat**: Filebeat container to push keep-backend logs to logstash 
+- **Keep Frontend**: The Keep UI service configured to use the proxy
+- **Keep Backend**: The Keep API service
+- **Keep WebSocket**: The WebSocket server for real-time updates
+
+## Configuration
+
+### Environment Variables
+
+```env
+LOGSTASH_HOST=logstash-host
+LOGSTASH_PORT=5044
+```
+
+### Usage
+
+1. Start the elk environment:
+
+```bash
+docker compose -f docker-compose-elk.yml up
+```
+
+2. To run in detached mode:
+
+```bash
+docker compose -f docker-compose-elk.yml up -d
+```
+
+3. To stop all services:
+
+```bash
+docker compose -f docker-compose-elk.yml down
+```
+
+### Accessing Services
+
+- Keep Backend: http://localhost:8080
+- Kibana: http://localhost:5601
+
+### Kibana configuration
+
+- Goto http://localhost:5601/app/discover
+- Click "Create Data view"
+- Add any name you want
+- Add index pattern to `keep-backend-logs-*`
+- Save data view and insect logs
+
+
+## Custom Configuration
+
+### Modifying Proxy Settings
+
+To modify the Filebeat configuration:
+
+1. Edit `filebeat.yml`
+2. Restart the filebeat service:
+
+```bash
+docker compose -f docker-compose-elk.yml restart filebeat
+```
+
+### Modifying Logstash Settings
+
+To modify the Logstash configuration:
+
+1. Edit `logstash.conf`
+2. Restart the logstash service:
+
+```bash
+docker compose -f docker-compose-elk.yml restart logstash
+```
+
+## Security Considerations
+
+- This setup is intended for development environments only
+- SSL is disabled for all services for simplification
+
+## Contributing
+
+When modifying the elk setup:
+
+1. Document any changes to configuration files
+2. Test the setup of elk environments
+3. Update this README if adding new features or configurations

elk/docker-compose-elk.yml

@@ -0,0 +1,91 @@
+services:
+  keep-backend-elk:
+    extends:
+      file: ../docker-compose.common.yml
+      service: keep-backend-common
+    image: us-central1-docker.pkg.dev/keephq/keep/keep-api
+    environment:
+      - AUTH_TYPE=NO_AUTH
+    volumes:
+      - ./state:/state
+
+  keep-websocket-server:
+    extends:
+      file: ../docker-compose.common.yml
+      service: keep-websocket-server-common
+
+  elastic:
+   image: docker.elastic.co/elasticsearch/elasticsearch:8.17.0
+   labels:
+     co.elastic.logs/module: elasticsearch
+   volumes:
+     - elastic_data:/usr/share/elasticsearch/data
+   ports:
+     - "9200:9200"
+   environment:
+     - node.name=elastic
+     - cluster.name=keep-elk
+     - discovery.type=single-node
+     - ELASTIC_PASSWORD=elastic
+     - bootstrap.memory_lock=true
+     - xpack.security.enabled=false
+     - xpack.security.enrollment.enabled=false
+     - xpack.security.transport.ssl.enabled=false
+     - xpack.license.self_generated.type=basic
+
+  kibana:
+     depends_on:
+       - elastic
+     image: docker.elastic.co/kibana/kibana:8.17.0
+     labels:
+       co.elastic.logs/module: kibana
+     volumes:
+       - kibana_data:/usr/share/kibana/data
+     ports:
+       - 5601:5601
+     environment:
+       - SERVERNAME=kibana
+       - ELASTICSEARCH_HOSTS=http://elastic:9200
+       - ELASTICSEARCH_USERNAME=kibana_system
+       - ELASTICSEARCH_PASSWORD=kibana
+       - XPACK_APM_SERVICEMAPENABLED="true"
+       - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${ENCRYPTION_KEY}
+
+  filebeat:
+    image: docker.elastic.co/beats/filebeat:8.17.0
+    container_name: filebeat
+    user: root
+    volumes:
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
+    environment:
+      - LOGSTASH_HOST=logstash01
+    command: [ "--strict.perms=false" ]  # Disable strict permissions to avoid permission errors
+
+  logstash:
+    depends_on:
+      - elastic
+      - kibana
+    image: docker.elastic.co/logstash/logstash:8.17.0
+    labels:
+      co.elastic.logs/module: logstash
+    user: root
+    ports:
+      - "5001:5000"
+      - "5044:5044"
+      - "9600:9600"
+    volumes:
+      - logstash_data:/usr/share/logstash/data
+      - "./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro"
+    environment:
+      - xpack.monitoring.enabled=false
+      - ELASTIC_USER=elastic
+      - ELASTIC_PASSWORD=elastic
+      - ELASTIC_HOSTS=http://elastic:9200
+
+
+volumes:
+  elastic_data:
+  kibana_data:
+  logstash_data:

elk/filebeat.yml

@@ -0,0 +1,22 @@
+filebeat.inputs:
+  - type: container
+    paths:
+      - /var/lib/docker/containers/*/*.log
+    stream: stdout  # Only capture stdout
+    json.keys_under_root: true  # Parse JSON-formatted logs automatically
+    json.add_error_key: true  # Add error field if JSON parsing fails
+    processors:
+      - decode_json_fields:
+          fields: [ "message" ]  # Try to decode the `message` field as JSON
+          target: ""           # Merge decoded fields at the root level
+          overwrite_keys: true # Overwrite existing keys if present
+      - add_docker_metadata:  # Enrich logs with Docker metadata
+          host: "unix:///var/run/docker.sock"
+      - drop_event:
+          when.not.contains.container.labels:
+            com_docker_compose_service: "keep-backend-elk"
+
+output.logstash:
+  hosts: ["logstash:5044"]  # Replace with your Logstash host and port
+
+logging.level: info  # Set Filebeat logging level

elk/logstash.conf

@@ -0,0 +1,19 @@
+input {
+  beats {
+    port => 5044  # Match the port used in Filebeat configuration
+  }
+}
+
+filter {
+  json {
+    source => "message"
+  }
+}
+
+output {
+  stdout { codec => rubydebug }  # For debugging
+  elasticsearch {
+    hosts => ["http://elastic:9200"]
+    index => "keep-backend-logs-%{+YYYY.MM.dd}"
+  }
+}