From 8eb620da575b4fb31a4fca4a61393fd48cca3b77 Mon Sep 17 00:00:00 2001 From: Vladimir Filonov Date: Wed, 18 Dec 2024 17:16:54 +0400 Subject: [PATCH] docs: Add example of ELK integration using filebeat (#2838) Co-authored-by: Shahar Glazner Co-authored-by: Tal --- elk/README.md | 102 +++++++++++++++++++++++++++++++++++++ elk/docker-compose-elk.yml | 91 +++++++++++++++++++++++++++++++++ elk/filebeat.yml | 22 ++++++++ elk/logstash.conf | 19 +++++++ 4 files changed, 234 insertions(+) create mode 100644 elk/README.md create mode 100644 elk/docker-compose-elk.yml create mode 100644 elk/filebeat.yml create mode 100644 elk/logstash.conf diff --git a/elk/README.md b/elk/README.md new file mode 100644 index 000000000..395dba0e3 --- /dev/null +++ b/elk/README.md @@ -0,0 +1,102 @@ +# ELK-stack integration + +This directory contains the configuration files and Docker services needed to run Keep with a filebeat container. Useful if you want to test integration of Keep backend logs with Logstash and Kibana. + +## Directory Structure + +``` +proxy/ +├── docker-compose-elk.yml # Docker Compose configuration for elk integtation +├── filebeat.yaml # Filebeat configuration file +├── logstash.conf # Logstash configuration example to save keep-backend logs +└── README.md # This files +``` + +## Components + +The setup consists of several services: + +- **Filebeat**: Filebeat container to push keep-backend logs to logstash +- **Keep Frontend**: The Keep UI service configured to use the proxy +- **Keep Backend**: The Keep API service +- **Keep WebSocket**: The WebSocket server for real-time updates + +## Configuration + +### Environment Variables + +```env +LOGSTASH_HOST=logstash-host +LOGSTASH_PORT=5044 +``` + +### Usage + +1. Start the elk environment: + +```bash +docker compose -f docker-compose-elk.yml up +``` + +2. To run in detached mode: + +```bash +docker compose -f docker-compose-elk.yml up -d +``` + +3. To stop all services: + +```bash +docker compose -f docker-compose-elk.yml down +``` + +### Accessing Services + +- Keep Backend: http://localhost:8080 +- Kibana: http://localhost:5601 + +### Kibana configuration + +- Goto http://localhost:5601/app/discover +- Click "Create Data view" +- Add any name you want +- Add index pattern to `keep-backend-logs-*` +- Save data view and insect logs + + +## Custom Configuration + +### Modifying Proxy Settings + +To modify the Filebeat configuration: + +1. Edit `filebeat.yml` +2. Restart the filebeat service: + +```bash +docker compose -f docker-compose-elk.yml restart filebeat +``` + +### Modifying Logstash Settings + +To modify the Logstash configuration: + +1. Edit `logstash.conf` +2. Restart the logstash service: + +```bash +docker compose -f docker-compose-elk.yml restart logstash +``` + +## Security Considerations + +- This setup is intended for development environments only +- SSL is disabled for all services for simplification + +## Contributing + +When modifying the elk setup: + +1. Document any changes to configuration files +2. Test the setup of elk environments +3. Update this README if adding new features or configurations diff --git a/elk/docker-compose-elk.yml b/elk/docker-compose-elk.yml new file mode 100644 index 000000000..6c4b256c2 --- /dev/null +++ b/elk/docker-compose-elk.yml @@ -0,0 +1,91 @@ +services: + keep-backend-elk: + extends: + file: ../docker-compose.common.yml + service: keep-backend-common + image: us-central1-docker.pkg.dev/keephq/keep/keep-api + environment: + - AUTH_TYPE=NO_AUTH + volumes: + - ./state:/state + + keep-websocket-server: + extends: + file: ../docker-compose.common.yml + service: keep-websocket-server-common + + elastic: + image: docker.elastic.co/elasticsearch/elasticsearch:8.17.0 + labels: + co.elastic.logs/module: elasticsearch + volumes: + - elastic_data:/usr/share/elasticsearch/data + ports: + - "9200:9200" + environment: + - node.name=elastic + - cluster.name=keep-elk + - discovery.type=single-node + - ELASTIC_PASSWORD=elastic + - bootstrap.memory_lock=true + - xpack.security.enabled=false + - xpack.security.enrollment.enabled=false + - xpack.security.transport.ssl.enabled=false + - xpack.license.self_generated.type=basic + + kibana: + depends_on: + - elastic + image: docker.elastic.co/kibana/kibana:8.17.0 + labels: + co.elastic.logs/module: kibana + volumes: + - kibana_data:/usr/share/kibana/data + ports: + - 5601:5601 + environment: + - SERVERNAME=kibana + - ELASTICSEARCH_HOSTS=http://elastic:9200 + - ELASTICSEARCH_USERNAME=kibana_system + - ELASTICSEARCH_PASSWORD=kibana + - XPACK_APM_SERVICEMAPENABLED="true" + - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${ENCRYPTION_KEY} + + filebeat: + image: docker.elastic.co/beats/filebeat:8.17.0 + container_name: filebeat + user: root + volumes: + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro + environment: + - LOGSTASH_HOST=logstash01 + command: [ "--strict.perms=false" ] # Disable strict permissions to avoid permission errors + + logstash: + depends_on: + - elastic + - kibana + image: docker.elastic.co/logstash/logstash:8.17.0 + labels: + co.elastic.logs/module: logstash + user: root + ports: + - "5001:5000" + - "5044:5044" + - "9600:9600" + volumes: + - logstash_data:/usr/share/logstash/data + - "./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro" + environment: + - xpack.monitoring.enabled=false + - ELASTIC_USER=elastic + - ELASTIC_PASSWORD=elastic + - ELASTIC_HOSTS=http://elastic:9200 + + +volumes: + elastic_data: + kibana_data: + logstash_data: diff --git a/elk/filebeat.yml b/elk/filebeat.yml new file mode 100644 index 000000000..205c08432 --- /dev/null +++ b/elk/filebeat.yml @@ -0,0 +1,22 @@ +filebeat.inputs: + - type: container + paths: + - /var/lib/docker/containers/*/*.log + stream: stdout # Only capture stdout + json.keys_under_root: true # Parse JSON-formatted logs automatically + json.add_error_key: true # Add error field if JSON parsing fails + processors: + - decode_json_fields: + fields: [ "message" ] # Try to decode the `message` field as JSON + target: "" # Merge decoded fields at the root level + overwrite_keys: true # Overwrite existing keys if present + - add_docker_metadata: # Enrich logs with Docker metadata + host: "unix:///var/run/docker.sock" + - drop_event: + when.not.contains.container.labels: + com_docker_compose_service: "keep-backend-elk" + +output.logstash: + hosts: ["logstash:5044"] # Replace with your Logstash host and port + +logging.level: info # Set Filebeat logging level diff --git a/elk/logstash.conf b/elk/logstash.conf new file mode 100644 index 000000000..6f5aeb239 --- /dev/null +++ b/elk/logstash.conf @@ -0,0 +1,19 @@ +input { + beats { + port => 5044 # Match the port used in Filebeat configuration + } +} + +filter { + json { + source => "message" + } +} + +output { + stdout { codec => rubydebug } # For debugging + elasticsearch { + hosts => ["http://elastic:9200"] + index => "keep-backend-logs-%{+YYYY.MM.dd}" + } +}