diff --git a/kafka/connect-plugins/.gitignore b/kafka/connect-plugins/.gitignore new file mode 100644 index 00000000..662a0c0a --- /dev/null +++ b/kafka/connect-plugins/.gitignore @@ -0,0 +1,2 @@ +confluentinc* +*.zip \ No newline at end of file diff --git a/kafka/connect-plugins/.gitkeep b/kafka/connect-plugins/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/kafka/docker-compose.yml b/kafka/docker-compose.yml new file mode 100644 index 00000000..0d4b5a44 --- /dev/null +++ b/kafka/docker-compose.yml @@ -0,0 +1,113 @@ +# Copyright 2023 The original authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +version: '3.8' +services: + kafka: + image: "confluentinc/cp-kafka:7.5.0" + ports: + - "9092:9092" + container_name: kafka + environment: + KAFKA_NODE_ID: 101 + # random cluster ID used for formatting LOG_DIR for KRaft + CLUSTER_ID: 'xtzWWN4bTjitpL3kfd9s5g' + KAFKA_CONTROLLER_QUORUM_VOTERS: '101@kafka:29093' + KAFKA_PROCESS_ROLES: 'broker,controller' + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092' + KAFKA_LISTENERS: 'PLAINTEXT://kafka:29092,PLAINTEXT_HOST://0.0.0.0:9092,CONTROLLER://kafka:29093' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka:29092 + CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1 + CONFLUENT_METRICS_ENABLE: 'false' + KAFKA_AUTHORIZER_CLASS_NAME: org.apache.kafka.metadata.authorizer.StandardAuthorizer + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true' + KAFKA_SUPER_USERS: 'User:anonymous' + networks: + - kafka-platform + + schema-registry: + image: confluentinc/cp-schema-registry:7.5.0 + hostname: schema-registry + container_name: schema-registry + ports: + - "8081:8081" + environment: + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka:29092 + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 + depends_on: + - kafka + networks: + - kafka-platform + akhq: + image: tchiotludo/akhq + hostname: akhq + container_name: akhq + depends_on: + - kafka + ports: + - "8087:8080" + environment: + AKHQ_CONFIGURATION: | + akhq: + connections: + docker-kafka-server: + properties: + bootstrap.servers: "kafka:29092" + schema-registry: + url: "http://schema-registry:8081" + connect: + - name: "connect" + url: "http://connect:8083" + networks: + - kafka-platform + + connect: + image: confluentinc/cp-kafka-connect:7.5.0 + container_name: connect + depends_on: + - kafka + volumes: + - ./connect-plugins:/connect-plugins + ports: + - "8083:8083" + - "8000:8000" + environment: + CONNECT_BOOTSTRAP_SERVERS: 'kafka:29092' + CONNECT_REST_ADVERTISED_HOST_NAME: connect + CONNECT_REST_PORT: 8083 + CONNECT_GROUP_ID: kafka-connect + CONNECT_CONFIG_STORAGE_TOPIC: _connect-configs + CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 + CONNECT_OFFSET_STORAGE_TOPIC: _connect-offsets + CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_STATUS_STORAGE_TOPIC: _connect-status + CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter + CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.storage.StringConverter + CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_PLUGIN_PATH: "/usr/local/share/kafka/plugins,/usr/share/filestream-connectors,/connect-plugins" + networks: + - kafka-platform + +networks: + kafka-platform: + driver: bridge \ No newline at end of file diff --git a/kafka/plugin-config.json b/kafka/plugin-config.json new file mode 100644 index 00000000..6c385d47 --- /dev/null +++ b/kafka/plugin-config.json @@ -0,0 +1,17 @@ +{ + "name": "elasticsearch-sink", + "config": { + "connector.class": "io.confluent.connect.elasticsearch.ElasticsearchSinkConnector", + "tasks.max": "1", + "topics": "example-topic", + "key.ignore": "true", + "schema.ignore": "true", + "connection.url": "https://logingest.lightstep.com:443", + "connection.username": "lightstep", + "connection.password": "your-access-token", + "type.name": "_doc", + "name": "elasticsearch-sink", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "value.converter.schemas.enable": "false" + } +} \ No newline at end of file diff --git a/kafka/readme.md b/kafka/readme.md new file mode 100644 index 00000000..61f11f5c --- /dev/null +++ b/kafka/readme.md @@ -0,0 +1,28 @@ +### Stream logs from Kafka + +Streams Kafka topics to ServiceNow Cloud Observability logs + +#### Run + +1) Download and unzip https://www.confluent.io/hub/confluentinc/kafka-connect-elasticsearch into `connect-plugins/` + +2) Run Kafka + +``` + docker-compose up + + # wait a few minutes... +``` + +3) Configure connector + +``` + curl -X POST http://localhost:8083/connectors -H 'Content-Type: application/json' -d @plugin-config.json +``` + +4) Send some data + +``` +docker exec -i kafka bash -c "echo '{\"userId\": \"1\", \"action\": \"login\"}' | /usr/bin/kafka-console-producer --broker-list kafka:9092 --topic example-topic" + +``` \ No newline at end of file