From b04d3a5449f5968f76ff0407782b33fc07b3ef6d Mon Sep 17 00:00:00 2001 From: Tung Bui Date: Sun, 1 Sep 2024 15:15:19 +0700 Subject: [PATCH 1/5] Init Kafka --- topics/kafka/README.md | 53 ++++++++++++++++++++++++++++++++++++ topics/kafka/basic/README.md | 0 2 files changed, 53 insertions(+) create mode 100644 topics/kafka/README.md create mode 100644 topics/kafka/basic/README.md diff --git a/topics/kafka/README.md b/topics/kafka/README.md new file mode 100644 index 0000000..af9cca8 --- /dev/null +++ b/topics/kafka/README.md @@ -0,0 +1,53 @@ +## 1. What is Apache Kafka? + +- [Introduction to Apache Kafka](https://kafka.apache.org/documentation/) +- [Youtube - What is Apache Kafka?](https://youtu.be/vHbvbwSEYGo?si=SbouSV-0NZzigsXV) + +### Overview + +- Apache Kafka is a distributed event streaming platform capable of handling trillions of events a day. It is used for building real-time data pipelines and streaming applications. Kafka is horizontally scalable, fault-tolerant, and fast. + +- Kafka allows you to publish, subscribe to, store, and process streams of records in real-time. It is often used in scenarios where data needs to be processed or moved between systems efficiently, such as log aggregation, real-time analytics, or as a backbone for microservices. + +### Kafka Architecture + +- [Understanding Kafka Architecture](https://kafka.apache.org/10/documentation/streams/architecture) + +### Official Website Documentation for Apache Kafka + +- [Apache Kafka Documentation](https://kafka.apache.org/documentation/) + +## 2. Prerequisites + +- Basic Linux command line skills +- Understanding of distributed systems and event streaming concepts + +## 3. Installation + +### How to install Apache Kafka? + +- [Kafka Quickstart Guide](https://kafka.apache.org/quickstart) + +## 4. Basics of Apache Kafka + +### Getting Started with Kafka + +- [Kafka 101: Getting Started with Kafka](https://kafka.apache.org/quickstart) + +### Kafka Hello World + +- See: [basic](./basic/) + +## 5. Beyond the Basics + +- TODO + +## 6. More... + +### Kafka cheatsheet + +- https://www.redpanda.com/guides/kafka-tutorial-kafka-cheat-sheet + +### Recommended Books + +- None diff --git a/topics/kafka/basic/README.md b/topics/kafka/basic/README.md new file mode 100644 index 0000000..e69de29 From 3457b9ee50c255a89cbab82cacc24996a427b369 Mon Sep 17 00:00:00 2001 From: Tung Bui Date: Sun, 1 Sep 2024 15:19:49 +0700 Subject: [PATCH 2/5] Init Kafka - basics --- topics/kafka/basic/README.md | 56 +++++++++++++++++++++++++++ topics/kafka/basic/docker-compose.yml | 22 +++++++++++ 2 files changed, 78 insertions(+) create mode 100644 topics/kafka/basic/docker-compose.yml diff --git a/topics/kafka/basic/README.md b/topics/kafka/basic/README.md index e69de29..43155ea 100644 --- a/topics/kafka/basic/README.md +++ b/topics/kafka/basic/README.md @@ -0,0 +1,56 @@ +## Kafka Basics + +Here's a basic "Hello World" example for Apache Kafka using Docker and Docker Compose. This will set up a Kafka broker and a Zookeeper instance, allowing you to produce and consume messages. + +### 1. Create a `docker-compose.yml` File + +Create a [docker-compose.yml](./docker-compose.yml) file that defines the services for Zookeeper and Kafka. + +### 2. Start Kafka and Zookeeper + +Run the following command to start the Kafka and Zookeeper containers: + +```bash +cd devops-basics/topics/kafka/basic +docker-compose up -d +``` + +This command will start Zookeeper and Kafka in the background. + +### 3. Create a Kafka Topic + +Once the containers are running, create a Kafka topic named `helloworld`. + +```bash +docker exec kafka kafka-topics.sh --create --topic helloworld --bootstrap-server localhost:9092 --partitions 1 --replication-factor 1 +``` + +### 4. Produce Messages to the Kafka Topic + +To send a message to the `helloworld` topic: + +```bash +docker exec -it kafka kafka-console-producer.sh --topic helloworld --bootstrap-server localhost:9092 +``` + +Type a message (e.g., `Hello, Kafka!`) and press Enter. This sends the message to the Kafka topic. + +### 5. Consume Messages from the Kafka Topic + +To read the message from the `helloworld` topic: + +```bash +docker exec -it kafka kafka-console-consumer.sh --topic helloworld --bootstrap-server localhost:9092 --from-beginning +``` + +You should see the message you produced earlier. + +### 6. Cleanup + +To stop and remove the Kafka and Zookeeper containers, run: + +```bash +docker-compose down +``` + +This basic setup allows you to get hands-on experience with Kafka using Docker and Docker Compose. You can extend this setup to explore more advanced Kafka features. diff --git a/topics/kafka/basic/docker-compose.yml b/topics/kafka/basic/docker-compose.yml new file mode 100644 index 0000000..6ad2c2a --- /dev/null +++ b/topics/kafka/basic/docker-compose.yml @@ -0,0 +1,22 @@ +version: '3.8' +services: + zookeeper: + image: bitnami/zookeeper:latest + container_name: zookeeper + environment: + - ZOO_ENABLE_AUTH=no + ports: + - '2181:2181' + + kafka: + image: bitnami/kafka:latest + container_name: kafka + ports: + - '9092:9092' + environment: + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_LISTENERS=PLAINTEXT://:9092 + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 + depends_on: + - zookeeper From c9cad4d6c845759ce037b550afcef70765e4fa64 Mon Sep 17 00:00:00 2001 From: Tung Bui Date: Sun, 1 Sep 2024 15:23:31 +0700 Subject: [PATCH 3/5] Add Kafka to the table --- README.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index edbc862..797d9e4 100755 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ This is the **first** of my DevOps trio repositories: [devops-basics](https://gi DevOps combines development (Dev) and operations (Ops) to increase the efficiency, speed, and security of software development and delivery compared to traditional processes. A more nimble software development lifecycle results in a competitive advantage for businesses and their customers (source: GitLab) ### Overview + - ➡️ [getting-started](./getting-started/) - ➡️ [DevOps toolchain](https://en.wikipedia.org/wiki/DevOps_toolchain) - ➡️ [roadmap.sh/devops](https://roadmap.sh/devops) @@ -230,22 +231,27 @@ We cover a wide range of DevOps topics in our content library, explore them unde 📖 hashicorp.com/vault ⏩ coming-soon - + SQL sql 📖 sql/README.md ✔️ mysql-basics - - + HAProxy haproxy 📖 www.haproxy.org ✔️ HAProxy basics - + + + Kafka + kafka + 📖 kafka.apache.org/ + ✔️ Kafka basics + - And **more upcoming topics...⏩** you can star/follow this repository to get more up-to-dated content ⭐ From eef3660df5a9a54cfab547e2844adb585cddafb4 Mon Sep 17 00:00:00 2001 From: Tung Bui Date: Sun, 1 Sep 2024 15:26:20 +0700 Subject: [PATCH 4/5] Fix zookeeper --- topics/kafka/basic/README.md | 1 + topics/kafka/basic/docker-compose.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/topics/kafka/basic/README.md b/topics/kafka/basic/README.md index 43155ea..d4f970a 100644 --- a/topics/kafka/basic/README.md +++ b/topics/kafka/basic/README.md @@ -50,6 +50,7 @@ You should see the message you produced earlier. To stop and remove the Kafka and Zookeeper containers, run: ```bash +cd devops-basics/topics/kafka/basic docker-compose down ``` diff --git a/topics/kafka/basic/docker-compose.yml b/topics/kafka/basic/docker-compose.yml index 6ad2c2a..bb43d5c 100644 --- a/topics/kafka/basic/docker-compose.yml +++ b/topics/kafka/basic/docker-compose.yml @@ -5,6 +5,7 @@ services: container_name: zookeeper environment: - ZOO_ENABLE_AUTH=no + - ALLOW_ANONYMOUS_LOGIN=yes ports: - '2181:2181' From 20b2210e2e787d09102ccb25cc5c3f0b241540c1 Mon Sep 17 00:00:00 2001 From: Tung Bui Date: Sun, 1 Sep 2024 15:29:20 +0700 Subject: [PATCH 5/5] Adjust doc --- topics/kafka/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/topics/kafka/README.md b/topics/kafka/README.md index af9cca8..d96a230 100644 --- a/topics/kafka/README.md +++ b/topics/kafka/README.md @@ -34,9 +34,9 @@ - [Kafka 101: Getting Started with Kafka](https://kafka.apache.org/quickstart) -### Kafka Hello World +### Kafka Basics 👋 -- See: [basic](./basic/) +- See: [**basic**](./basic/) ## 5. Beyond the Basics