initial commit

.dockerignore

@@ -0,0 +1,4 @@
+build
+README.md
+CHANGELOG.md
+.gitignore

.gitignore

@@ -0,0 +1,12 @@
+# IDE
+.idea
+.vscode
+
+# Output of the go covetage tool
+*.out
+
+# Test binary `go test -c`
+*.test
+!.env.test
+
+protokaf

.golangci.yml

@@ -0,0 +1,75 @@
+run:
+  # default concurrency is a available CPU number
+  concurrency: 4
+
+  # timeout for analysis, e.g. 30s, 5m, default is 1m
+  timeout: 5m
+
+  # exit code when at least one issue was found, default is 1
+  issues-exit-code: 1
+
+  # include test files or not, default is true
+  tests: true
+
+  # skip download modules
+  modules-download-mode: readonly
+
+# output configuration options
+output:
+  # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
+  format: colored-line-number
+
+  # print lines of code with issue, default is true
+  print-issued-lines: true
+
+  # print linter name in the end of issue text, default is true
+  print-linter-name: true
+
+linters-settings:
+  funlen:
+    lines: 90
+  golint:
+    min-confidence: 0.9
+  lll:
+    line-length: 140
+
+linters:
+  enable:
+    - govet
+    - errcheck
+    - staticcheck
+    - unused
+    - gosimple
+    - structcheck
+    - varcheck
+    - ineffassign
+    - deadcode
+    - typecheck
+    - bodyclose
+    - revive
+    - stylecheck
+    - gosec
+    - unconvert
+    - dupl
+    - goconst
+    - gocyclo
+    - gocognit
+    - gofmt
+    - depguard
+    - misspell
+    - lll
+    - dogsled
+    - nakedret
+    - prealloc
+    - exportloopref
+    - gocritic
+    - funlen
+    - whitespace
+
+issues:
+  # List of regexps of issue texts to exclude, empty list by default.
+  # But independently from this option we use default exclude patterns,
+  # it can be disabled by `exclude-use-default: false`. To list all
+  # excluded by default patterns execute `golangci-lint run --help`
+  exclude:
+     - Using the variable on range scope `tt` in function literal

.protokaf.yaml

@@ -0,0 +1,7 @@
+debug: true
+broker: 0.0.0.0:9092
+#broker: 0.0.0.0:9093 # with auth
+#kafka-auth-dsn: "SCRAM-SHA-256:admin:secret"
+output: json # text
+proto:
+  - internal/proto/testdata/example.proto

Makefile

@@ -0,0 +1,18 @@
+include scripts/*.mk
+
+APP_NAME=protokaf
+
+run:	## Run protokaf
+	@go run .
+
+test:	## Run tests
+	go test -cover -p 1 -count=1 ./...
+
+none:
+	sleep 31536000
+
+lint:	## Run linter
+	golangci-lint run
+
+install:
+	go install .

README.md

@@ -0,0 +1,160 @@
+# protokaf
+
+Kafka producer and consumer tool in protobuf format.
+
+## Features
+- Sending and receiving messages
+- Trace messages with Jaeger
+- Sending messages using a template with random data
+
+## Install
+```sh
+go install github.com/SberMarket-Tech/protokaf@latest
+```
+
+## Configuration
+Configuration file is optional. If no configuration file was specified with `-F ..` on the command line, `protokaf` will try `.protokaf.yaml`, `$HOME/.protokaf.yaml`.
+
+**Example of `.protokaf.yaml`**
+```yaml
+debug: true
+broker: "<addr>:<port>"
+kafka-auth-dsn: "SCRAM-SHA-512:<namespace>:<passwd>"
+proto: "<dir>/<protofile>"
+```
+
+## Help
+```sh
+$ protokaf help
+```
+
+## List metadata
+```sh
+$ protokaf list [-t <topic>(,<topic>...)]
+1 brokers:
+ broker 1 "127.0.0.1:9093"
+2 topics:
+  topic "test-topic", partitions: 1
+    partition 0, leader 1, replicas: [1] (offline: []), isrs: [1]
+  topic "test", partitions: 1
+    partition 0, leader 1, replicas: [1] (offline: []), isrs: [1]
+```
+
+## Produce
+### Help
+```sh
+$ protokaf produce -h
+```
+
+### Examples
+This proto file will be used in the examples below. 
+
+`api/example.ptoto`
+```protobuf
+syntax = "proto3";
+
+package example;
+
+message HelloRequest {
+  string name = 1;
+  int32 age = 2;
+}
+```
+
+**A simple produce message**
+```sh
+$ protokaf produce HelloRequest \
+    --broker kafka:9092 \
+    --proto api/example.proto \
+    --topic test \
+    --data '{"name": "Alice", "age": 11}'
+```
+
+**Produce message with headers**
+```sh
+$ protokaf produce HelloRequest \
+    --broker kafka:9092 \
+    --proto api/example.proto \
+    --topic test \
+    --header "priority=high" \
+    --header "application=protokaf" \
+    --data '{"name": "Alice", "age": 11}'
+```
+
+**Produce message with <a href="#template">template</a>**
+```sh
+$ protokaf produce HelloRequest \
+    --broker kafka:9092 \
+    --proto api/example.proto \
+    --topic test \
+    --data '{"name": {{randomFemaleName | quote}}, "age": {{randomNumber 10 20}}}' \
+    --count 10 \
+    --seed 42
+```
+
+**Produce message with Kafka auth**
+```sh
+$ protokaf produce HelloRequest \
+    --broker kafka:9093 \
+    --kafka-auth-dsn "SCRAM-SHA-512:login:passwd" \
+    --proto api/example.proto \
+    --topic test \
+    --data '{"name": "Alice", "age": 11}'
+```
+
+**Read data from stdin or flag**
+
+Read message `HelloRequest` from `stdin`, produce to `test` topic
+```sh
+$ echo '{"name": "Alice", "age": 11}' | protokaf produce HelloRequest -t test
+```
+
+Read message `HelloRequest` from `-d` value, produce to `test` topic
+```sh
+$ protokaf produce HelloRequest -t test -d '{"name": "Alice", "age": 11}'
+```
+
+### Template<a id="template"></a>
+**Template options**
+* `--seed <int>` You can set number greater then zero to produce the same pseudo-random sequence of messages
+* `--count <int>` Useful for generating messages with random data
+* `--concurrency <int>` Number of message senders to run concurrently for const concurrency producing
+
+**Show all template functions**
+```sh
+$ protokaf produce --template-functions-print
+```
+
+## Consume
+### Help
+```sh
+$ protokaf help consume
+```
+
+### Examples
+```sh
+$ protokaf consume HelloRequest \
+    --broker kafka:9092 \
+    --proto api/example.proto \
+    --group mygroup \
+    --topic test
+```
+
+**Read messages from Kafka `test` topic, use group `mygroup`, print to `stdout`**
+```sh
+$ protokaf consume HelloRequest -G mygroup -t test
+```
+
+**Read the last `10` messages from `test` topic, then exit**
+```sh
+$ protokaf consume HelloRequest -G mygroup -t test -c 10
+```
+
+## Testing
+
+### Prepare test environment
+```sh
+make docker-dev-up
+make kafka-users
+make install # optional (you can use 'go run . <args> <flags>')
+```

build/docker/Dockerfile

@@ -0,0 +1,6 @@
+FROM golang:1.16.5
+
+# copy files for build
+COPY . .
+
+CMD ["make", "run"]

build/docker/docker-compose.yaml

@@ -0,0 +1,74 @@
+version: "3"
+
+services:
+  app:
+    container_name: ${APP_PROJECT}_app
+    build:
+      context: ../..
+      dockerfile: ./build/docker/Dockerfile
+      args:
+        - APP_PATH=${APP_PATH}
+    command: make none
+    depends_on:
+      - kafka
+    links:
+      - kafka:kafka.local
+    working_dir: ${APP_PATH}
+    volumes:
+      - ../..:${APP_PATH}
+      - $GOPATH/pkg/mod:/go/pkg/mod
+  kafka:
+    container_name: ${APP_PROJECT}_kafka
+    image: wurstmeister/kafka:2.13-2.7.0
+    hostname: kafka
+    ports:
+      - "9092:9092"
+      - "9093:9093"
+    environment:
+      KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
+      KAFKA_BROKER_ID: 1
+      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
+      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_NUM_PARTITIONS: 1
+      KAFKA_DEFAULT_REPLICATION_FACTOR: 1
+      KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: 1
+      KAFKA_ADVERTISED_LISTENERS: INSIDE_SASL://kafka:19093,INSIDE://kafka:19092,OUTSIDE_SASL://${DOCKER_HOST_IP:-127.0.0.1}:9093,OUTSIDE://${DOCKER_HOST_IP:-127.0.0.1}:9092
+      KAFKA_LISTENERS: INSIDE://:19092,INSIDE_SASL://:19093,OUTSIDE://:9092,OUTSIDE_SASL://:9093
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,INSIDE_SASL:SASL_PLAINTEXT,OUTSIDE_SASL:SASL_PLAINTEXT,OUTSIDE:PLAINTEXT
+
+      # auth related options:
+      KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf"
+      KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
+      KAFKA_SASL_ENABLED_MECHANISMS: "PLAIN,SCRAM-SHA-256,SCRAM-SHA-512"
+      KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.authorizer.AclAuthorizer"
+      KAFKA_SUPER_USERS: "User:admin"
+      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - kafka-data:/opt/kafka
+      - ./kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf
+    depends_on:
+      - zookeeper
+    restart: always
+  zookeeper:
+    container_name: ${APP_PROJECT}_zookeeper
+    image: zookeeper:3.4.14
+    ports:
+      - "2181"
+  jaeger:
+    image: jaegertracing/all-in-one:1.22
+    container_name: ${APP_PROJECT}_jaeger
+    environment:
+      COLLECTOR_ZIPKIN_HTTP_PORT: 9411
+    ports:
+      - "6831:6831/udp"
+      - "16686:16686"
+      - "9411:9411"
+
+volumes:
+  kafka-data:
+    driver: local
+
+networks:
+  default:
+    name: ${APP_PROJECT}

build/docker/kafka_server_jaas.conf

@@ -0,0 +1,7 @@
+KafkaServer {
+  org.apache.kafka.common.security.scram.ScramLoginModule required
+  username="admin"
+  password="secret"
+  user_admin="secret"
+  user_client="secret";
+};