Update to confluent kafka
MODIFY
.github/workflows/test.yml
.github/workflows/test.yml
@@ -66,16 +66,6 @@ jobs:
strategy:
fail-fast: false
matrix:
- tests:
- - mix test --include integration
- - mix test --include consumer_group
- - mix test --include server_0_p_10_and_later
- - mix test --include server_0_p_10_p_1
- - mix test --include server_0_p_9_p_0
- - mix test --include server_0_p_8_p_0
- - mix test --include new_client
- - mix test
-
include:
- elixir: 1.12.3
erlang: 24.3
@@ -108,4 +98,4 @@ jobs:
- name: Docker ps
run: docker ps -a
- name: Run Tests
- run: ${{ matrix.tests }}
+ run: ./scripts/all_tests.sh

MODIFY
config/config.exs
config/config.exs
@@ -14,9 +14,9 @@ config :kafka_ex,
# server.properties file.
# In the case below you would set "advertised.host.name=localhost"
brokers: [
+ {"localhost", 9092},
{"localhost", 9093},
- {"localhost", 9094},
- {"localhost", 9095}
+ {"localhost", 9094}
],
#
# OR:
@@ -61,6 +61,8 @@ config :kafka_ex,
# see SSL OPTION DESCRIPTIONS - CLIENT SIDE at http://erlang.org/doc/man/ssl.html
# for supported options
ssl_options: [
+ # Fix warnings. More at https://github.com/erlang/otp/issues/5352
+ verify: :verify_none,
cacertfile: File.cwd!() <> "/ssl/ca-cert",
certfile: File.cwd!() <> "/ssl/cert.pem",
keyfile: File.cwd!() <> "/ssl/key.pem"

MODIFY
docker-compose-kafka.env
docker-compose-kafka.env
-
#################################
# Common Kafka config
#################################
# Note: any property `something.bla` can be configure by setting `KAFKA_SOMETHING_BLA`.
-######## topic creation
-
-KAFKA_CREATE_TOPICS=consumer_group_implementation_test:4:2,test0p8p0:4:2
-
######## zookeeper
KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
@@ -17,19 +12,17 @@ KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS=6000
######## advertised hosts names and protocols
# alternative to KAFKA_ADVERTISED_HOST_NAME is: HOSTNAME_COMMAND: ip addr | grep -o "inet [0-9.]*" | grep -v "127\.0\.0\.1" | grep -o "[0-9.]*"
-KAFKA_ADVERTISED_HOST_NAME=localhost
-KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:SSL
-KAFKA_ADVERTISED_PROTOCOL_NAME=OUTSIDE
-KAFKA_PROTOCOL_NAME=INSIDE
+KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:SSL
+KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL
######## SSL
-KAFKA_SSL_KEYSTORE_LOCATION=/ssl/server.keystore.jks
+KAFKA_SSL_KEYSTORE_LOCATION=/etc/kafka/secrets/kafka.server.keystore.jks
KAFKA_SSL_KEYSTORE_PASSWORD=kafka_ex
KAFKA_SSL_KEY_PASSWORD=kafka_ex
-KAFKA_SSL_TRUSTSTORE_LOCATION=/ssl/server.truststore.jks
+
+KAFKA_SSL_TRUSTSTORE_LOCATION=/etc/kafka/secrets/kafka.server.truststore.jks
KAFKA_SSL_TRUSTSTORE_PASSWORD=kafka_ex
-KAFKA_SSL_SECURE_RANDOM_IMPLEMENTATION=SHA1PRNG
######## Config
@@ -37,7 +30,7 @@ KAFKA_DELETE_TOPIC_ENABLE=true
KAFKA_NUM_NETWORK_THREADS=1
KAFKA_NUM_IO_THREADS=1
KAFKA_BACKGROUND_THREADS=1
-KAFKA_LOG_CLEANER_THREADS=0
+KAFKA_LOG_CLEANER_THREADS=1
# KAFKA_SOCKET_SEND_BUFFER_BYTES=102400
# KAFKA_SOCKET_RECEIVE_BUFFER_BYTES=102400
# KAFKA_SOCKET_REQUEST_MAX_BYTES=104857600

MODIFY
docker-compose.yml
docker-compose.yml
-version: '2.4'
+version: "3.8"
services:
zookeeper:
- image: wurstmeister/zookeeper:3.4.6
+ image: confluentinc/cp-zookeeper:5.5.1
ports:
- - "2181:2181"
- healthcheck:
- test: ["CMD-SHELL", "echo ruok | nc -w 2 127.0.0.1 2181 | grep -q imok"]
- interval: 5s
- timeout: 10s
- retries: 15
- start_period: 10s
+ - '32181:32181'
+ environment:
+ ZOOKEEPER_CLIENT_PORT: 32181
+ ZOOKEEPER_TICK_TIME: 2000
- kafka1:
- hostname: kafka1
- image: wurstmeister/kafka:0.11.0.1
+ kafka-1:
+ image: confluentinc/cp-kafka:5.5.1
ports:
- - "9093:9093"
- volumes:
- - ./ssl:/ssl
- - ./scripts/kafka_check_health:/kafka_check_health
+ - '9092:9092'
+ depends_on:
+ - zookeeper
env_file: docker-compose-kafka.env
environment:
KAFKA_BROKER_ID: 1
- KAFKA_ADVERTISED_PORT: 9093
- KAFKA_PORT: 9092
- KAFKA_HOST_NAME: kafka1
+ KAFKA_ZOOKEEPER_CONNECT: zookeeper:32181
+ KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-1:29092,EXTERNAL://localhost:9092
healthcheck:
- test: /kafka_check_health
- interval: 10s
+ test: kafka-topics --bootstrap-server kafka-1:29092 --list
+ interval: 30s
timeout: 10s
- retries: 30
- start_period: 10s
- depends_on:
- zookeeper:
- condition: service_healthy
+ retries: 4
+ volumes:
+ - ./ssl/kafka.server.keystore.jks:/etc/kafka/secrets/kafka.server.keystore.jks:ro,z
+ - ./ssl/kafka.server.truststore.jks:/etc/kafka/secrets/kafka.server.truststore.jks:ro,z
- kafka2:
- hostname: kafka2
- image: wurstmeister/kafka:0.11.0.1
+ kafka-2:
+ image: confluentinc/cp-kafka:5.5.1
ports:
- - "9094:9094"
- volumes:
- - ./ssl:/ssl
- - ./scripts/kafka_check_health:/kafka_check_health
+ - '9093:9093'
+ depends_on:
+ - zookeeper
env_file: docker-compose-kafka.env
environment:
KAFKA_BROKER_ID: 2
- KAFKA_ADVERTISED_PORT: 9094
- KAFKA_PORT: 9092
- KAFKA_HOST_NAME: kafka2
+ KAFKA_ZOOKEEPER_CONNECT: zookeeper:32181
+ KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-2:29093,EXTERNAL://localhost:9093
healthcheck:
- test: /kafka_check_health
- interval: 10s
+ test: kafka-topics --bootstrap-server kafka-2:29093 --list
+ interval: 30s
timeout: 10s
- retries: 30
- start_period: 10s
- depends_on:
- zookeeper:
- condition: service_healthy
+ retries: 4
+ volumes:
+ - ./ssl/kafka.server.keystore.jks:/etc/kafka/secrets/kafka.server.keystore.jks:ro,z
+ - ./ssl/kafka.server.truststore.jks:/etc/kafka/secrets/kafka.server.truststore.jks:ro,z
- kafka3:
- hostname: kafka3
- image: wurstmeister/kafka:0.11.0.1
+ kafka-3:
+ image: confluentinc/cp-kafka:5.5.1
ports:
- - "9095:9095"
- volumes:
- - ./ssl:/ssl
- - ./scripts/kafka_check_health:/kafka_check_health
+ - '9094:9094'
+ depends_on:
+ - zookeeper
env_file: docker-compose-kafka.env
environment:
KAFKA_BROKER_ID: 3
- KAFKA_ADVERTISED_PORT: 9095
- KAFKA_PORT: 9092
- KAFKA_HOST_NAME: kafka3
+ KAFKA_ZOOKEEPER_CONNECT: zookeeper:32181
+ KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-3:29094,EXTERNAL://localhost:9094
healthcheck:
- test: /kafka_check_health
- interval: 10s
+ test: kafka-topics --bootstrap-server kafka-2:29093 --list
+ interval: 30s
timeout: 10s
- retries: 30
- start_period: 10s
- depends_on:
- zookeeper:
- condition: service_healthy
+ retries: 4
+ volumes:
+ - ./ssl/kafka.server.keystore.jks:/etc/kafka/secrets/kafka.server.keystore.jks:ro,z
+ - ./ssl/kafka.server.truststore.jks:/etc/kafka/secrets/kafka.server.truststore.jks:ro,z
- # This is a dummy service that forces all other services to be healthy before
- # docker-compose up can be considered successful.
ready:
image: busybox:1.31-musl
command: tail -f /dev/null
depends_on:
- kafka1:
+ kafka-1:
condition: service_healthy
- kafka2:
+ kafka-2:
condition: service_healthy
- kafka3:
+ kafka-3:
condition: service_healthy
\ No newline at end of file

DELETE
scripts/kafka_check_health
-#! /bin/bash
-
-cd $(dirname $0)
-
-broker_ready() {
- # Inspired by: https://github.com/wurstmeister/kafka-docker/issues/167#issuecomment-439849789
- current_broker=$(cat $KAFKA_HOME/config/server.properties | awk 'BEGIN{FS="="}/^broker.id=/{print $2}')
- brokers=$($KAFKA_HOME/bin/zookeeper-shell.sh zookeeper:2181 <<< "ls /brokers/ids" | tail -1)
-
- echo "brokers: $brokers"
- echo "current_broker: $current_broker"
-
- echo "${brokers}" | jq '.[]' | grep -q "^${current_broker}$";
- return $?
-}
-
-#######################
-
-broker_ready
-
-#######################

RENAME
ssl/server.keystore.jks
ssl/kafka.server.keystore.jks
RENAME
ssl/server.truststore.jks
ssl/kafka.server.truststore.jks
Update to confluent kafka