Use env vars in docker-compose to configure everything
MODIFY
config/config.exs
config/config.exs
@@ -14,9 +14,9 @@ config :kafka_ex,
# server.properties file.
# In the case below you would set "advertised.host.name=localhost"
brokers: [
- {"localhost", 9092},
{"localhost", 9093},
- {"localhost", 9094}
+ {"localhost", 9094},
+ {"localhost", 9095}
],
#
# OR:

ADD
docker-compose-kafka.env
+#################################
+# Common Kafka config
+#################################
+
+# Note: any property `something.bla` can be configure by setting `KAFKA_SOMETHING_BLA`.
+
+######## topic creation
+
+KAFKA_CREATE_TOPICS=consumer_group_implementation_test:4:2,test0p8p0:4:2
+
+######## zookeeper
+
+KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS=6000
+
+######## advertised hosts names and protocols
+
+# alternative to KAFKA_ADVERTISED_HOST_NAME is: HOSTNAME_COMMAND: ip addr | grep -o "inet [0-9.]*" | grep -v "127\.0\.0\.1" | grep -o "[0-9.]*"
+KAFKA_ADVERTISED_HOST_NAME=localhost
+KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:SSL,OUTSIDE:SSL
+KAFKA_ADVERTISED_PROTOCOL_NAME=OUTSIDE
+KAFKA_PROTOCOL_NAME=INSIDE
+
+######## SSL
+
+KAFKA_SSL_KEYSTORE_LOCATION=/ssl/server.keystore.jks
+KAFKA_SSL_KEYSTORE_PASSWORD=kafka_ex
+KAFKA_SSL_KEY_PASSWORD=kafka_ex
+KAFKA_SSL_TRUSTSTORE_LOCATION=/ssl/server.truststore.jks
+KAFKA_SSL_TRUSTSTORE_PASSWORD=kafka_ex
+KAFKA_SSL_SECURE_RANDOM_IMPLEMENTATION=SHA1PRNG
+
+######## Config
+
+KAFKA_DELETE_TOPIC_ENABLE=true
+# KAFKA_NUM_NETWORK_THREADS=3
+# KAFKA_NUM_IO_THREADS=8
+# KAFKA_SOCKET_SEND_BUFFER_BYTES=102400
+# KAFKA_SOCKET_RECEIVE_BUFFER_BYTES=102400
+# KAFKA_SOCKET_REQUEST_MAX_BYTES=104857600
+KAFKA_LOG_DIRS=/tmp/kafka_log
+KAFKA_NUM_PARTITIONS=1
+KAFKA_NUM_RECOVERY_THREADS_PER_DATA_DIR=1
+KAFKA_LOG_RETENTION_HOURS=168
+# KAFKA_LOG_SEGMENT_BYTES=1073741824
+# KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS=300000

MODIFY
docker-compose.yml
docker-compose.yml
version: '3.2'
+
services:
zookeeper:
image: wurstmeister/zookeeper:3.4.6
@@ -7,30 +8,38 @@ services:
kafka1:
image: wurstmeister/kafka:0.11.0.1
ports:
- - "9092:9092"
+ - "9093:9093"
depends_on:
- zookeeper
volumes:
- - ./kafka1/server.properties.in:/opt/kafka/config/server.properties.in
- - ./scripts/docker-start-kafka.sh:/usr/bin/start-kafka.sh
- ./ssl:/ssl
+ env_file: docker-compose-kafka.env
+ environment:
+ KAFKA_BROKER_ID: 1
+ KAFKA_ADVERTISED_PORT: 9093
+
kafka2:
image: wurstmeister/kafka:0.11.0.1
ports:
- - "9093:9093"
+ - "9094:9094"
depends_on:
- zookeeper
volumes:
- - ./kafka2/server.properties.in:/opt/kafka/config/server.properties.in
- - ./scripts/docker-start-kafka.sh:/usr/bin/start-kafka.sh
- ./ssl:/ssl
+ env_file: docker-compose-kafka.env
+ environment:
+ KAFKA_BROKER_ID: 2
+ KAFKA_ADVERTISED_PORT: 9094
+
kafka3:
image: wurstmeister/kafka:0.11.0.1
ports:
- - "9094:9094"
+ - "9095:9095"
depends_on:
- zookeeper
volumes:
- - ./kafka3/server.properties.in:/opt/kafka/config/server.properties.in
- - ./scripts/docker-start-kafka.sh:/usr/bin/start-kafka.sh
- ./ssl:/ssl
+ env_file: docker-compose-kafka.env
+ environment:
+ KAFKA_BROKER_ID: 3
+ KAFKA_ADVERTISED_PORT: 9095

MODIFY
scripts/README.md
scripts/README.md
+# Scripts
+
The scripts in this directory are used for testing KafkaEx and are not part of
the release package. Each script should have a comment near the top explaining
its purpose.

DELETE
scripts/docker-start-kafka.sh
-#!/bin/bash
-
-# This script is used when the docker kafka container starts
-#
-# This version is specialized from
-# https://github.com/wurstmeister/kafka-docker/blob/master/start-kafka.sh
-# to work with our setup (esp. with ssl)
-#
-# Note that we mainly manage the server.properties file during `docker_up.sh`;
-# the wurstmeister/kafka behavior of translating KAFKA_* env vars to settings
-# is removed from this script.
-
-set -ev
-
-cp ${KAFKA_HOME}/config/server.properties.in ${KAFKA_HOME}/config/server.properties
-
-KAFKA_PID=0
-
-# see https://medium.com/@gchudnov/trapping-signals-in-docker-containers-7a57fdda7d86#.bh35ir4u5
-term_handler() {
- echo 'Stopping Kafka....'
- if [ $KAFKA_PID -ne 0 ]; then
- kill -s TERM "$KAFKA_PID"
- wait "$KAFKA_PID"
- fi
- echo 'Kafka stopped.'
- exit
-}
-
-
-# Capture kill requests to stop properly
-trap "term_handler" SIGHUP SIGINT SIGTERM
-$KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/config/server.properties &
-KAFKA_PID=$!
-
-wait "$KAFKA_PID"

DELETE
scripts/docker_ip.sh
-#!/bin/bash
-# Finds a private IP for your machine. Tested on MacOS 10.14, Ubuntu 16.04, and Centos 7
-
-# 192.168.x.x, 172.[16-31].x.x, 10.x.x.x
-regex='inet (192\.168\.|172\.1[6789]\.|172\.2[0-9]\.|172\.3[01]\.|10\.)'
-
-if [ "$(uname)" == "Darwin" ]; then
- # MacOS
- ifconfig | grep -E "$regex" | grep broadcast | awk -F ' ' '{print $2}' | head -n 1
-else
- # Linux
- ip addr | grep -E "$regex" | grep brd | awk -F ' ' '{print $2}' | cut -d/ -f1 | head -n 1
-fi
-

MODIFY
scripts/docker_up.sh
scripts/docker_up.sh
#!/bin/bash
-# Launches a dockerized kafka cluster configured for testing with KafkaEx
-#
-# This script attempts to auto-detect the ip address of an active network
-# interface using `./scripts/docker_ip.sh`.
-#
-# This script should be run from the project root
-
-set -e
-
-# Kafka needs to know our ip address so that it can advertise valid
-# connnection details
-
-export DOCKER_IP=$(./scripts/docker_ip.sh)
-
-# for debugging purposes
-echo Detected active ip address ${DOCKER_IP}
-
-for i in 1 2 3
-do
- port=$(expr 9092 + ${i} - 1)
- mkdir -p kafka${i}
- target=kafka${i}/server.properties.in
- cp ./server.properties ${target}
- # configure broker and port
- sed -i.bak "s/@broker_id@/${i}/g" ${target}
- sed -i.bak "s/@port@/${port}/g" ${target}
- # the @pwd become root directory so we get /ssl
- sed -i.bak "s|@pwd@||g" ${target}
- # point at zookeeper in docker
- sed -i.bak "s/localhost:2181/zookeeper:2181/" ${target}
- # delete the existing listeners line
- sed -i.bak "/^listeners=/d" ${target}
- # add an advertised.listeners and listeners together at the end
- echo "advertised.listeners=SSL://${DOCKER_IP}:${port}" >> ${target}
- echo "listeners=SSL://0.0.0.0:${port}" >> ${target}
-done
+# This script starts zookeeper and the 3 kafka nodes used for testing
docker-compose up -d
-
-# create topics needed for testing
-docker-compose exec kafka3 /bin/bash -c "KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 KAFKA_PORT=9094 KAFKA_CREATE_TOPICS=consumer_group_implementation_test:4:2,test0p8p0:4:2 create-topics.sh"

DELETE
scripts/docker_up_ubuntu.sh
-#!/bin/bash
-
-# see docker_up.sh
-# This script should be run from the project root
-
-set -e
-
-IP_IFACE="docker0"
-DOCKER_IP=$(ip addr show dev $IP_IFACE | grep -o "inet[^/]*" | grep -o "[0-9]\{1,3\}[.][0-9.]*")
-
-for i in 1 2 3
-do
- port=$(expr 9092 + ${i} - 1)
- mkdir -p kafka${i}
- target=kafka${i}/server.properties.in
- cp ./server.properties ${target}
- # configure broker and port
- sed -i.bak "s/@broker_id@/${i}/g" ${target}
- sed -i.bak "s/@port@/${port}/g" ${target}
- # the @pwd become root directory so we get /ssl
- sed -i.bak "s|@pwd@||g" ${target}
- # point at zookeeper in docker
- sed -i.bak "s/localhost:2181/zookeeper:2181/" ${target}
- # delete the existing listeners line
- sed -i.bak "/^listeners=/d" ${target}
- # add an advertised.listeners and listeners together at the end
- echo "advertised.listeners=SSL://${DOCKER_IP}:${port}" >> ${target}
- echo "listeners=SSL://0.0.0.0:${port}" >> ${target}
-done
-
-docker-compose up -d
-
-# create topics needed for testing
-docker-compose exec kafka3 /bin/bash -c "KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 KAFKA_PORT=9094 KAFKA_CREATE_TOPICS=consumer_group_implementation_test:4:2,test0p8p0:4:2 create-topics.sh"

Use env vars in docker-compose to configure everything