Improve docker-compose by adding healthcheck condition at startup
MODIFY
config/test.exs
config/test.exs
use Mix.Config
-config :ex_unit, capture_log: true
+config :ex_unit, capture_log: is_nil(System.get_env("SHOW_LOGS"))
config :kafka_ex,
sync_timeout: 60_000
+
+# Help debug tests that are tricky to understand
+config :logger, :console,
+ format: "$time [$level] [$metadata] $message\n",
+ metadata: [:module, :function, :pid]

MODIFY
docker-compose-kafka.env
docker-compose-kafka.env
@@ -17,7 +17,7 @@ KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS=6000
# alternative to KAFKA_ADVERTISED_HOST_NAME is: HOSTNAME_COMMAND: ip addr | grep -o "inet [0-9.]*" | grep -v "127\.0\.0\.1" | grep -o "[0-9.]*"
KAFKA_ADVERTISED_HOST_NAME=localhost
-KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:SSL,OUTSIDE:SSL
+KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:SSL
KAFKA_ADVERTISED_PROTOCOL_NAME=OUTSIDE
KAFKA_PROTOCOL_NAME=INSIDE

MODIFY
docker-compose.yml
docker-compose.yml
-version: '3.2'
+version: '2.4'
services:
zookeeper:
image: wurstmeister/zookeeper:3.4.6
ports:
- "2181:2181"
+ healthcheck:
+ test: ["CMD-SHELL", "echo ruok | nc -w 2 127.0.0.1 2181 | grep -q imok"]
+ interval: 5s
+ timeout: 10s
+ retries: 15
+ start_period: 10s
+
kafka1:
+ hostname: kafka1
image: wurstmeister/kafka:0.11.0.1
ports:
- "9093:9093"
@@ -13,12 +21,25 @@ services:
- zookeeper
volumes:
- ./ssl:/ssl
+ - ./scripts/kafka_check_health:/kafka_check_health
env_file: docker-compose-kafka.env
environment:
KAFKA_BROKER_ID: 1
KAFKA_ADVERTISED_PORT: 9093
+ KAFKA_PORT: 9092
+ KAFKA_HOST_NAME: kafka1
+ healthcheck:
+ test: /kafka_check_health
+ interval: 10s
+ timeout: 10s
+ retries: 30
+ start_period: 10s
+ depends_on:
+ zookeeper:
+ condition: service_healthy
kafka2:
+ hostname: kafka2
image: wurstmeister/kafka:0.11.0.1
ports:
- "9094:9094"
@@ -26,12 +47,25 @@ services:
- zookeeper
volumes:
- ./ssl:/ssl
+ - ./scripts/kafka_check_health:/kafka_check_health
env_file: docker-compose-kafka.env
environment:
KAFKA_BROKER_ID: 2
KAFKA_ADVERTISED_PORT: 9094
+ KAFKA_PORT: 9092
+ KAFKA_HOST_NAME: kafka2
+ healthcheck:
+ test: /kafka_check_health
+ interval: 10s
+ timeout: 10s
+ retries: 30
+ start_period: 10s
+ depends_on:
+ zookeeper:
+ condition: service_healthy
kafka3:
+ hostname: kafka3
image: wurstmeister/kafka:0.11.0.1
ports:
- "9095:9095"
@@ -39,7 +73,32 @@ services:
- zookeeper
volumes:
- ./ssl:/ssl
+ - ./scripts/kafka_check_health:/kafka_check_health
env_file: docker-compose-kafka.env
environment:
KAFKA_BROKER_ID: 3
KAFKA_ADVERTISED_PORT: 9095
+ KAFKA_PORT: 9092
+ KAFKA_HOST_NAME: kafka3
+ healthcheck:
+ test: /kafka_check_health
+ interval: 10s
+ timeout: 10s
+ retries: 30
+ start_period: 10s
+ depends_on:
+ zookeeper:
+ condition: service_healthy
+
+ # This is a dummy service that forces all other services to be healthy before
+ # docker-compose up can be considered successful.
+ ready:
+ image: busybox:1.31-musl
+ command: tail -f /dev/null
+ depends_on:
+ kafka1:
+ condition: service_healthy
+ kafka2:
+ condition: service_healthy
+ kafka3:
+ condition: service_healthy

ADD
scripts/kafka_check_health
+#! /bin/bash
+
+cd $(dirname $0)
+
+broker_ready() {
+ # Inspired by: https://github.com/wurstmeister/kafka-docker/issues/167#issuecomment-439849789
+ current_broker=$(cat $KAFKA_HOME/config/server.properties | awk 'BEGIN{FS="="}/^broker.id=/{print $2}')
+ brokers=$($KAFKA_HOME/bin/zookeeper-shell.sh zookeeper:2181 <<< "ls /brokers/ids" | tail -1)
+
+ echo "brokers: $brokers"
+ echo "current_broker: $current_broker"
+
+ echo "${brokers}" | jq '.[]' | grep -q "^${current_broker}$";
+ return $?
+}
+
+#######################
+
+broker_ready
+
+#######################

Improve docker-compose by adding healthcheck condition at startup