feat(fvt): test wider range of kafkas

Experiment with whether we can successfully run against a wider range of kafka broker versions. Assuming this is successful, we could run a larger FV against the main branch, whilst keeping the PR builds to a smaller subset of versions using GitHub actions re-usable workflows https://docs.github.com/en/actions/using-workflows/reusing-workflows#using-a-matrix-strategy-with-a-reusable-workflow Signed-off-by: Dominic Evans <dominic.evans@uk.ibm.com>
IBM · Aug 21, 2023 · 4ebb5f7 · 4ebb5f7
1 parent d3f0a9f
commit 4ebb5f7
Show file tree

Hide file tree

Showing 5 changed files with 57 additions and 24 deletions.
diff --git a/.github/workflows/fvt.yml b/.github/workflows/fvt.yml
@@ -15,11 +15,29 @@ jobs:
       fail-fast: false
       matrix:
         go-version: [1.21.x]
-        kafka-version: [3.3.2, 3.4.1, 3.5.1]
+        kafka-version: [1.0.2, 2.0.1, 2.2.2, 2.6.2, 2.8.2, 3.0.2, 3.3.2, 3.5.1]
+        include:
+          - kafka-version: 1.0.2
+            scala-version: 2.11
+          - kafka-version: 2.0.1
+            scala-version: 2.12
+          - kafka-version: 2.2.2
+            scala-version: 2.12
+          - kafka-version: 2.6.2
+            scala-version: 2.12
+          - kafka-version: 2.8.2
+            scala-version: 2.12
+          - kafka-version: 3.0.2
+            scala-version: 2.12
+          - kafka-version: 3.3.2
+            scala-version: 2.13
+          - kafka-version: 3.5.1
+            scala-version: 2.13
     env:
       DEBUG: true
       GOFLAGS: -trimpath
       KAFKA_VERSION: ${{ matrix.kafka-version }}
+      SCALA_VERSION: ${{ matrix.scala-version }}
     steps:
     - uses: actions/checkout@v3
     - name: Setup Docker

diff --git a/Dockerfile.kafka b/Dockerfile.kafka
@@ -33,6 +33,11 @@ RUN cd /tmp \
  && for DIR in /opt/kafka-*; do cp -v jaxb-api-2.3.0.jar $DIR/libs/ ; done \
  && rm -f jaxb-api-2.3.0.jar
 
+WORKDIR /opt/kafka-${KAFKA_VERSION}
+
+ENV JAVA_MAJOR_VERSION=11
+
+RUN sed -e "s/JAVA_MAJOR_VERSION=.*/JAVA_MAJOR_VERSION=${JAVA_MAJOR_VERSION}/" -i"" ./bin/kafka-run-class.sh
 
 COPY entrypoint.sh /
 

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,6 +1,7 @@
 version: '3.9'
 services:
   zookeeper-1:
+    hostname: 'zookeeper-1'
     image: 'docker.io/library/zookeeper:3.6.3'
     restart: always
     environment:
@@ -12,6 +13,7 @@ services:
       ZOO_MAX_CLIENT_CNXNS: '0'
       ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok'
   zookeeper-2:
+    hostname: 'zookeeper-2'
     image: 'docker.io/library/zookeeper:3.6.3'
     restart: always
     environment:
@@ -23,6 +25,7 @@ services:
       ZOO_MAX_CLIENT_CNXNS: '0'
       ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok'
   zookeeper-3:
+    hostname: 'zookeeper-3'
     image: 'docker.io/library/zookeeper:3.6.3'
     restart: always
     environment:
@@ -34,6 +37,7 @@ services:
       ZOO_MAX_CLIENT_CNXNS: '0'
       ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok'
   kafka-1:
+    hostname: 'kafka-1'
     image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
     build:
       context: .
@@ -45,12 +49,9 @@ services:
       test:
         [
           'CMD',
-          '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh',
+          '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
           '--bootstrap-server',
           'kafka-1:9091',
-          '--broker',
-          '1',
-          '--describe',
         ]
       interval: 15s
       timeout: 15s
@@ -78,7 +79,10 @@ services:
       KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
       KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
+      KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
   kafka-2:
+    hostname: 'kafka-2'
     image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
     build:
       context: .
@@ -90,12 +94,9 @@ services:
       test:
         [
           'CMD',
-          '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh',
+          '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
           '--bootstrap-server',
           'kafka-2:9091',
-          '--broker',
-          '2',
-          '--describe',
         ]
       interval: 15s
       timeout: 15s
@@ -123,7 +124,10 @@ services:
       KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
       KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
+      KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
   kafka-3:
+    hostname: 'kafka-3'
     image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
     build:
       context: .
@@ -135,12 +139,9 @@ services:
       test:
         [
           'CMD',
-          '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh',
+          '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
           '--bootstrap-server',
           'kafka-3:9091',
-          '--broker',
-          '3',
-          '--describe',
         ]
       interval: 15s
       timeout: 15s
@@ -168,7 +169,10 @@ services:
       KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
       KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
+      KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
   kafka-4:
+    hostname: 'kafka-4'
     image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
     build:
       context: .
@@ -180,12 +184,9 @@ services:
       test:
         [
           'CMD',
-          '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh',
+          '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
           '--bootstrap-server',
           'kafka-4:9091',
-          '--broker',
-          '4',
-          '--describe',
         ]
       interval: 15s
       timeout: 15s
@@ -213,7 +214,10 @@ services:
       KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
       KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
+      KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
   kafka-5:
+    hostname: 'kafka-5'
     image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
     build:
       context: .
@@ -225,12 +229,9 @@ services:
       test:
         [
           'CMD',
-          '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh',
+          '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
           '--bootstrap-server',
           'kafka-5:9091',
-          '--broker',
-          '5',
-          '--describe',
         ]
       interval: 15s
       timeout: 15s
@@ -258,7 +259,10 @@ services:
       KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
       KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
+      KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
   toxiproxy:
+    hostname: 'toxiproxy'
     image: 'ghcr.io/shopify/toxiproxy:2.4.0'
     healthcheck:
       test: ['CMD', '/toxiproxy-cli', 'l']

diff --git a/entrypoint.sh b/entrypoint.sh
@@ -11,7 +11,9 @@ fi
 cd "${KAFKA_HOME}" || exit 1
 
 # discard all empty/commented lines
-sed -e '/^#/d' -e '/^$/d' -i"" config/server.properties
+sed -e '/^#/d' -e '/^$/d' -i".orig" config/server.properties
+
+echo "########################################################################" >>config/server.properties
 
 # emulate kafka_configure_from_environment_variables from bitnami/bitnami-docker-kafka
 for var in "${!KAFKA_CFG_@}"; do

diff --git a/functional_consumer_test.go b/functional_consumer_test.go
@@ -46,7 +46,10 @@ func TestConsumerHighWaterMarkOffset(t *testing.T) {
 	setupFunctionalTest(t)
 	defer teardownFunctionalTest(t)
 
-	p, err := NewSyncProducer(FunctionalTestEnv.KafkaBrokerAddrs, nil)
+	config := NewFunctionalTestConfig()
+	config.Producer.Return.Successes = true
+
+	p, err := NewSyncProducer(FunctionalTestEnv.KafkaBrokerAddrs, config)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -57,7 +60,7 @@ func TestConsumerHighWaterMarkOffset(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	c, err := NewConsumer(FunctionalTestEnv.KafkaBrokerAddrs, NewFunctionalTestConfig())
+	c, err := NewConsumer(FunctionalTestEnv.KafkaBrokerAddrs, config)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -127,6 +130,7 @@ func TestVersionMatrixLZ4(t *testing.T) {
 
 // Support for zstd codec was introduced in v2.1.0.0
 func TestVersionMatrixZstd(t *testing.T) {
+	checkKafkaVersion(t, "2.1.0")
 	metrics.UseNilMetrics = true // disable Sarama's go-metrics library
 	t.Cleanup(func() {
 		metrics.UseNilMetrics = false