Merge remote-tracking branch 'apache-github/trunk' into KAFKA-13900

* apache-github/trunk: (447 commits)
  KAFKA-13959: Controller should unfence Broker with busy metadata log (apache#12274)
  KAFKA-10199: Expose read only task from state updater (apache#12497)
  KAFKA-14154; Return NOT_CONTROLLER from AlterPartition if leader is ahead of controller (apache#12506)
  KAFKA-13986; Brokers should include node.id in fetches to metadata quorum (apache#12498)
  KAFKA-14163; Retry compilation after zinc compile cache error (apache#12507)
  Remove duplicate common.message.* from clients:test jar file (apache#12407)
  KAFKA-13060: Replace EasyMock and PowerMock with Mockito in WorkerGroupMemberTest.java (apache#12484)
  Fix the rate window size calculation for edge cases (apache#12184)
  MINOR: Upgrade gradle to 7.5.1 and bump other build/test dependencies (apache#12495)
  KAFKA-14140: Ensure an offline or in-controlled-shutdown replica is not eligible to join ISR in ZK mode (apache#12487)
  KAFKA-14114: Add Metadata Error Related Metrics
  MINOR: BrokerMetadataSnapshotter must avoid exceeding batch size (apache#12486)
  MINOR: Upgrade mockito test dependencies (apache#12460)
  KAFKA-14144:; Compare AlterPartition LeaderAndIsr before fencing partition epoch (apache#12489)
  KAFKA-14134: Replace EasyMock with Mockito for WorkerConnectorTest (apache#12472)
  MINOR: Update scala version in bin scripts to 2.13.8 (apache#12477)
  KAFKA-14104; Add CRC validation when iterating over Metadata Log Records (apache#12457)
  MINOR: add :server-common test dependency to :storage (apache#12488)
  KAFKA-14107: Upgrade Jetty version for CVE fixes (apache#12440)
  KAFKA-14124: improve quorum controller fault handling (apache#12447)
  ...

Jenkinsfile

@@ -19,7 +19,7 @@
 
 def doValidation() {
   sh """
-    ./gradlew -PscalaVersion=$SCALA_VERSION clean compileJava compileScala compileTestJava compileTestScala \
+    ./retry_zinc ./gradlew -PscalaVersion=$SCALA_VERSION clean compileJava compileScala compileTestJava compileTestScala \
         spotlessScalaCheck checkstyleMain checkstyleTest spotbugsMain rat \
         --profile --no-daemon --continue -PxmlSpotBugsReport=true
   """
@@ -160,42 +160,6 @@ pipeline {
             echo 'Skipping Kafka Streams archetype test for Java 17'
           }
         }
-
-        stage('ARM') {
-          agent { label 'arm4' }
-          options {
-            timeout(time: 2, unit: 'HOURS') 
-            timestamps()
-          }
-          environment {
-            SCALA_VERSION=2.12
-          }
-          steps {
-            doValidation()
-            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
-              doTest(env, 'unitTest')
-            }
-            echo 'Skipping Kafka Streams archetype test for ARM build'
-          }
-        }
-
-        stage('PowerPC') {
-          agent { label 'ppc64le' }
-          options {
-            timeout(time: 2, unit: 'HOURS')
-            timestamps()
-          }
-          environment {
-            SCALA_VERSION=2.12
-          }
-          steps {
-            doValidation()
-            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
-              doTest(env, 'unitTest')
-            }
-            echo 'Skipping Kafka Streams archetype test for PowerPC build'
-          }
-        }
 
         // To avoid excessive Jenkins resource usage, we only run the stages
         // above at the PR stage. The ones below are executed after changes

LICENSE-binary

@@ -208,50 +208,53 @@ License Version 2.0:
 audience-annotations-0.5.0
 commons-cli-1.4
 commons-lang3-3.8.1
-jackson-annotations-2.12.3
-jackson-core-2.12.3
-jackson-databind-2.12.3
-jackson-dataformat-csv-2.12.3
-jackson-datatype-jdk8-2.12.3
-jackson-jaxrs-base-2.12.3
-jackson-jaxrs-json-provider-2.12.3
-jackson-module-jaxb-annotations-2.12.3
+jackson-annotations-2.12.6
+jackson-core-2.12.6
+jackson-databind-2.12.6.1
+jackson-dataformat-csv-2.12.6
+jackson-datatype-jdk8-2.12.6
+jackson-jaxrs-base-2.12.6
+jackson-jaxrs-json-provider-2.12.6
+jackson-module-jaxb-annotations-2.12.6
 jackson-module-paranamer-2.10.5
-jackson-module-scala_2.13-2.12.3
+jackson-module-scala_2.13-2.12.6
 jakarta.validation-api-2.0.2
 javassist-3.27.0-GA
-jetty-client-9.4.43.v20210629
-jetty-continuation-9.4.43.v20210629
-jetty-http-9.4.43.v20210629
-jetty-io-9.4.43.v20210629
-jetty-security-9.4.43.v20210629
-jetty-server-9.4.43.v20210629
-jetty-servlet-9.4.43.v20210629
-jetty-servlets-9.4.43.v20210629
-jetty-util-9.4.43.v20210629
-jetty-util-ajax-9.4.43.v20210629
+jetty-client-9.4.44.v20210927
+jetty-continuation-9.4.44.v20210927
+jetty-http-9.4.44.v20210927
+jetty-io-9.4.44.v20210927
+jetty-security-9.4.44.v20210927
+jetty-server-9.4.44.v20210927
+jetty-servlet-9.4.44.v20210927
+jetty-servlets-9.4.44.v20210927
+jetty-util-9.4.44.v20210927
+jetty-util-ajax-9.4.44.v20210927
 jersey-common-2.34
 jersey-server-2.34
-jose4j-0.7.8
-log4j-1.2.17
+jose4j-0.7.9
 lz4-java-1.8.0
-maven-artifact-3.8.1
+maven-artifact-3.8.4
 metrics-core-4.1.12.1
-netty-buffer-4.1.68.Final
-netty-codec-4.1.68.Final
-netty-common-4.1.68.Final
-netty-handler-4.1.68.Final
-netty-resolver-4.1.68.Final
-netty-transport-4.1.68.Final
-netty-transport-native-epoll-4.1.68.Final
-netty-transport-native-unix-common-4.1.68.Final
-plexus-utils-3.2.1
-rocksdbjni-6.22.1.1
-scala-collection-compat_2.13-2.4.4
-scala-library-2.13.6
-scala-logging_2.13-3.9.3
-scala-reflect-2.13.6
-scala-java8-compat_2.13-1.0.0
+metrics-core-2.2.0
+netty-buffer-4.1.73.Final
+netty-codec-4.1.73.Final
+netty-common-4.1.73.Final
+netty-handler-4.1.73.Final
+netty-resolver-4.1.73.Final
+netty-tcnative-classes-2.0.46.Final
+netty-transport-4.1.73.Final
+netty-transport-classes-epoll-4.1.73.Final
+netty-transport-native-epoll-4.1.73.Final
+netty-transport-native-unix-common-4.1.73.Final
+plexus-utils-3.3.0
+reload4j-1.2.19
+rocksdbjni-6.29.4.1
+scala-collection-compat_2.13-2.6.0
+scala-library-2.13.8
+scala-logging_2.13-3.9.4
+scala-reflect-2.13.8
+scala-java8-compat_2.13-1.0.2
 snappy-java-1.1.8.4
 zookeeper-3.6.3
 zookeeper-jute-3.6.3
@@ -300,18 +303,18 @@ MIT License
 
 argparse4j-0.7.0, see: licenses/argparse-MIT
 jopt-simple-5.0.4, see: licenses/jopt-simple-MIT
-slf4j-api-1.7.30, see: licenses/slf4j-MIT
-slf4j-log4j12-1.7.30, see: licenses/slf4j-MIT
+slf4j-api-1.7.36, see: licenses/slf4j-MIT
+slf4j-reload4j-1.7.36, see: licenses/slf4j-MIT
 
 ---------------------------------------
 BSD 2-Clause
 
-zstd-jni-1.5.0-4 see: licenses/zstd-jni-BSD-2-clause
+zstd-jni-1.5.2-1 see: licenses/zstd-jni-BSD-2-clause
 
 ---------------------------------------
 BSD 3-Clause
 
-jline-3.12.1, see: licenses/jline-BSD-3-clause
+jline-3.21.0, see: licenses/jline-BSD-3-clause
 paranamer-2.8, see: licenses/paranamer-BSD-3-clause
 
 ---------------------------------------

README.md

@@ -37,13 +37,16 @@ Follow instructions in https://kafka.apache.org/quickstart
     ./gradlew integrationTest
 
 ### Force re-running tests without code change ###
-    ./gradlew cleanTest test
-    ./gradlew cleanTest unitTest
-    ./gradlew cleanTest integrationTest
+    ./gradlew -Prerun-tests test
+    ./gradlew -Prerun-tests unitTest
+    ./gradlew -Prerun-tests integrationTest
 
 ### Running a particular unit/integration test ###
     ./gradlew clients:test --tests RequestResponseTest
 
+### Repeatedly running a particular unit/integration test ###
+    I=0; while ./gradlew clients:test -Prerun-tests --tests RequestResponseTest --fail-fast; do (( I=$I+1 )); echo "Completed run: $I"; sleep 1; done
+
 ### Running a particular test method within a unit/integration test ###
     ./gradlew core:test --tests kafka.api.ProducerFailureHandlingTest.testCannotSendToInternalTopic
     ./gradlew clients:test --tests org.apache.kafka.clients.MetadataTest.testTimeToNextUpdate
@@ -180,15 +183,21 @@ Please note for this to work you should create/update user maven settings (typic
      ...
 
 
-### Installing the jars to the local Maven repository ###
-The recommended command is:
+### Installing ALL the jars to the local Maven repository ###
+The recommended command to build for both Scala 2.12 and 2.13 is:
 
     ./gradlewAll publishToMavenLocal
 
 For backwards compatibility, the following also works:
 
     ./gradlewAll install
 
+### Installing specific projects to the local Maven repository ###
+
+    ./gradlew -PskipSigning :streams:publishToMavenLocal
+
+If needed, you can specify the Scala version with `-PscalaVersion=2.13`.
+
 ### Building the test jar ###
     ./gradlew testJar
 

Vagrantfile

@@ -51,6 +51,9 @@ ec2_subnet_id = nil
 # Only override this by setting it to false if you're running in a VPC and you
 # are running Vagrant from within that VPC as well.
 ec2_associate_public_ip = nil
+ec2_iam_instance_profile_name = nil
+
+ebs_volume_type = 'gp3'
 
 jdk_major = '8'
 jdk_full = '8u202-linux-x64'
@@ -60,6 +63,18 @@ if File.exists?(local_config_file) then
   eval(File.read(local_config_file), binding, "Vagrantfile.local")
 end
 
+# override any instance type set by Vagrantfile.local or above via an environment variable
+if ENV['INSTANCE_TYPE'] then
+  ec2_instance_type = ENV['INSTANCE_TYPE']
+end
+
+# choose size based on overridden size
+if ec2_instance_type.start_with?("m3") then
+  ebs_volume_size = 20
+else
+  ebs_volume_size = 40
+end
+
 # TODO(ksweeney): RAM requirements are not empirical and can probably be significantly lowered.
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.hostmanager.enabled = enable_hostmanager
@@ -118,9 +133,11 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     aws.region = ec2_region
     aws.availability_zone = ec2_az
     aws.instance_type = ec2_instance_type
+
     aws.ami = ec2_ami
     aws.security_groups = ec2_security_groups
     aws.subnet_id = ec2_subnet_id
+    aws.block_device_mapping = [{ 'DeviceName' => '/dev/sda1', 'Ebs.VolumeType' => ebs_volume_type, 'Ebs.VolumeSize' => ebs_volume_size }]
     # If a subnet is specified, default to turning on a public IP unless the
     # user explicitly specifies the option. Without a public IP, Vagrant won't
     # be able to SSH into the hosts unless Vagrant is also running in the VPC.
@@ -133,6 +150,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
       region.spot_instance = ec2_spot_instance
       region.spot_max_price = ec2_spot_max_price
     end
+    aws.iam_instance_profile_name = ec2_iam_instance_profile_name
 
     # Exclude some directories that can grow very large from syncing
     override.vm.synced_folder ".", "/vagrant", type: "rsync", rsync__exclude: ['.git', 'core/data/', 'logs/', 'tests/results/', 'results/']

bin/kafka-run-class.sh

@@ -32,7 +32,7 @@ if [ -z "$INCLUDE_TEST_JARS" ]; then
 fi
 
 # Exclude jars not necessary for running commands.
-regex="(-(test|test-sources|src|scaladoc|javadoc)\.jar|jar.asc)$"
+regex="(-(test|test-sources|src|scaladoc|javadoc)\.jar|jar.asc|connect-file.*\.jar)$"
 should_include_file() {
   if [ "$INCLUDE_TEST_JARS" = true ]; then
     return 0
@@ -48,7 +48,7 @@ should_include_file() {
 base_dir=$(dirname $0)/..
 
 if [ -z "$SCALA_VERSION" ]; then
-  SCALA_VERSION=2.13.6
+  SCALA_VERSION=2.13.8
   if [[ -f "$base_dir/gradle.properties" ]]; then
     SCALA_VERSION=`grep "^scalaVersion=" "$base_dir/gradle.properties" | cut -d= -f 2`
   fi
@@ -171,7 +171,7 @@ do
   CLASSPATH="$CLASSPATH:$dir/*"
 done
 
-for cc_pkg in "api" "transforms" "runtime" "file" "mirror" "mirror-client" "json" "tools" "basic-auth-extension"
+for cc_pkg in "api" "transforms" "runtime" "mirror" "mirror-client" "json" "tools" "basic-auth-extension"
 do
   for file in "$base_dir"/connect/${cc_pkg}/build/libs/connect-${cc_pkg}*.jar;
   do

bin/windows/kafka-run-class.bat

@@ -27,7 +27,7 @@ set BASE_DIR=%CD%
 popd
 
 IF ["%SCALA_VERSION%"] EQU [""] (
-  set SCALA_VERSION=2.13.6
+  set SCALA_VERSION=2.13.8
 )
 
 IF ["%SCALA_BINARY_VERSION%"] EQU [""] (

bin/windows/kafka-storage.bat

@@ -0,0 +1,17 @@
+@echo off
+rem Licensed to the Apache Software Foundation (ASF) under one or more
+rem contributor license agreements.  See the NOTICE file distributed with
+rem this work for additional information regarding copyright ownership.
+rem The ASF licenses this file to You under the Apache License, Version 2.0
+rem (the "License"); you may not use this file except in compliance with
+rem the License.  You may obtain a copy of the License at
+rem
+rem     http://www.apache.org/licenses/LICENSE-2.0
+rem
+rem Unless required by applicable law or agreed to in writing, software
+rem distributed under the License is distributed on an "AS IS" BASIS,
+rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+rem See the License for the specific language governing permissions and
+rem limitations under the License.
+
+"%~dp0kafka-run-class.bat" kafka.tools.StorageTool %*