KAFKA-12940: Enable JDK 16 builds in Jenkins (#10702)

JDK 15 no longer receives updates, so we want to switch from JDK 15 to JDK 16.
However, we have a number of tests that don't yet pass with JDK 16.

Instead of replacing JDK 15 with JDK 16, we have both for now and we either
disable (via annotations) or exclude (via gradle) the tests that don't pass with
JDK 16 yet. The annotations approach is better, but it doesn't work for tests
that rely on the PowerMock JUnit 4 runner.

Also add `--illegal-access=permit` when building with JDK 16 to make MiniKdc
work for now. This has been removed in JDK 17, so we'll have to figure out
another solution when we migrate to that.

Relevant JIRAs for the disabled tests: KAFKA-12790, KAFKA-12941, KAFKA-12942.

Moved some assertions from `testTlsDefaults` to `testUnsupportedTlsVersion`
since the former claims to test the success case while the former tests the failure case.

Reviewers: Chia-Ping Tsai <chia7712@gmail.com>

Jenkinsfile

@@ -142,6 +142,7 @@ pipeline {
           }
         }
 
+        // Remove this when all tests pass with JDK 16
         stage('JDK 15 and Scala 2.13') {
           agent { label 'ubuntu' }
           tools {
@@ -161,6 +162,25 @@ pipeline {
           }
         }
 
+        stage('JDK 16 and Scala 2.13') {
+          agent { label 'ubuntu' }
+          tools {
+            jdk 'jdk_16_latest'
+          }
+          options {
+            timeout(time: 8, unit: 'HOURS') 
+            timestamps()
+          }
+          environment {
+            SCALA_VERSION=2.13
+          }
+          steps {
+            doValidation()
+            doTest(env)
+            echo 'Skipping Kafka Streams archetype test for Java 16'
+          }
+        }
+
         stage('ARM') {
           agent { label 'arm4' }
           options {
@@ -231,14 +251,14 @@ pipeline {
           }
         }
 
-        stage('JDK 15 and Scala 2.12') {
+        stage('JDK 16 and Scala 2.12') {
           when {
             not { changeRequest() }
             beforeAgent true
           }
           agent { label 'ubuntu' }
           tools {
-            jdk 'jdk_15_latest'
+            jdk 'jdk_16_latest'
           }
           options {
             timeout(time: 8, unit: 'HOURS') 
@@ -250,7 +270,7 @@ pipeline {
           steps {
             doValidation()
             doTest(env)
-            echo 'Skipping Kafka Streams archetype test for Java 15'
+            echo 'Skipping Kafka Streams archetype test for Java 16'
           }
         }
       }

build.gradle

@@ -102,6 +102,8 @@ ext {
 
   defaultMaxHeapSize = "2g"
   defaultJvmArgs = ["-Xss4m", "-XX:+UseParallelGC"]
+  if (JavaVersion.current() == JavaVersion.VERSION_16)  
+    defaultJvmArgs.add("--illegal-access=permit")
 
   userMaxForks = project.hasProperty('maxParallelForks') ? maxParallelForks.toInteger() : null
   userIgnoreFailures = project.hasProperty('ignoreFailures') ? ignoreFailures : false
@@ -353,6 +355,27 @@ subprojects {
     }
   }
 
+  // The suites are for running sets of tests in IDEs.
+  // Gradle will run each test class, so we exclude the suites to avoid redundantly running the tests twice.
+  def testsToExclude = ['**/*Suite.class']
+  // Exclude PowerMock tests when running with Java 16 until a version of PowerMock that supports Java 16 is released
+  // The relevant issues are https://github.com/powermock/powermock/issues/1094 and https://github.com/powermock/powermock/issues/1099
+  if (JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_16)) {
+    testsToExclude.addAll([
+      // connect tests
+      "**/AbstractHerderTest.*", "**/ConnectClusterStateImplTest.*", "**/ConnectorPluginsResourceTest.*",
+      "**/ConnectorsResourceTest.*", "**/DistributedHerderTest.*", "**/FileOffsetBakingStoreTest.*",
+      "**/ErrorHandlingTaskTest.*", "**/KafkaConfigBackingStoreTest.*", "**/KafkaOffsetBackingStoreTest.*",
+      "**/KafkaBasedLogTest.*", "**/OffsetStorageWriterTest.*", "**/StandaloneHerderTest.*",
+      "**/SourceTaskOffsetCommitterTest.*", "**/WorkerConfigTransformerTest.*", "**/WorkerGroupMemberTest.*",
+      "**/WorkerSinkTaskTest.*", "**/WorkerSinkTaskThreadedTest.*", "**/WorkerSourceTaskTest.*",
+      "**/WorkerTaskTest.*", "**/WorkerTest.*", "**/RestServerTest.*",
+      // streams tests
+      "**/KafkaStreamsTest.*", "**/RepartitionTopicsTest.*", "**/RocksDBMetricsRecorderTest.*",
+      "**/StreamsMetricsImplTest.*", "**/StateManagerUtilTest.*", "**/TableSourceNodeTest.*"
+    ])
+  }
+
   test {
     maxParallelForks = userMaxForks ?: Runtime.runtime.availableProcessors()
     ignoreFailures = userIgnoreFailures
@@ -367,9 +390,7 @@ subprojects {
     }
     logTestStdout.rehydrate(delegate, owner, this)()
 
-    // The suites are for running sets of tests in IDEs.
-    // Gradle will run each test class, so we exclude the suites to avoid redundantly running the tests twice.
-    exclude '**/*Suite.class'
+    exclude testsToExclude
 
     if (shouldUseJUnit5)
       useJUnitPlatform()
@@ -395,9 +416,7 @@ subprojects {
     }
     logTestStdout.rehydrate(delegate, owner, this)()
 
-    // The suites are for running sets of tests in IDEs.
-    // Gradle will run each test class, so we exclude the suites to avoid redundantly running the tests twice.
-    exclude '**/*Suite.class'
+    exclude testsToExclude
 
     if (shouldUseJUnit5) {
       useJUnitPlatform {
@@ -429,9 +448,7 @@ subprojects {
     }
     logTestStdout.rehydrate(delegate, owner, this)()
 
-    // The suites are for running sets of tests in IDEs.
-    // Gradle will run each test class, so we exclude the suites to avoid redundantly running the tests twice.
-    exclude '**/*Suite.class'
+    exclude testsToExclude
 
     if (shouldUseJUnit5) {
       useJUnitPlatform {

clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java

@@ -36,6 +36,8 @@ import org.apache.kafka.common.utils.Utils;
 import org.apache.kafka.test.TestSslUtils;
 import org.apache.kafka.test.TestUtils;
 import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.condition.DisabledOnJre;
+import org.junit.jupiter.api.condition.JRE;
 import org.junit.jupiter.api.extension.ExtensionContext;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
@@ -591,7 +593,7 @@ public class SslTransportLayerTest {
     }
 
     /**
-     * Tests that connection success with the default TLS version.
+     * Tests that connection succeeds with the default TLS version.
      */
     @ParameterizedTest
     @ArgumentsSource(SslTransportLayerArgumentsProvider.class)
@@ -611,12 +613,6 @@ public class SslTransportLayerTest {
         NetworkTestUtils.checkClientConnection(selector, "0", 10, 100);
         server.verifyAuthenticationMetrics(1, 0);
         selector.close();
-
-        checkAuthenticationFailed(args, "1", "TLSv1.1");
-        server.verifyAuthenticationMetrics(1, 1);
-
-        checkAuthenticationFailed(args, "2", "TLSv1");
-        server.verifyAuthenticationMetrics(1, 2);
     }
 
     /** Checks connection failed using the specified {@code tlsVersion}. */
@@ -636,12 +632,15 @@ public class SslTransportLayerTest {
      */
     @ParameterizedTest
     @ArgumentsSource(SslTransportLayerArgumentsProvider.class)
-    public void testUnsupportedTLSVersion(Args args) throws Exception {
-        args.sslServerConfigs.put(SslConfigs.SSL_ENABLED_PROTOCOLS_CONFIG, Arrays.asList("TLSv1.2"));
+    @DisabledOnJre(JRE.JAVA_16)
+    public void testUnsupportedTlsVersion(Args args) throws Exception {
         server = createEchoServer(args, SecurityProtocol.SSL);
 
         checkAuthenticationFailed(args, "0", "TLSv1.1");
         server.verifyAuthenticationMetrics(0, 1);
+
+        checkAuthenticationFailed(args, "0", "TLSv1");
+        server.verifyAuthenticationMetrics(0, 2);
     }
 
     /**