KAFKA-4422; Drop support for Scala 2.10 (KIP-119)

Author: Ismael Juma <ismael@juma.me.uk>

Reviewers: Ewen Cheslack-Postava <me@ewencp.org>

Closes #2956 from ijuma/kafka-4422-drop-support-scala-2.10

README.md

@@ -76,15 +76,17 @@ The release file can be found inside `./core/build/distributions/`.
 ### Cleaning the build ###
     ./gradlew clean
 
-### Running a task on a particular version of Scala (either 2.10.6 or 2.11.8) ###
-*Note that if building the jars with a version other than 2.10.6, you need to set the `SCALA_VERSION` variable or change it in `bin/kafka-run-class.sh` to run the quick start.*
+### Running a task on a particular version of Scala (either 2.11.x or 2.12.x) ###
+*Note that if building the jars with a version other than 2.11.11, you need to set the `SCALA_VERSION` variable or change it in `bin/kafka-run-class.sh` to run the quick start.*
 
-You can pass either the major version (eg 2.11) or the full version (eg 2.11.8):
+You can pass either the major version (eg 2.11) or the full version (eg 2.11.11):
 
     ./gradlew -PscalaVersion=2.11 jar
     ./gradlew -PscalaVersion=2.11 test
     ./gradlew -PscalaVersion=2.11 releaseTarGz
 
+Scala 2.12.x requires Java 8.
+
 ### Running a task for a specific project ###
 This is for `core`, `examples` and `clients`
 

bin/kafka-run-class.sh

@@ -48,7 +48,7 @@ should_include_file() {
 base_dir=$(dirname $0)/..
 
 if [ -z "$SCALA_VERSION" ]; then
-  SCALA_VERSION=2.10.6
+  SCALA_VERSION=2.11.11
 fi
 
 if [ -z "$SCALA_BINARY_VERSION" ]; then

bin/windows/kafka-run-class.bat

@@ -27,7 +27,7 @@ set BASE_DIR=%CD%
 popd
 
 IF ["%SCALA_VERSION%"] EQU [""] (
-  set SCALA_VERSION=2.10.6
+  set SCALA_VERSION=2.11.11
 )
 
 IF ["%SCALA_BINARY_VERSION%"] EQU [""] (

build.gradle

@@ -416,48 +416,48 @@ task jacocoRootReport(type: org.gradle.testing.jacoco.tasks.JacocoReport) {
 
 task reportCoverage(dependsOn: ['jacocoRootReport', 'core:reportCoverage'])
 
-for ( sv in ['2_10', '2_11', '2_12'] ) {
-  String svInDot = sv.replaceAll( "_", ".")
+for ( sv in availableScalaVersions ) {
+  String taskSuffix = sv.replaceAll("\\.", "_")
 
-  tasks.create(name: "jar_core_${sv}", type: GradleBuild) {
+  tasks.create(name: "jar_core_${taskSuffix}", type: GradleBuild) {
     startParameter = project.getGradle().getStartParameter().newInstance()
-    startParameter.projectProperties += [scalaVersion: "${svInDot}"]
+    startParameter.projectProperties += [scalaVersion: "${sv}"]
     tasks = ['core:jar']
   }
 
-  tasks.create(name: "test_core_${sv}", type: GradleBuild) {
+  tasks.create(name: "test_core_${taskSuffix}", type: GradleBuild) {
     startParameter = project.getGradle().getStartParameter().newInstance()
-    startParameter.projectProperties += [scalaVersion: "${svInDot}"]
+    startParameter.projectProperties += [scalaVersion: "${sv}"]
     tasks = ['core:test']
   }
 
-  tasks.create(name: "srcJar_${sv}", type: GradleBuild) {
+  tasks.create(name: "srcJar_${taskSuffix}", type: GradleBuild) {
     startParameter = project.getGradle().getStartParameter().newInstance()
-    startParameter.projectProperties += [scalaVersion: "${svInDot}"]
+    startParameter.projectProperties += [scalaVersion: "${sv}"]
     tasks = ['core:srcJar']
   }
 
-  tasks.create(name: "docsJar_${sv}", type: GradleBuild) {
+  tasks.create(name: "docsJar_${taskSuffix}", type: GradleBuild) {
     startParameter = project.getGradle().getStartParameter().newInstance()
-    startParameter.projectProperties += [scalaVersion: "${svInDot}"]
+    startParameter.projectProperties += [scalaVersion: "${sv}"]
     tasks = ['core:docsJar']
   }
 
-  tasks.create(name: "install_${sv}", type: GradleBuild) {
+  tasks.create(name: "install_${taskSuffix}", type: GradleBuild) {
     startParameter = project.getGradle().getStartParameter().newInstance()
-    startParameter.projectProperties += [scalaVersion: "${svInDot}"]
+    startParameter.projectProperties += [scalaVersion: "${sv}"]
     tasks = ['install']
   }
 
-  tasks.create(name: "releaseTarGz_${sv}", type: GradleBuild) {
+  tasks.create(name: "releaseTarGz_${taskSuffix}", type: GradleBuild) {
     startParameter = project.getGradle().getStartParameter().newInstance()
-    startParameter.projectProperties += [scalaVersion: "${svInDot}"]
+    startParameter.projectProperties += [scalaVersion: "${sv}"]
     tasks = ['releaseTarGz']
   }
 
-  tasks.create(name: "uploadCoreArchives_${sv}", type: GradleBuild) {
+  tasks.create(name: "uploadCoreArchives_${taskSuffix}", type: GradleBuild) {
     startParameter = project.getGradle().getStartParameter().newInstance()
-    startParameter.projectProperties += [scalaVersion: "${svInDot}"]
+    startParameter.projectProperties += [scalaVersion: "${sv}"]
     tasks = ['core:uploadArchives']
   }
 }
@@ -465,21 +465,26 @@ for ( sv in ['2_10', '2_11', '2_12'] ) {
 def connectPkgs = ['connect:api', 'connect:runtime', 'connect:transforms', 'connect:json', 'connect:file']
 def pkgs = ['clients', 'examples', 'log4j-appender', 'tools', 'streams', 'streams:examples'] + connectPkgs
 
+/** Create one task per default Scala version */
+def withDefScalaVersions(taskName) {
+  defaultScalaVersions.collect { taskName + '_' + it.replaceAll('\\.', '_') }
+}
+
 tasks.create(name: "jarConnect", dependsOn: connectPkgs.collect { it + ":jar" }) {}
-tasks.create(name: "jarAll", dependsOn: ['jar_core_2_10', 'jar_core_2_11'] + pkgs.collect { it + ":jar" }) { }
+tasks.create(name: "jarAll", dependsOn: withDefScalaVersions('jar_core') + pkgs.collect { it + ":jar" }) { }
 
-tasks.create(name: "srcJarAll", dependsOn: ['srcJar_2_10', 'srcJar_2_11'] + pkgs.collect { it + ":srcJar" }) { }
+tasks.create(name: "srcJarAll", dependsOn: withDefScalaVersions('srcJar') + pkgs.collect { it + ":srcJar" }) { }
 
-tasks.create(name: "docsJarAll", dependsOn: ['docsJar_2_10', 'docsJar_2_11'] + pkgs.collect { it + ":docsJar" }) { }
+tasks.create(name: "docsJarAll", dependsOn: withDefScalaVersions('docsJar') + pkgs.collect { it + ":docsJar" }) { }
 
 tasks.create(name: "testConnect", dependsOn: connectPkgs.collect { it + ":test" }) {}
-tasks.create(name: "testAll", dependsOn: ['test_core_2_10', 'test_core_2_11'] + pkgs.collect { it + ":test" }) { }
+tasks.create(name: "testAll", dependsOn: withDefScalaVersions('test_core') + pkgs.collect { it + ":test" }) { }
 
-tasks.create(name: "installAll", dependsOn: ['install_2_10', 'install_2_11'] + pkgs.collect { it + ":install" }) { }
+tasks.create(name: "installAll", dependsOn: withDefScalaVersions('install') + pkgs.collect { it + ":install" }) { }
 
-tasks.create(name: "releaseTarGzAll", dependsOn: ['releaseTarGz_2_10', 'releaseTarGz_2_11']) { }
+tasks.create(name: "releaseTarGzAll", dependsOn: withDefScalaVersions('releaseTarGz')) { }
 
-tasks.create(name: "uploadArchivesAll", dependsOn: ['uploadCoreArchives_2_10', 'uploadCoreArchives_2_11'] + pkgs.collect { it + ":uploadArchives" }) { }
+tasks.create(name: "uploadArchivesAll", dependsOn: withDefScalaVersions('uploadCoreArchives') + pkgs.collect { it + ":uploadArchives" }) { }
 
 project(':core') {
   println "Building project 'core' with Scala version ${versions.scala}"
@@ -496,10 +501,7 @@ project(':core') {
     compile libs.slf4jlog4j
     compile libs.zkclient
     compile libs.zookeeper
-    // These modules were broken out of core scala in 2.10. We can remove special handling when 2.10 support is dropped.
-    if (versions.baseScala != '2.10') {
-      compile libs.scalaParserCombinators
-    }
+    compile libs.scalaParserCombinators
 
     testCompile project(':clients').sourceSets.test.output
     testCompile libs.bcpkix

docs/quickstart.html

@@ -187,7 +187,7 @@ my test message 2
 On Windows use:
 <pre>
 &gt; <b>wmic process get processid,caption,commandline | find "java.exe" | find "server-1.properties"</b>
-java.exe    java  -Xmx1G -Xms1G -server -XX:+UseG1GC ... build\libs\kafka_2.10-0.10.2.0.jar"  kafka.Kafka config\server-1.properties    <i>644</i>
+java.exe    java  -Xmx1G -Xms1G -server -XX:+UseG1GC ... build\libs\kafka_2.11-0.10.2.0.jar"  kafka.Kafka config\server-1.properties    <i>644</i>
 &gt; <b>taskkill /pid 644 /f</b>
 </pre>
 

gradle.properties

@@ -17,6 +17,6 @@ group=org.apache.kafka
 # NOTE: When you change this version number, you should also make sure to update
 # the version numbers in tests/kafkatest/__init__.py and kafka-merge-pr.py.
 version=0.11.0.0-SNAPSHOT
-scalaVersion=2.10.6
+scalaVersion=2.11.11
 task=build
 org.gradle.jvmargs=-XX:MaxPermSize=512m -Xmx1024m -Xss2m

gradle/dependencies.gradle

@@ -20,8 +20,32 @@
 ext {
   versions = [:]
   libs = [:]
+  
+  // Enabled by default when commands like `testAll` are invoked
+  defaultScalaVersions = [ '2.11' ]
+  // Available if -PscalaVersion is used, this is necessary because Scala 2.12 requires Java 8 while Kafka is usually
+  // built with Java 7
+  availableScalaVersions = [ '2.11', '2.12' ]
 }
 
+// Add Scala version
+def defaultScala211Version = '2.11.11'
+def defaultScala212Version = '2.12.2'
+if (hasProperty('scalaVersion')) {
+  if (scalaVersion == '2.11') {
+    versions["scala"] = defaultScala211Version
+  } else if (scalaVersion == '2.12') {
+    versions["scala"] = defaultScala212Version
+  } else {
+    versions["scala"] = scalaVersion
+  }
+} else {
+  versions["scala"] = defaultScala211Version
+}
+
+// Add base Scala version
+versions["baseScala"] = versions.scala.substring(0, versions.scala.lastIndexOf("."))
+
 versions += [
   apacheda: "1.0.0-M33",
   apacheds: "2.0.0-M21",
@@ -49,27 +73,6 @@ versions += [
   jfreechart: "1.0.0",
 ]
 
-// Add Scala version
-def defaultScala210Version = '2.10.6'
-def defaultScala211Version = '2.11.11'
-def defaultScala212Version = '2.12.2'
-if (hasProperty('scalaVersion')) {
-  if (scalaVersion == '2.10') {
-    versions["scala"] = defaultScala210Version
-  } else if (scalaVersion == '2.11') {
-    versions["scala"] = defaultScala211Version
-  } else if (scalaVersion == '2.12') {
-    versions["scala"] = defaultScala212Version
-  } else {
-    versions["scala"] = scalaVersion
-  }
-} else {
-  versions["scala"] = defaultScala210Version
-}
-
-// Add base Scala version
-versions["baseScala"] = versions.scala.substring(0, versions.scala.lastIndexOf("."))
-
 libs += [
   argparse4j: "net.sourceforge.argparse4j:argparse4j:$versions.argparse4j",
   apacheda: "org.apache.directory.api:api-all:$versions.apacheda",

tests/docker/Dockerfile

@@ -23,9 +23,10 @@ ADD ssh /root/.ssh
 RUN chmod 600 /root/.ssh/id_rsa
 RUN apt update && apt install -y unzip wget curl jq coreutils openssh-server net-tools vim python-pip python-dev libffi-dev libssl-dev
 RUN pip install -U pip && pip install --upgrade cffi ducktape==0.6.0
+# Test with multiple Scala version
 RUN mkdir -p "/opt/kafka-0.8.2.2" && curl -s "${MIRROR}kafka/0.8.2.2/kafka_2.10-0.8.2.2.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.8.2.2"
-RUN mkdir -p "/opt/kafka-0.9.0.1" && curl -s "${MIRROR}kafka/0.9.0.1/kafka_2.10-0.9.0.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.9.0.1"
-RUN mkdir -p "/opt/kafka-0.10.0.1" && curl -s "${MIRROR}kafka/0.10.0.1/kafka_2.10-0.10.0.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.10.0.1"
-RUN mkdir -p "/opt/kafka-0.10.1.1" && curl -s "${MIRROR}kafka/0.10.1.1/kafka_2.10-0.10.1.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.10.1.1"
+RUN mkdir -p "/opt/kafka-0.9.0.1" && curl -s "${MIRROR}kafka/0.9.0.1/kafka_2.11-0.9.0.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.9.0.1"
+RUN mkdir -p "/opt/kafka-0.10.0.1" && curl -s "${MIRROR}kafka/0.10.0.1/kafka_2.11-0.10.0.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.10.0.1"
+RUN mkdir -p "/opt/kafka-0.10.1.1" && curl -s "${MIRROR}kafka/0.10.1.1/kafka_2.11-0.10.1.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.10.1.1"
 
 CMD service ssh start && tail -f /dev/null

vagrant/base.sh

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -e
+set -ex
 
 if [ -z `which javac` ]; then
     apt-get -y update
@@ -65,9 +65,10 @@ fi
 
 get_kafka() {
     version=$1
+    scala_version=$2
 
     kafka_dir=/opt/kafka-$version
-    url=https://s3-us-west-2.amazonaws.com/kafka-packages-$version/kafka_2.10-$version.tgz
+    url=https://s3-us-west-2.amazonaws.com/kafka-packages-$version/kafka_$scala_version-$version.tgz
     if [ ! -d /opt/kafka-$version ]; then
         pushd /tmp
         curl -O $url
@@ -81,13 +82,14 @@ get_kafka() {
     fi
 }
 
-get_kafka 0.8.2.2
+# Test multiple Scala versions
+get_kafka 0.8.2.2 2.10
 chmod a+rw /opt/kafka-0.8.2.2
-get_kafka 0.9.0.1
+get_kafka 0.9.0.1 2.11
 chmod a+rw /opt/kafka-0.9.0.1
-get_kafka 0.10.0.1
+get_kafka 0.10.0.1 2.11
 chmod a+rw /opt/kafka-0.10.0.1
-get_kafka 0.10.1.1
+get_kafka 0.10.1.1 2.11
 chmod a+rw /opt/kafka-0.10.1.1