KAFKA-2807: Fix Kafka Connect packaging and move VerifiableSource/Sink into runtime jar.

Gradle does not handle subprojects with the same name (top-level tools vs
connect/tools) properly, making the dependency impossible to express correctly
since we need to move the ThroughputThrottler class into the top level tools
project. Moving the current set of tools into the runtime jar works fine since
they are only used for system tests at the moment.

Author: Ewen Cheslack-Postava <me@ewencp.org>

Reviewers: Gwen Shapira

Closes #512 from ewencp/kafka-2807-redux

build.gradle

@@ -230,7 +230,7 @@ for ( sv in ['2_10_5', '2_11_7'] ) {
   }
 }
 
-def connectPkgs = ['connect-api', 'connect-runtime', 'connect-json', 'connect-file', 'connect-tools']
+def connectPkgs = ['connect:api', 'connect:runtime', 'connect:json', 'connect:file']
 def pkgs = ['clients', 'examples', 'log4j-appender', 'tools', 'streams'] + connectPkgs
 
 tasks.create(name: "jarConnect", dependsOn: connectPkgs.collect { it + ":jar" }) {}
@@ -321,7 +321,7 @@ project(':core') {
     standardOutput = new File('docs/kafka_config.html').newOutputStream()
   }
 
-  task siteDocsTar(dependsOn: ['genProducerConfigDocs', 'genConsumerConfigDocs', 'genKafkaConfigDocs', ':connect-runtime:genConnectConfigDocs'], type: Tar) {
+  task siteDocsTar(dependsOn: ['genProducerConfigDocs', 'genConsumerConfigDocs', 'genKafkaConfigDocs', ':connect:runtime:genConnectConfigDocs'], type: Tar) {
     classifier = 'site-docs'
     compression = Compression.GZIP
     from project.file("../docs")
@@ -342,16 +342,14 @@ project(':core') {
     from(project.siteDocsTar) { into("site-docs/") }
     from(project(':tools').jar) { into("libs/") }
     from(project(':tools').configurations.runtime) { into("libs/") }
-    from(project(':connect-api').jar) { into("libs/") }
-    from(project(':connect-api').configurations.runtime) { into("libs/") }
-    from(project(':connect-runtime').jar) { into("libs/") }
-    from(project(':connect-runtime').configurations.runtime) { into("libs/") }
-    from(project(':connect-json').jar) { into("libs/") }
-    from(project(':connect-json').configurations.runtime) { into("libs/") }
-    from(project(':connect-file').jar) { into("libs/") }
-    from(project(':connect-file').configurations.runtime) { into("libs/") }
-    from(project(':connect-tools').jar) { into("libs/") }
-    from(project(':connect-tools').configurations.runtime) { into("libs/") }
+    from(project(':connect:api').jar) { into("libs/") }
+    from(project(':connect:api').configurations.runtime) { into("libs/") }
+    from(project(':connect:runtime').jar) { into("libs/") }
+    from(project(':connect:runtime').configurations.runtime) { into("libs/") }
+    from(project(':connect:json').jar) { into("libs/") }
+    from(project(':connect:json').configurations.runtime) { into("libs/") }
+    from(project(':connect:file').jar) { into("libs/") }
+    from(project(':connect:file').configurations.runtime) { into("libs/") }
   }
 
   jar {
@@ -638,7 +636,7 @@ project(':log4j-appender') {
   test.dependsOn('checkstyleMain', 'checkstyleTest')
 }
 
-project(':connect-api') {
+project(':connect:api') {
   apply plugin: 'checkstyle'
   archivesBaseName = "connect-api"
 
@@ -695,12 +693,12 @@ project(':connect-api') {
   test.dependsOn('checkstyleMain', 'checkstyleTest')
 }
 
-project(':connect-json') {
+project(':connect:json') {
   apply plugin: 'checkstyle'
   archivesBaseName = "connect-json"
 
   dependencies {
-    compile project(':connect-api')
+    compile project(':connect:api')
     compile "$slf4japi"
     compile "com.fasterxml.jackson.core:jackson-databind:$jackson_version"
 
@@ -756,13 +754,14 @@ project(':connect-json') {
   test.dependsOn('checkstyleMain', 'checkstyleTest')
 }
 
-project(':connect-runtime') {
+project(':connect:runtime') {
   apply plugin: 'checkstyle'
   archivesBaseName = "connect-runtime"
 
   dependencies {
-    compile project(':connect-api')
+    compile project(':connect:api')
     compile project(':clients')
+    compile project(':tools')
     compile "$slf4japi"
 
     compile "org.eclipse.jetty:jetty-server:$jetty_version"
@@ -776,7 +775,7 @@ project(':connect-runtime') {
     testCompile "$powermock_easymock"
     testCompile project(':clients').sourceSets.test.output
     testRuntime "$slf4jlog4j"
-    testRuntime project(":connect-json")
+    testRuntime project(":connect:json")
   }
 
   task testJar(type: Jar) {
@@ -830,75 +829,13 @@ project(':connect-runtime') {
   }
 }
 
-project(':connect-file') {
+project(':connect:file') {
   apply plugin: 'checkstyle'
   archivesBaseName = "connect-file"
 
   dependencies {
-    compile project(':connect-api')
-    compile "$slf4japi"
-
-    testCompile "$junit"
-    testCompile "$easymock"
-    testCompile "$powermock"
-    testCompile "$powermock_easymock"
-    testRuntime "$slf4jlog4j"
-  }
-
-  task testJar(type: Jar) {
-    classifier = 'test'
-    from sourceSets.test.output
-  }
-
-  test {
-    testLogging {
-      events "passed", "skipped", "failed"
-      exceptionFormat = 'full'
-    }
-  }
-
-  javadoc {
-    include "**/org/apache/kafka/connect/*"
-  }
-
-  tasks.create(name: "copyDependantLibs", type: Copy) {
-    from (configurations.testRuntime) {
-      include('slf4j-log4j12*')
-    }
-    from (configurations.runtime) {
-      exclude('kafka-clients*')
-      exclude('connect-*')
-    }
-    into "$buildDir/dependant-libs"
-  }
-
-  jar {
-    dependsOn copyDependantLibs
-  }
-
-  artifacts {
-    archives testJar
-  }
-
-  configurations {
-    archives.extendsFrom(testCompile)
-  }
-
-  checkstyle {
-    configFile = new File(rootDir, "checkstyle/checkstyle.xml")
-  }
-  test.dependsOn('checkstyleMain', 'checkstyleTest')
-}
-
-project(':connect-tools') {
-  apply plugin: 'checkstyle'
-  archivesBaseName = "connect-tools"
-
-  dependencies {
-    compile project(':connect-api')
-    compile project(':tools')
+    compile project(':connect:api')
     compile "$slf4japi"
-    compile "com.fasterxml.jackson.core:jackson-databind:$jackson_version"
 
     testCompile "$junit"
     testCompile "$easymock"

checkstyle/import-control.xml

@@ -207,6 +207,7 @@
 
     <subpackage name="tools">
       <allow pkg="org.apache.kafka.connect" />
+      <allow pkg="org.apache.kafka.tools" />
       <allow pkg="com.fasterxml.jackson" />
     </subpackage>
   </subpackage>

connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSourceTask.java

@@ -19,11 +19,11 @@ package org.apache.kafka.connect.tools;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.kafka.tools.ThroughputThrottler;
 import org.apache.kafka.connect.data.Schema;
 import org.apache.kafka.connect.errors.ConnectException;
 import org.apache.kafka.connect.source.SourceRecord;
 import org.apache.kafka.connect.source.SourceTask;
-import org.apache.kafka.tools.ThroughputThrottler;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

settings.gradle

@@ -15,4 +15,4 @@
 
 apply from: file('scala.gradle')
 include 'core', 'examples', 'clients', 'tools', 'streams', 'log4j-appender',
-        'connect-api', 'connect-runtime', 'connect-json', 'connect-file', 'connect-tools'
+        'connect:api', 'connect:runtime', 'connect:json', 'connect:file'

tools/src/main/java/org/apache/kafka/tools/ThroughputThrottler.java

@@ -13,7 +13,7 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- **/
+ */
 
 package org.apache.kafka.tools;