|
|
|
// Licensed to the Apache Software Foundation (ASF) under one or more
|
|
|
|
// contributor license agreements. See the NOTICE file distributed with
|
|
|
|
// this work for additional information regarding copyright ownership.
|
|
|
|
// The ASF licenses this file to You under the Apache License, Version 2.0
|
|
|
|
// (the "License"); you may not use this file except in compliance with
|
|
|
|
// the License. You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
import org.ajoberstar.grgit.Grgit
|
|
|
|
|
|
|
|
buildscript {
|
|
|
|
repositories {
|
|
|
|
mavenCentral()
|
|
|
|
jcenter()
|
|
|
|
maven {
|
|
|
|
url "https://plugins.gradle.org/m2/"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
apply from: file('gradle/buildscript.gradle'), to: buildscript
|
|
|
|
|
|
|
|
dependencies {
|
|
|
|
// For Apache Rat plugin to ignore non-Git files
|
|
|
|
classpath "org.ajoberstar:grgit:1.9.3"
|
|
|
|
classpath 'com.github.ben-manes:gradle-versions-plugin:0.20.0'
|
|
|
|
classpath 'org.scoverage:gradle-scoverage:2.5.0'
|
|
|
|
classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.2'
|
|
|
|
classpath 'org.owasp:dependency-check-gradle:3.3.4'
|
|
|
|
classpath "com.diffplug.spotless:spotless-plugin-gradle:3.16.0"
|
|
|
|
classpath "gradle.plugin.com.github.spotbugs:spotbugs-gradle-plugin:1.6.5"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
apply plugin: "com.diffplug.gradle.spotless"
|
|
|
|
spotless {
|
|
|
|
scala {
|
|
|
|
target 'streams/**/*.scala'
|
|
|
|
scalafmt('1.5.1').configFile('checkstyle/.scalafmt.conf')
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
apply from: "$rootDir/gradle/dependencies.gradle"
|
|
|
|
|
|
|
|
allprojects {
|
|
|
|
|
|
|
|
repositories {
|
|
|
|
mavenCentral()
|
|
|
|
}
|
|
|
|
|
|
|
|
apply plugin: 'idea'
|
|
|
|
apply plugin: 'org.owasp.dependencycheck'
|
|
|
|
apply plugin: 'com.github.ben-manes.versions'
|
|
|
|
|
|
|
|
dependencyUpdates {
|
|
|
|
revision="release"
|
|
|
|
resolutionStrategy = {
|
|
|
|
componentSelection { rules ->
|
|
|
|
rules.all { ComponentSelection selection ->
|
|
|
|
boolean rejected = ['snap', 'alpha', 'beta', 'rc', 'cr', 'm'].any { qualifier ->
|
|
|
|
selection.candidate.version ==~ /(?i).*[.-]${qualifier}[.\d-]*/
|
|
|
|
}
|
|
|
|
if (rejected) {
|
|
|
|
selection.reject('Release candidate')
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
configurations {
|
|
|
|
runtime {
|
|
|
|
resolutionStrategy {
|
|
|
|
force "com.fasterxml.jackson.core:jackson-annotations:$versions.jackson"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
tasks.withType(Javadoc) {
|
|
|
|
// disable the crazy super-strict doclint tool in Java 8
|
|
|
|
// noinspection SpellCheckingInspection
|
|
|
|
options.addStringOption('Xdoclint:none', '-quiet')
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
ext {
|
|
|
|
gradleVersion = "4.10.2"
|
|
|
|
minJavaVersion = "8"
|
|
|
|
buildVersionFileName = "kafka-version.properties"
|
|
|
|
|
|
|
|
userMaxForks = project.hasProperty('maxParallelForks') ? maxParallelForks.toInteger() : null
|
|
|
|
|
|
|
|
skipSigning = project.hasProperty('skipSigning') && skipSigning.toBoolean()
|
|
|
|
shouldSign = !skipSigning && !version.endsWith("SNAPSHOT") && project.gradle.startParameter.taskNames.any { it.contains("upload") }
|
|
|
|
|
|
|
|
mavenUrl = project.hasProperty('mavenUrl') ? project.mavenUrl : ''
|
|
|
|
mavenUsername = project.hasProperty('mavenUsername') ? project.mavenUsername : ''
|
|
|
|
mavenPassword = project.hasProperty('mavenPassword') ? project.mavenPassword : ''
|
|
|
|
|
|
|
|
userShowStandardStreams = project.hasProperty("showStandardStreams") ? showStandardStreams : null
|
|
|
|
|
|
|
|
userTestLoggingEvents = project.hasProperty("testLoggingEvents") ? Arrays.asList(testLoggingEvents.split(",")) : null
|
|
|
|
|
|
|
|
generatedDocsDir = new File("${project.rootDir}/docs/generated")
|
|
|
|
|
|
|
|
commitId = project.hasProperty('commitId') ? commitId : null
|
|
|
|
}
|
|
|
|
|
|
|
|
apply from: file('wrapper.gradle')
|
|
|
|
|
|
|
|
if (file('.git').exists()) {
|
|
|
|
apply from: file('gradle/rat.gradle')
|
|
|
|
rat {
|
|
|
|
// Exclude everything under the directory that git should be ignoring via .gitignore or that isn't checked in. These
|
|
|
|
// restrict us only to files that are checked in or are staged.
|
|
|
|
def repo = Grgit.open(project.getRootDir())
|
|
|
|
excludes = new ArrayList<String>(repo.clean(ignore: false, directories: true, dryRun: true))
|
|
|
|
// And some of the files that we have checked in should also be excluded from this check
|
|
|
|
excludes.addAll([
|
|
|
|
'**/.git/**',
|
|
|
|
'**/build/**',
|
|
|
|
'CONTRIBUTING.md',
|
|
|
|
'PULL_REQUEST_TEMPLATE.md',
|
|
|
|
'gradlew',
|
|
|
|
'gradlew.bat',
|
|
|
|
'TROGDOR.md',
|
|
|
|
'**/README.md',
|
|
|
|
'**/id_rsa',
|
|
|
|
'**/id_rsa.pub',
|
|
|
|
'checkstyle/suppressions.xml',
|
|
|
|
'streams/quickstart/java/src/test/resources/projects/basic/goal.txt',
|
|
|
|
'streams/streams-scala/logs/*'
|
|
|
|
])
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
subprojects {
|
|
|
|
apply plugin: 'java'
|
|
|
|
// apply the eclipse plugin only to subprojects that hold code. 'connect' is just a folder.
|
|
|
|
if (!project.name.equals('connect')) {
|
|
|
|
apply plugin: 'eclipse'
|
|
|
|
fineTuneEclipseClasspathFile(eclipse, project)
|
|
|
|
}
|
|
|
|
apply plugin: 'maven'
|
|
|
|
apply plugin: 'signing'
|
|
|
|
apply plugin: 'checkstyle'
|
|
|
|
if (!JavaVersion.current().isJava11Compatible())
|
|
|
|
apply plugin: "com.github.spotbugs"
|
|
|
|
|
|
|
|
sourceCompatibility = minJavaVersion
|
|
|
|
targetCompatibility = minJavaVersion
|
|
|
|
|
|
|
|
compileJava {
|
|
|
|
options.encoding = 'UTF-8'
|
|
|
|
options.compilerArgs << "-Xlint:deprecation,unchecked"
|
|
|
|
// --release is the recommended way to select the target release, but it's only supported in Java 9 so we also
|
|
|
|
// set --source and --target via `sourceCompatibility` and `targetCompatibility`. If/when Gradle supports `--release`
|
|
|
|
// natively (https://github.com/gradle/gradle/issues/2510), we should switch to that.
|
|
|
|
if (JavaVersion.current().isJava9Compatible())
|
|
|
|
options.compilerArgs << "--release" << minJavaVersion
|
|
|
|
}
|
|
|
|
|
|
|
|
uploadArchives {
|
|
|
|
repositories {
|
|
|
|
signing {
|
|
|
|
required { shouldSign }
|
|
|
|
sign configurations.archives
|
|
|
|
|
|
|
|
// To test locally, replace mavenUrl in ~/.gradle/gradle.properties to file://localhost/tmp/myRepo/
|
|
|
|
mavenDeployer {
|
|
|
|
beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) }
|
|
|
|
repository(url: "${mavenUrl}") {
|
|
|
|
authentication(userName: "${mavenUsername}", password: "${mavenPassword}")
|
|
|
|
}
|
|
|
|
afterEvaluate {
|
|
|
|
pom.artifactId = "${archivesBaseName}"
|
|
|
|
pom.project {
|
|
|
|
name 'Apache Kafka'
|
|
|
|
packaging 'jar'
|
|
|
|
url 'http://kafka.apache.org'
|
|
|
|
licenses {
|
|
|
|
license {
|
|
|
|
name 'The Apache Software License, Version 2.0'
|
|
|
|
url 'http://www.apache.org/licenses/LICENSE-2.0.txt'
|
|
|
|
distribution 'repo'
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
def testLoggingEvents = ["passed", "skipped", "failed"]
|
|
|
|
def testShowStandardStreams = false
|
|
|
|
def testExceptionFormat = 'full'
|
|
|
|
|
|
|
|
test {
|
|
|
|
maxParallelForks = userMaxForks ?: Runtime.runtime.availableProcessors()
|
|
|
|
|
|
|
|
minHeapSize = "256m"
|
|
|
|
maxHeapSize = "2048m"
|
|
|
|
|
|
|
|
testLogging {
|
|
|
|
events = userTestLoggingEvents ?: testLoggingEvents
|
|
|
|
showStandardStreams = userShowStandardStreams ?: testShowStandardStreams
|
|
|
|
exceptionFormat = testExceptionFormat
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
task integrationTest(type: Test, dependsOn: compileJava) {
|
|
|
|
maxParallelForks = userMaxForks ?: Runtime.runtime.availableProcessors()
|
|
|
|
|
|
|
|
minHeapSize = "256m"
|
|
|
|
maxHeapSize = "2048m"
|
|
|
|
|
|
|
|
testLogging {
|
|
|
|
events = userTestLoggingEvents ?: testLoggingEvents
|
|
|
|
showStandardStreams = userShowStandardStreams ?: testShowStandardStreams
|
|
|
|
exceptionFormat = testExceptionFormat
|
|
|
|
}
|
|
|
|
|
|
|
|
useJUnit {
|
|
|
|
includeCategories 'org.apache.kafka.test.IntegrationTest'
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
task unitTest(type: Test, dependsOn: compileJava) {
|
|
|
|
maxParallelForks = userMaxForks ?: Runtime.runtime.availableProcessors()
|
|
|
|
|
|
|
|
minHeapSize = "256m"
|
|
|
|
maxHeapSize = "2048m"
|
|
|
|
|
|
|
|
testLogging {
|
|
|
|
events = userTestLoggingEvents ?: testLoggingEvents
|
|
|
|
showStandardStreams = userShowStandardStreams ?: testShowStandardStreams
|
|
|
|
exceptionFormat = testExceptionFormat
|
|
|
|
}
|
|
|
|
|
|
|
|
useJUnit {
|
|
|
|
excludeCategories 'org.apache.kafka.test.IntegrationTest'
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
jar {
|
|
|
|
from "$rootDir/LICENSE"
|
|
|
|
from "$rootDir/NOTICE"
|
|
|
|
}
|
|
|
|
|
|
|
|
task srcJar(type: Jar) {
|
|
|
|
classifier = 'sources'
|
|
|
|
from "$rootDir/LICENSE"
|
|
|
|
from "$rootDir/NOTICE"
|
|
|
|
from sourceSets.main.allSource
|
|
|
|
}
|
|
|
|
|
|
|
|
task javadocJar(type: Jar, dependsOn: javadoc) {
|
|
|
|
classifier 'javadoc'
|
|
|
|
from "$rootDir/LICENSE"
|
|
|
|
from "$rootDir/NOTICE"
|
|
|
|
from javadoc.destinationDir
|
|
|
|
}
|
|
|
|
|
|
|
|
task docsJar(dependsOn: javadocJar)
|
|
|
|
|
|
|
|
javadoc {
|
|
|
|
options.charSet = 'UTF-8'
|
|
|
|
options.docEncoding = 'UTF-8'
|
|
|
|
options.encoding = 'UTF-8'
|
|
|
|
}
|
|
|
|
|
|
|
|
task systemTestLibs(dependsOn: jar)
|
|
|
|
|
|
|
|
artifacts {
|
|
|
|
archives srcJar
|
|
|
|
archives javadocJar
|
|
|
|
}
|
|
|
|
|
|
|
|
if(!sourceSets.test.allSource.isEmpty()) {
|
|
|
|
task testJar(type: Jar) {
|
|
|
|
classifier = 'test'
|
|
|
|
from "$rootDir/LICENSE"
|
|
|
|
from "$rootDir/NOTICE"
|
|
|
|
from sourceSets.test.output
|
|
|
|
}
|
|
|
|
|
|
|
|
task testSrcJar(type: Jar, dependsOn: testJar) {
|
|
|
|
classifier = 'test-sources'
|
|
|
|
from "$rootDir/LICENSE"
|
|
|
|
from "$rootDir/NOTICE"
|
|
|
|
from sourceSets.test.allSource
|
|
|
|
}
|
|
|
|
|
|
|
|
artifacts {
|
|
|
|
archives testJar
|
|
|
|
archives testSrcJar
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
plugins.withType(ScalaPlugin) {
|
|
|
|
task scaladocJar(type:Jar) {
|
|
|
|
classifier = 'scaladoc'
|
|
|
|
from "$rootDir/LICENSE"
|
|
|
|
from "$rootDir/NOTICE"
|
|
|
|
from scaladoc.destinationDir
|
|
|
|
}
|
|
|
|
|
|
|
|
//documentation task should also trigger building scala doc jar
|
|
|
|
docsJar.dependsOn scaladocJar
|
|
|
|
|
|
|
|
artifacts {
|
|
|
|
archives scaladocJar
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
tasks.withType(ScalaCompile) {
|
|
|
|
scalaCompileOptions.additionalParameters = [
|
|
|
|
"-deprecation",
|
|
|
|
"-unchecked",
|
|
|
|
"-encoding", "utf8",
|
|
|
|
"-Xlog-reflective-calls",
|
|
|
|
"-feature",
|
|
|
|
"-language:postfixOps",
|
|
|
|
"-language:implicitConversions",
|
|
|
|
"-language:existentials",
|
|
|
|
"-Xlint:by-name-right-associative",
|
|
|
|
"-Xlint:delayedinit-select",
|
|
|
|
"-Xlint:doc-detached",
|
|
|
|
"-Xlint:missing-interpolator",
|
|
|
|
"-Xlint:nullary-override",
|
|
|
|
"-Xlint:nullary-unit",
|
|
|
|
"-Xlint:option-implicit",
|
|
|
|
"-Xlint:package-object-classes",
|
|
|
|
"-Xlint:poly-implicit-overload",
|
|
|
|
"-Xlint:private-shadow",
|
|
|
|
"-Xlint:stars-align",
|
|
|
|
"-Xlint:type-parameter-shadow",
|
|
|
|
"-Xlint:unsound-match"
|
|
|
|
]
|
|
|
|
|
|
|
|
if (versions.baseScala != '2.11') {
|
|
|
|
scalaCompileOptions.additionalParameters += [
|
|
|
|
"-Xlint:constant",
|
|
|
|
"-Xlint:unused"
|
|
|
|
]
|
|
|
|
}
|
|
|
|
|
|
|
|
configure(scalaCompileOptions.forkOptions) {
|
|
|
|
memoryMaximumSize = '1g'
|
|
|
|
jvmArgs = ['-Xss2m']
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
checkstyle {
|
|
|
|
configFile = new File(rootDir, "checkstyle/checkstyle.xml")
|
|
|
|
configProperties = checkstyleConfigProperties("import-control.xml")
|
|
|
|
toolVersion = '8.10'
|
|
|
|
}
|
|
|
|
test.dependsOn('checkstyleMain', 'checkstyleTest')
|
|
|
|
|
|
|
|
if (!JavaVersion.current().isJava11Compatible()) {
|
|
|
|
spotbugs {
|
|
|
|
toolVersion = '3.1.8'
|
|
|
|
excludeFilter = file("$rootDir/gradle/spotbugs-exclude.xml")
|
|
|
|
ignoreFailures = false
|
|
|
|
}
|
|
|
|
test.dependsOn('spotbugsMain')
|
|
|
|
|
|
|
|
tasks.withType(com.github.spotbugs.SpotBugsTask) {
|
|
|
|
reports {
|
|
|
|
// Continue supporting `xmlFindBugsReport` for compatibility
|
|
|
|
xml.enabled(project.hasProperty('xmlSpotBugsReport') || project.hasProperty('xmlFindBugsReport'))
|
|
|
|
html.enabled(!project.hasProperty('xmlSpotBugsReport') && !project.hasProperty('xmlFindBugsReport'))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Ignore core since its a scala project
|
|
|
|
if (it.path != ':core') {
|
|
|
|
apply plugin: "jacoco"
|
|
|
|
|
|
|
|
jacoco {
|
|
|
|
toolVersion = "0.8.2"
|
|
|
|
}
|
|
|
|
|
|
|
|
// NOTE: Jacoco Gradle plugin does not support "offline instrumentation" this means that classes mocked by PowerMock
|
|
|
|
// may report 0 coverage, since the source was modified after initial instrumentation.
|
|
|
|
// See https://github.com/jacoco/jacoco/issues/51
|
|
|
|
jacocoTestReport {
|
|
|
|
dependsOn tasks.test
|
|
|
|
sourceSets sourceSets.main
|
|
|
|
reports {
|
|
|
|
html.enabled = true
|
|
|
|
xml.enabled = true
|
|
|
|
csv.enabled = false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
def coverageGen = it.path == ':core' ? 'reportScoverage' : 'jacocoTestReport'
|
|
|
|
task reportCoverage(dependsOn: [coverageGen])
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
gradle.taskGraph.whenReady { taskGraph ->
|
|
|
|
taskGraph.getAllTasks().findAll { it.name.contains('spotbugsScoverage') || it.name.contains('spotbugsTest') }.each { task ->
|
|
|
|
task.enabled = false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
def fineTuneEclipseClasspathFile(eclipse, project) {
|
|
|
|
eclipse.classpath.file {
|
|
|
|
beforeMerged { cp ->
|
|
|
|
cp.entries.clear()
|
|
|
|
// for the core project add the directories defined under test/scala as separate source directories
|
|
|
|
if (project.name.equals('core')) {
|
|
|
|
cp.entries.add(new org.gradle.plugins.ide.eclipse.model.SourceFolder("src/test/scala/integration", null))
|
|
|
|
cp.entries.add(new org.gradle.plugins.ide.eclipse.model.SourceFolder("src/test/scala/other", null))
|
|
|
|
cp.entries.add(new org.gradle.plugins.ide.eclipse.model.SourceFolder("src/test/scala/unit", null))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
whenMerged { cp ->
|
|
|
|
// for the core project exclude the separate sub-directories defined under test/scala. These are added as source dirs above
|
|
|
|
if (project.name.equals('core')) {
|
|
|
|
cp.entries.findAll { it.kind == "src" && it.path.equals("src/test/scala") }*.excludes = ["integration/", "other/", "unit/"]
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* Set all eclipse build output to go to 'build_eclipse' directory. This is to ensure that gradle and eclipse use different
|
|
|
|
* build output directories, and also avoid using the eclpise default of 'bin' which clashes with some of our script directories.
|
|
|
|
* https://discuss.gradle.org/t/eclipse-generated-files-should-be-put-in-the-same-place-as-the-gradle-generated-files/6986/2
|
|
|
|
*/
|
|
|
|
cp.entries.findAll { it.kind == "output" }*.path = "build_eclipse"
|
|
|
|
/*
|
|
|
|
* Some projects have explicitly added test output dependencies. These are required for the gradle build but not required
|
|
|
|
* in Eclipse since the dependent projects are added as dependencies. So clean up these from the generated classpath.
|
|
|
|
*/
|
|
|
|
cp.entries.removeAll { it.kind == "lib" && it.path.matches(".*/build/(classes|resources)/test") }
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
def checkstyleConfigProperties(configFileName) {
|
|
|
|
[importControlFile: "$rootDir/checkstyle/$configFileName",
|
|
|
|
suppressionsFile: "$rootDir/checkstyle/suppressions.xml",
|
|
|
|
headerFile: "$rootDir/checkstyle/java.header"]
|
|
|
|
}
|
|
|
|
|
|
|
|
// Aggregates all jacoco results into the root project directory
|
|
|
|
task jacocoRootReport(type: org.gradle.testing.jacoco.tasks.JacocoReport) {
|
|
|
|
def javaProjects = subprojects.findAll { it.path != ':core' }
|
|
|
|
|
|
|
|
description = 'Generates an aggregate report from all subprojects'
|
|
|
|
dependsOn(javaProjects.test)
|
|
|
|
|
|
|
|
additionalSourceDirs = files(javaProjects.sourceSets.main.allSource.srcDirs)
|
|
|
|
sourceDirectories = files(javaProjects.sourceSets.main.allSource.srcDirs)
|
|
|
|
classDirectories = files(javaProjects.sourceSets.main.output)
|
|
|
|
executionData = files(javaProjects.jacocoTestReport.executionData)
|
|
|
|
|
|
|
|
reports {
|
|
|
|
html.enabled = true
|
|
|
|
xml.enabled = true
|
|
|
|
}
|
|
|
|
|
|
|
|
// workaround to ignore projects that don't have any tests at all
|
|
|
|
onlyIf = { true }
|
|
|
|
doFirst {
|
|
|
|
executionData = files(executionData.findAll { it.exists() })
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
task reportCoverage(dependsOn: ['jacocoRootReport', 'core:reportCoverage'])
|
|
|
|
|
|
|
|
for ( sv in availableScalaVersions ) {
|
|
|
|
String taskSuffix = sv.replaceAll("\\.", "_")
|
|
|
|
|
|
|
|
tasks.create(name: "jarScala_${taskSuffix}", type: GradleBuild) {
|
|
|
|
startParameter = project.getGradle().getStartParameter().newInstance()
|
|
|
|
startParameter.projectProperties += [scalaVersion: "${sv}"]
|
|
|
|
tasks = ['core:jar', 'streams:streams-scala:jar']
|
|
|
|
}
|
|
|
|
|
|
|
|
tasks.create(name: "testScala_${taskSuffix}", type: GradleBuild) {
|
|
|
|
startParameter = project.getGradle().getStartParameter().newInstance()
|
|
|
|
startParameter.projectProperties += [scalaVersion: "${sv}"]
|
|
|
|
tasks = ['core:test', 'streams:streams-scala:test']
|
|
|
|
}
|
|
|
|
|
|
|
|
tasks.create(name: "srcJar_${taskSuffix}", type: GradleBuild) {
|
|
|
|
startParameter = project.getGradle().getStartParameter().newInstance()
|
|
|
|
startParameter.projectProperties += [scalaVersion: "${sv}"]
|
|
|
|
tasks = ['core:srcJar', 'streams:streams-scala:srcJar']
|
|
|
|
}
|
|
|
|
|
|
|
|
tasks.create(name: "docsJar_${taskSuffix}", type: GradleBuild) {
|
|
|
|
startParameter = project.getGradle().getStartParameter().newInstance()
|
|
|
|
startParameter.projectProperties += [scalaVersion: "${sv}"]
|
|
|
|
tasks = ['core:docsJar', 'streams:streams-scala:docsJar']
|
|
|
|
}
|
|
|
|
|
|
|
|
tasks.create(name: "install_${taskSuffix}", type: GradleBuild) {
|
|
|
|
startParameter = project.getGradle().getStartParameter().newInstance()
|
|
|
|
startParameter.projectProperties += [scalaVersion: "${sv}"]
|
|
|
|
tasks = ['install']
|
|
|
|
}
|
|
|
|
|
|
|
|
tasks.create(name: "releaseTarGz_${taskSuffix}", type: GradleBuild) {
|
|
|
|
startParameter = project.getGradle().getStartParameter().newInstance()
|
|
|
|
startParameter.projectProperties += [scalaVersion: "${sv}"]
|
|
|
|
tasks = ['releaseTarGz']
|
|
|
|
}
|
|
|
|
|
|
|
|
tasks.create(name: "uploadScalaArchives_${taskSuffix}", type: GradleBuild) {
|
|
|
|
startParameter = project.getGradle().getStartParameter().newInstance()
|
|
|
|
startParameter.projectProperties += [scalaVersion: "${sv}"]
|
|
|
|
tasks = ['core:uploadArchives', 'streams:streams-scala:uploadArchives']
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
def connectPkgs = ['connect:api', 'connect:runtime', 'connect:transforms', 'connect:json', 'connect:file', 'connect:basic-auth-extension']
|
|
|
|
def pkgs = ['clients', 'examples', 'log4j-appender', 'tools', 'streams', 'streams:streams-scala', 'streams:test-utils', 'streams:examples'] + connectPkgs
|
KAFKA-2366; Initial patch for Copycat
This is an initial patch implementing the basics of Copycat for KIP-26.
The intent here is to start a review of the key pieces of the core API and get a reasonably functional, baseline, non-distributed implementation of Copycat in place to get things rolling. The current patch has a number of known issues that need to be addressed before a final version:
* Some build-related issues. Specifically, requires some locally-installed dependencies (see below), ignores checkstyle for the runtime data library because it's lifted from Avro currently and likely won't last in its current form, and some Gradle task dependencies aren't quite right because I haven't gotten rid of the dependency on `core` (which should now be an easy patch since new consumer groups are in a much better state).
* This patch currently depends on some Confluent trunk code because I prototyped with our Avro serializers w/ schema-registry support. We need to figure out what we want to provide as an example built-in set of serializers. Unlike core Kafka where we could ignore the issue, providing only ByteArray or String serializers, this is pretty central to how Copycat works.
* This patch uses a hacked up version of Avro as its runtime data format. Not sure if we want to go through the entire API discussion just to get some basic code committed, so I filed KAFKA-2367 to handle that separately. The core connector APIs and the runtime data APIs are entirely orthogonal.
* This patch needs some updates to get aligned with recent new consumer changes (specifically, I'm aware of the ConcurrentModificationException issue on exit). More generally, the new consumer is in flux but Copycat depends on it, so there are likely to be some negative interactions.
* The layout feels a bit awkward to me right now because I ported it from a Maven layout. We don't have nearly the same level of granularity in Kafka currently (core and clients, plus the mostly ignored examples, log4j-appender, and a couple of contribs). We might want to reorganize, although keeping data+api separate from runtime and connector plugins is useful for minimizing dependencies.
* There are a variety of other things (e.g., I'm not happy with the exception hierarchy/how they are currently handled, TopicPartition doesn't really need to be duplicated unless we want Copycat entirely isolated from the Kafka APIs, etc), but I expect those we'll cover in the review.
Before commenting on the patch, it's probably worth reviewing https://issues.apache.org/jira/browse/KAFKA-2365 and https://issues.apache.org/jira/browse/KAFKA-2366 to get an idea of what I had in mind for a) what we ultimately want with all the Copycat patches and b) what we aim to cover in this initial patch. My hope is that we can use a WIP patch (after the current obvious deficiencies are addressed) while recognizing that we want to make iterative progress with a bunch of subsequent PRs.
Author: Ewen Cheslack-Postava <me@ewencp.org>
Reviewers: Ismael Juma, Gwen Shapira
Closes #99 from ewencp/copycat and squashes the following commits:
a3a47a6 [Ewen Cheslack-Postava] Simplify Copycat exceptions, make them a subclass of KafkaException.
8c108b0 [Ewen Cheslack-Postava] Rename Coordinator to Herder to avoid confusion with the consumer coordinator.
7bf8075 [Ewen Cheslack-Postava] Make Copycat CLI speific to standalone mode, clean up some config and get rid of config storage in standalone mode.
656a003 [Ewen Cheslack-Postava] Clarify and expand the explanation of the Copycat Coordinator interface.
c0e5fdc [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
0fa7a36 [Ewen Cheslack-Postava] Mark Copycat classes as unstable and reduce visibility of some classes where possible.
d55d31e [Ewen Cheslack-Postava] Reorganize Copycat code to put it all under one top-level directory.
b29cb2c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
d713a21 [Ewen Cheslack-Postava] Address Gwen's review comments.
6787a85 [Ewen Cheslack-Postava] Make Converter generic to match serializers since some serialization formats do not require a base class of Object; update many other classes to have generic key and value class type parameters to match this change.
b194c73 [Ewen Cheslack-Postava] Split Copycat converter option into two options for key and value.
0b5a1a0 [Ewen Cheslack-Postava] Normalize naming to use partition for both source and Kafka, adjusting naming in CopycatRecord classes to clearly differentiate.
e345142 [Ewen Cheslack-Postava] Remove Copycat reflection utils, use existing Utils and ConfigDef functionality from clients package.
be5c387 [Ewen Cheslack-Postava] Minor cleanup
122423e [Ewen Cheslack-Postava] Style cleanup
6ba87de [Ewen Cheslack-Postava] Remove most of the Avro-based mock runtime data API, only preserving enough schema functionality to support basic primitive types for an initial patch.
4674d13 [Ewen Cheslack-Postava] Address review comments, clean up some code styling.
25b5739 [Ewen Cheslack-Postava] Fix sink task offset commit concurrency issue by moving it to the worker thread and waking up the consumer to ensure it exits promptly.
0aefe21 [Ewen Cheslack-Postava] Add log4j settings for Copycat.
220e42d [Ewen Cheslack-Postava] Replace Avro serializer with JSON serializer.
1243a7c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
5a618c6 [Ewen Cheslack-Postava] Remove offset serializers, instead reusing the existing serializers and removing schema projection support.
e849e10 [Ewen Cheslack-Postava] Remove duplicated TopicPartition implementation.
dec1379 [Ewen Cheslack-Postava] Switch to using new consumer coordinator instead of manually assigning partitions. Remove dependency of copycat-runtime on core.
4a9b4f3 [Ewen Cheslack-Postava] Add some helpful Copycat-specific build and test targets that cover all Copycat packages.
31cd1ca [Ewen Cheslack-Postava] Add CLI tools for Copycat.
e14942c [Ewen Cheslack-Postava] Add Copycat file connector.
0233456 [Ewen Cheslack-Postava] Add copycat-avro and copycat-runtime
11981d2 [Ewen Cheslack-Postava] Add copycat-data and copycat-api
9 years ago
|
|
|
|
|
|
|
/** Create one task per default Scala version */
|
|
|
|
def withDefScalaVersions(taskName) {
|
|
|
|
defaultScalaVersions.collect { taskName + '_' + it.replaceAll('\\.', '_') }
|
|
|
|
}
|
|
|
|
|
|
|
|
tasks.create(name: "jarConnect", dependsOn: connectPkgs.collect { it + ":jar" }) {}
|
|
|
|
tasks.create(name: "jarAll", dependsOn: withDefScalaVersions('jarScala') + pkgs.collect { it + ":jar" }) { }
|
|
|
|
|
|
|
|
tasks.create(name: "srcJarAll", dependsOn: withDefScalaVersions('srcJar') + pkgs.collect { it + ":srcJar" }) { }
|
|
|
|
|
|
|
|
tasks.create(name: "docsJarAll", dependsOn: withDefScalaVersions('docsJar') + pkgs.collect { it + ":docsJar" }) { }
|
|
|
|
|
|
|
|
tasks.create(name: "testConnect", dependsOn: connectPkgs.collect { it + ":test" }) {}
|
|
|
|
tasks.create(name: "testAll", dependsOn: withDefScalaVersions('testScala') + pkgs.collect { it + ":test" }) { }
|
|
|
|
|
|
|
|
tasks.create(name: "installAll", dependsOn: withDefScalaVersions('install') + pkgs.collect { it + ":install" }) { }
|
|
|
|
|
|
|
|
tasks.create(name: "releaseTarGzAll", dependsOn: withDefScalaVersions('releaseTarGz')) { }
|
|
|
|
|
|
|
|
tasks.create(name: "uploadArchivesAll", dependsOn: withDefScalaVersions('uploadScalaArchives') + pkgs.collect { it + ":uploadArchives" }) { }
|
|
|
|
|
|
|
|
project(':core') {
|
|
|
|
println "Building project 'core' with Scala version ${versions.scala}"
|
|
|
|
|
|
|
|
apply plugin: 'scala'
|
|
|
|
apply plugin: "org.scoverage"
|
|
|
|
archivesBaseName = "kafka_${versions.baseScala}"
|
|
|
|
|
|
|
|
dependencies {
|
|
|
|
compile project(':clients')
|
|
|
|
compile libs.jacksonDatabind
|
|
|
|
compile libs.joptSimple
|
|
|
|
compile libs.metrics
|
|
|
|
compile libs.scalaLibrary
|
|
|
|
// only needed transitively, but set it explicitly to ensure it has the same version as scala-library
|
|
|
|
compile libs.scalaReflect
|
|
|
|
compile libs.scalaLogging
|
|
|
|
compile libs.slf4jApi
|
|
|
|
compile(libs.zkclient) {
|
|
|
|
exclude module: 'zookeeper'
|
|
|
|
}
|
|
|
|
compile(libs.zookeeper) {
|
|
|
|
exclude module: 'slf4j-log4j12'
|
|
|
|
exclude module: 'log4j'
|
|
|
|
exclude module: 'netty'
|
|
|
|
}
|
|
|
|
|
|
|
|
compileOnly libs.log4j
|
|
|
|
|
|
|
|
testCompile project(':clients').sourceSets.test.output
|
|
|
|
testCompile libs.bcpkix
|
|
|
|
testCompile libs.mockitoCore
|
|
|
|
testCompile libs.easymock
|
|
|
|
testCompile(libs.apacheda) {
|
|
|
|
exclude group: 'xml-apis', module: 'xml-apis'
|
|
|
|
// `mina-core` is a transitive dependency for `apacheds` and `apacheda`.
|
|
|
|
// It is safer to use from `apacheds` since that is the implementation.
|
|
|
|
exclude module: 'mina-core'
|
|
|
|
}
|
|
|
|
testCompile libs.apachedsCoreApi
|
|
|
|
testCompile libs.apachedsInterceptorKerberos
|
|
|
|
testCompile libs.apachedsProtocolShared
|
|
|
|
testCompile libs.apachedsProtocolKerberos
|
|
|
|
testCompile libs.apachedsProtocolLdap
|
|
|
|
testCompile libs.apachedsLdifPartition
|
|
|
|
testCompile libs.apachedsMavibotPartition
|
|
|
|
testCompile libs.apachedsJdbmPartition
|
|
|
|
testCompile libs.junit
|
|
|
|
testCompile libs.scalatest
|
|
|
|
testCompile libs.slf4jlog4j
|
|
|
|
testCompile libs.jfreechart
|
|
|
|
|
|
|
|
scoverage libs.scoveragePlugin
|
|
|
|
scoverage libs.scoverageRuntime
|
|
|
|
}
|
|
|
|
|
|
|
|
scoverage {
|
|
|
|
reportDir = file("${rootProject.buildDir}/scoverage")
|
|
|
|
highlighting = false
|
|
|
|
}
|
|
|
|
checkScoverage {
|
|
|
|
minimumRate = 0.0
|
|
|
|
}
|
|
|
|
checkScoverage.shouldRunAfter('test')
|
|
|
|
|
|
|
|
configurations {
|
|
|
|
// manually excludes some unnecessary dependencies
|
|
|
|
compile.exclude module: 'javax'
|
|
|
|
compile.exclude module: 'jline'
|
|
|
|
compile.exclude module: 'jms'
|
|
|
|
compile.exclude module: 'jmxri'
|
|
|
|
compile.exclude module: 'jmxtools'
|
|
|
|
compile.exclude module: 'mail'
|
|
|
|
// To prevent a UniqueResourceException due the same resource existing in both
|
|
|
|
// org.apache.directory.api/api-all and org.apache.directory.api/api-ldap-schema-data
|
|
|
|
testCompile.exclude module: 'api-ldap-schema-data'
|
|
|
|
}
|
|
|
|
|
|
|
|
tasks.create(name: "copyDependantLibs", type: Copy) {
|
|
|
|
from (configurations.testRuntime) {
|
|
|
|
include('slf4j-log4j12*')
|
|
|
|
include('log4j*jar')
|
|
|
|
}
|
|
|
|
from (configurations.runtime) {
|
|
|
|
exclude('kafka-clients*')
|
|
|
|
}
|
|
|
|
into "$buildDir/dependant-libs-${versions.scala}"
|
|
|
|
duplicatesStrategy 'exclude'
|
|
|
|
}
|
|
|
|
|
|
|
|
task genProtocolErrorDocs(type: JavaExec) {
|
|
|
|
classpath = sourceSets.main.runtimeClasspath
|
|
|
|
main = 'org.apache.kafka.common.protocol.Errors'
|
|
|
|
if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
|
|
|
|
standardOutput = new File(generatedDocsDir, "protocol_errors.html").newOutputStream()
|
|
|
|
}
|
|
|
|
|
|
|
|
task genProtocolTypesDocs(type: JavaExec) {
|
|
|
|
classpath = sourceSets.main.runtimeClasspath
|
|
|
|
main = 'org.apache.kafka.common.protocol.types.Type'
|
|
|
|
if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
|
|
|
|
standardOutput = new File(generatedDocsDir, "protocol_types.html").newOutputStream()
|
|
|
|
}
|
|
|
|
|
|
|
|
task genProtocolApiKeyDocs(type: JavaExec) {
|
|
|
|
classpath = sourceSets.main.runtimeClasspath
|
|
|
|
main = 'org.apache.kafka.common.protocol.ApiKeys'
|
|
|
|
if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
|
|
|
|
standardOutput = new File(generatedDocsDir, "protocol_api_keys.html").newOutputStream()
|
|
|
|
}
|
|
|
|
|
|
|
|
task genProtocolMessageDocs(type: JavaExec) {
|
|
|
|
classpath = sourceSets.main.runtimeClasspath
|
|
|
|
main = 'org.apache.kafka.common.protocol.Protocol'
|
|
|
|
if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
|
|
|
|
standardOutput = new File(generatedDocsDir, "protocol_messages.html").newOutputStream()
|
|
|
|
}
|
|
|
|
|
|
|
|
task genAdminClientConfigDocs(type: JavaExec) {
|
|
|
|
classpath = sourceSets.main.runtimeClasspath
|
|
|
|
main = 'org.apache.kafka.clients.admin.AdminClientConfig'
|
|
|
|
if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
|
|
|
|
standardOutput = new File(generatedDocsDir, "admin_client_config.html").newOutputStream()
|
|
|
|
}
|
|
|
|
|
|
|
|
task genProducerConfigDocs(type: JavaExec) {
|
|
|
|
classpath = sourceSets.main.runtimeClasspath
|
|
|
|
main = 'org.apache.kafka.clients.producer.ProducerConfig'
|
|
|
|
if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
|
|
|
|
standardOutput = new File(generatedDocsDir, "producer_config.html").newOutputStream()
|
|
|
|
}
|
|
|
|
|
|
|
|
task genConsumerConfigDocs(type: JavaExec) {
|
|
|
|
classpath = sourceSets.main.runtimeClasspath
|
|
|
|
main = 'org.apache.kafka.clients.consumer.ConsumerConfig'
|
|
|
|
if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
|
|
|
|
standardOutput = new File(generatedDocsDir, "consumer_config.html").newOutputStream()
|
|
|
|
}
|
|
|
|
|
|
|
|
task genKafkaConfigDocs(type: JavaExec) {
|
|
|
|
classpath = sourceSets.main.runtimeClasspath
|
|
|
|
main = 'kafka.server.KafkaConfig'
|
|
|
|
if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
|
|
|
|
standardOutput = new File(generatedDocsDir, "kafka_config.html").newOutputStream()
|
|
|
|
}
|
|
|
|
|
KAFKA-3809: Auto-generate documentation for topic-level configuration
ijuma said that it would make sense to split out this work from KAFKA-3234, since KAFKA-3234 had both a mechanical change (generating docs) as well as a change requiring discussion (deprecating/renaming config options).
jjkoshy, I hope you don't mind that I took over this work. It's been 3 months since the last activity on KAFKA-3234, so I thought it would be okay to take over.
This work is essentially is the first 5-6 commits from Joel's https://github.com/apache/kafka/pull/907. However, since I'm not very experienced with git, I didn't do a direct merge/rebase, but instead largely hand-merged it. I did some minor cleanup. All credit goes to Joel, all blame goes to me. :)
For reference, I attached the auto-generated configuration.html file (as a PDF, because github won't let me attache html).
[configuration.pdf](https://github.com/apache/kafka/files/323901/configuration.pdf)
This is my first time writing Scala, so let me know if there are any changes needed.
I don't know who is the right person to review this. ijuma, can you help me redirect this to the appropriate person? Thanks.
Author: James Cheng <jylcheng@yahoo.com>
Reviewers: Ismael Juma <ismael@juma.me.uk>, Joel Koshy <jjkoshy@gmail.com>, Ewen Cheslack-Postava <ewen@confluent.io>
Closes #1527 from wushujames/generate_topic_docs
8 years ago
|
|
|
task genTopicConfigDocs(type: JavaExec) {
|
|
|
|
classpath = sourceSets.main.runtimeClasspath
|
|
|
|
main = 'kafka.log.LogConfig'
|
|
|
|
if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
|
|
|
|
standardOutput = new File(generatedDocsDir, "topic_config.html").newOutputStream()
|
|
|
|
}
|
|
|
|
|
|
|
|
task genConsumerMetricsDocs(type: JavaExec) {
|
|
|
|
classpath = sourceSets.test.runtimeClasspath
|
|
|
|
main = 'org.apache.kafka.clients.consumer.internals.ConsumerMetrics'
|
|
|
|
if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
|
|
|
|
standardOutput = new File(generatedDocsDir, "consumer_metrics.html").newOutputStream()
|
|
|
|
}
|
|
|
|
|
|
|
|
task genProducerMetricsDocs(type: JavaExec) {
|
|
|
|
classpath = sourceSets.test.runtimeClasspath
|
|
|
|
main = 'org.apache.kafka.clients.producer.internals.ProducerMetrics'
|
|
|
|
if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
|
|
|
|
standardOutput = new File(generatedDocsDir, "producer_metrics.html").newOutputStream()
|
|
|
|
}
|
|
|
|
|
|
|
|
task siteDocsTar(dependsOn: ['genProtocolErrorDocs', 'genProtocolTypesDocs', 'genProtocolApiKeyDocs', 'genProtocolMessageDocs',
|
|
|
|
'genAdminClientConfigDocs', 'genProducerConfigDocs', 'genConsumerConfigDocs',
|
|
|
|
'genKafkaConfigDocs', 'genTopicConfigDocs',
|
|
|
|
':connect:runtime:genConnectConfigDocs', ':connect:runtime:genConnectTransformationDocs',
|
|
|
|
':connect:runtime:genSinkConnectorConfigDocs', ':connect:runtime:genSourceConnectorConfigDocs',
|
|
|
|
':streams:genStreamsConfigDocs', 'genConsumerMetricsDocs', 'genProducerMetricsDocs',
|
|
|
|
':connect:runtime:genConnectMetricsDocs'], type: Tar) {
|
|
|
|
classifier = 'site-docs'
|
|
|
|
compression = Compression.GZIP
|
|
|
|
from project.file("$rootDir/docs")
|
|
|
|
into 'site-docs'
|
|
|
|
duplicatesStrategy 'exclude'
|
|
|
|
}
|
|
|
|
|
|
|
|
tasks.create(name: "releaseTarGz", dependsOn: configurations.archives.artifacts, type: Tar) {
|
|
|
|
into "kafka_${versions.baseScala}-${version}"
|
|
|
|
compression = Compression.GZIP
|
|
|
|
from(project.file("$rootDir/bin")) { into "bin/" }
|
|
|
|
from(project.file("$rootDir/config")) { into "config/" }
|
|
|
|
from "$rootDir/LICENSE"
|
|
|
|
from "$rootDir/NOTICE"
|
|
|
|
from(configurations.runtime) { into("libs/") }
|
|
|
|
from(configurations.archives.artifacts.files) { into("libs/") }
|
|
|
|
from(project.siteDocsTar) { into("site-docs/") }
|
|
|
|
from(project(':tools').jar) { into("libs/") }
|
|
|
|
from(project(':tools').configurations.runtime) { into("libs/") }
|
|
|
|
from(project(':connect:api').jar) { into("libs/") }
|
|
|
|
from(project(':connect:api').configurations.runtime) { into("libs/") }
|
|
|
|
from(project(':connect:runtime').jar) { into("libs/") }
|
|
|
|
from(project(':connect:runtime').configurations.runtime) { into("libs/") }
|
|
|
|
from(project(':connect:transforms').jar) { into("libs/") }
|
|
|
|
from(project(':connect:transforms').configurations.runtime) { into("libs/") }
|
|
|
|
from(project(':connect:json').jar) { into("libs/") }
|
|
|
|
from(project(':connect:json').configurations.runtime) { into("libs/") }
|
|
|
|
from(project(':connect:file').jar) { into("libs/") }
|
|
|
|
from(project(':connect:file').configurations.runtime) { into("libs/") }
|
|
|
|
from(project(':connect:basic-auth-extension').jar) { into("libs/") }
|
|
|
|
from(project(':connect:basic-auth-extension').configurations.runtime) { into("libs/") }
|
|
|
|
from(project(':streams').jar) { into("libs/") }
|
|
|
|
from(project(':streams').configurations.runtime) { into("libs/") }
|
|
|
|
from(project(':streams:streams-scala').jar) { into("libs/") }
|
|
|
|
from(project(':streams:streams-scala').configurations.runtime) { into("libs/") }
|
|
|
|
from(project(':streams:test-utils').jar) { into("libs/") }
|
|
|
|
from(project(':streams:test-utils').configurations.runtime) { into("libs/") }
|
|
|
|
from(project(':streams:examples').jar) { into("libs/") }
|
|
|
|
from(project(':streams:examples').configurations.runtime) { into("libs/") }
|
|
|
|
duplicatesStrategy 'exclude'
|
|
|
|
}
|
|
|
|
|
|
|
|
jar {
|
|
|
|
dependsOn('copyDependantLibs')
|
|
|
|
}
|
|
|
|
|
|
|
|
jar.manifest {
|
|
|
|
attributes(
|
|
|
|
'Version': "${version}"
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
|
|
|
tasks.create(name: "copyDependantTestLibs", type: Copy) {
|
|
|
|
from (configurations.testRuntime) {
|
|
|
|
include('*.jar')
|
|
|
|
}
|
|
|
|
into "$buildDir/dependant-testlibs"
|
|
|
|
//By default gradle does not handle test dependencies between the sub-projects
|
|
|
|
//This line is to include clients project test jar to dependant-testlibs
|
|
|
|
from (project(':clients').testJar ) { "$buildDir/dependant-testlibs" }
|
|
|
|
duplicatesStrategy 'exclude'
|
|
|
|
}
|
|
|
|
|
|
|
|
systemTestLibs.dependsOn('jar', 'testJar', 'copyDependantTestLibs')
|
|
|
|
|
|
|
|
checkstyle {
|
|
|
|
configProperties = checkstyleConfigProperties("import-control-core.xml")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
project(':examples') {
|
|
|
|
archivesBaseName = "kafka-examples"
|
|
|
|
|
|
|
|
dependencies {
|
|
|
|
compile project(':core')
|
|
|
|
}
|
|
|
|
|
|
|
|
javadoc {
|
|
|
|
enabled = false
|
|
|
|
}
|
|
|
|
|
|
|
|
checkstyle {
|
|
|
|
configProperties = checkstyleConfigProperties("import-control-core.xml")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
project(':clients') {
|
|
|
|
archivesBaseName = "kafka-clients"
|
|
|
|
|
|
|
|
configurations {
|
|
|
|
jacksonDatabindConfig
|
|
|
|
}
|
|
|
|
|
|
|
|
// add jacksonDatabindConfig as provided scope config with high priority (1000)
|
|
|
|
conf2ScopeMappings.addMapping(1000, configurations.jacksonDatabindConfig, "provided")
|
|
|
|
|
|
|
|
dependencies {
|
|
|
|
compile libs.zstd
|
|
|
|
compile libs.lz4
|
|
|
|
compile libs.snappy
|
|
|
|
compile libs.slf4jApi
|
|
|
|
compileOnly libs.jacksonDatabind // for SASL/OAUTHBEARER bearer token parsing
|
|
|
|
|
|
|
|
jacksonDatabindConfig libs.jacksonDatabind // to publish as provided scope dependency.
|
|
|
|
|
|
|
|
testCompile libs.bcpkix
|
|
|
|
testCompile libs.junit
|
|
|
|
testCompile libs.mockitoCore
|
|
|
|
|
|
|
|
testRuntime libs.slf4jlog4j
|
|
|
|
testRuntime libs.jacksonDatabind
|
|
|
|
}
|
|
|
|
|
|
|
|
task determineCommitId {
|
|
|
|
def takeFromHash = 16
|
|
|
|
if (commitId) {
|
|
|
|
commitId = commitId.take(takeFromHash)
|
|
|
|
} else if (file("$rootDir/.git/HEAD").exists()) {
|
|
|
|
def headRef = file("$rootDir/.git/HEAD").text
|
|
|
|
if (headRef.contains('ref: ')) {
|
|
|
|
headRef = headRef.replaceAll('ref: ', '').trim()
|
|
|
|
if (file("$rootDir/.git/$headRef").exists()) {
|
|
|
|
commitId = file("$rootDir/.git/$headRef").text.trim().take(takeFromHash)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
commitId = headRef.trim().take(takeFromHash)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
commitId = "unknown"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
task createVersionFile(dependsOn: determineCommitId) {
|
|
|
|
ext.receiptFile = file("$buildDir/kafka/$buildVersionFileName")
|
|
|
|
outputs.file receiptFile
|
|
|
|
outputs.upToDateWhen { false }
|
|
|
|
doLast {
|
|
|
|
def data = [
|
|
|
|
commitId: commitId,
|
|
|
|
version: version,
|
|
|
|
]
|
|
|
|
|
|
|
|
receiptFile.parentFile.mkdirs()
|
|
|
|
def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n")
|
|
|
|
receiptFile.setText(content, "ISO-8859-1")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
jar {
|
|
|
|
dependsOn createVersionFile
|
|
|
|
from("$buildDir") {
|
|
|
|
include "kafka/$buildVersionFileName"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
clean.doFirst {
|
|
|
|
delete "$buildDir/kafka/"
|
|
|
|
}
|
|
|
|
|
|
|
|
javadoc {
|
|
|
|
include "**/org/apache/kafka/clients/admin/*"
|
|
|
|
include "**/org/apache/kafka/clients/consumer/*"
|
|
|
|
include "**/org/apache/kafka/clients/producer/*"
|
|
|
|
include "**/org/apache/kafka/common/*"
|
|
|
|
include "**/org/apache/kafka/common/acl/*"
|
|
|
|
include "**/org/apache/kafka/common/annotation/*"
|
|
|
|
include "**/org/apache/kafka/common/errors/*"
|
|
|
|
include "**/org/apache/kafka/common/header/*"
|
|
|
|
include "**/org/apache/kafka/common/resource/*"
|
|
|
|
include "**/org/apache/kafka/common/serialization/*"
|
|
|
|
include "**/org/apache/kafka/common/config/*"
|
|
|
|
include "**/org/apache/kafka/common/config/provider/*"
|
|
|
|
include "**/org/apache/kafka/common/security/auth/*"
|
|
|
|
include "**/org/apache/kafka/common/security/plain/*"
|
|
|
|
include "**/org/apache/kafka/common/security/scram/*"
|
|
|
|
include "**/org/apache/kafka/common/security/token/delegation/*"
|
|
|
|
include "**/org/apache/kafka/common/security/oauthbearer/*"
|
|
|
|
include "**/org/apache/kafka/server/policy/*"
|
|
|
|
include "**/org/apache/kafka/server/quota/*"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
project(':tools') {
|
|
|
|
archivesBaseName = "kafka-tools"
|
|
|
|
|
|
|
|
dependencies {
|
|
|
|
compile project(':clients')
|
|
|
|
compile project(':log4j-appender')
|
|
|
|
compile libs.argparse4j
|
|
|
|
compile libs.jacksonDatabind
|
|
|
|
compile libs.slf4jApi
|
|
|
|
|
|
|
|
compile libs.jacksonJaxrsJsonProvider
|
|
|
|
compile libs.jerseyContainerServlet
|
|
|
|
compile libs.jerseyHk2
|
|
|
|
compile libs.jaxbApi // Jersey dependency that was available in the JDK before Java 9
|
|
|
|
compile libs.activation // Jersey dependency that was available in the JDK before Java 9
|
|
|
|
compile libs.jettyServer
|
|
|
|
compile libs.jettyServlet
|
|
|
|
compile libs.jettyServlets
|
|
|
|
|
|
|
|
testCompile project(':clients')
|
|
|
|
testCompile libs.junit
|
|
|
|
testCompile project(':clients').sourceSets.test.output
|
|
|
|
testCompile libs.easymock
|
|
|
|
testCompile libs.powermockJunit4
|
|
|
|
testCompile libs.powermockEasymock
|
|
|
|
|
|
|
|
testRuntime libs.slf4jlog4j
|
|
|
|
}
|
|
|
|
|
|
|
|
javadoc {
|
|
|
|
enabled = false
|
|
|
|
}
|
|
|
|
|
|
|
|
tasks.create(name: "copyDependantLibs", type: Copy) {
|
|
|
|
from (configurations.testRuntime) {
|
|
|
|
include('slf4j-log4j12*')
|
|
|
|
include('log4j*jar')
|
|
|
|
}
|
|
|
|
from (configurations.runtime) {
|
|
|
|
exclude('kafka-clients*')
|
|
|
|
}
|
|
|
|
into "$buildDir/dependant-libs-${versions.scala}"
|
|
|
|
duplicatesStrategy 'exclude'
|
|
|
|
}
|
|
|
|
|
|
|
|
jar {
|
|
|
|
dependsOn 'copyDependantLibs'
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
project(':streams') {
|
|
|
|
archivesBaseName = "kafka-streams"
|
|
|
|
|
|
|
|
dependencies {
|
|
|
|
compile project(':clients')
|
|
|
|
compile project(':connect:json') // this dependency should be removed after we unify data API
|
|
|
|
compile libs.slf4jApi
|
|
|
|
compile libs.rocksDBJni
|
|
|
|
|
|
|
|
// testCompileOnly prevents streams from exporting a dependency on test-utils, which would cause a dependency cycle
|
|
|
|
testCompileOnly project(':streams:test-utils')
|
|
|
|
testCompile project(':clients').sourceSets.test.output
|
|
|
|
testCompile project(':core')
|
|
|
|
testCompile project(':core').sourceSets.test.output
|
|
|
|
testCompile libs.log4j
|
|
|
|
testCompile libs.junit
|
|
|
|
testCompile libs.easymock
|
|
|
|
testCompile libs.bcpkix
|
|
|
|
testCompile libs.hamcrest
|
|
|
|
|
|
|
|
testRuntimeOnly project(':streams:test-utils')
|
|
|
|
testRuntime libs.slf4jlog4j
|
|
|
|
}
|
|
|
|
|
|
|
|
javadoc {
|
|
|
|
include "**/org/apache/kafka/streams/**"
|
|
|
|
exclude "**/internals/**"
|
|
|
|
}
|
|
|
|
|
|
|
|
tasks.create(name: "copyDependantLibs", type: Copy) {
|
|
|
|
from (configurations.testRuntime) {
|
|
|
|
include('slf4j-log4j12*')
|
|
|
|
include('log4j*jar')
|
|
|
|
}
|
|
|
|
from (configurations.runtime) {
|
|
|
|
exclude('kafka-clients*')
|
|
|
|
}
|
|
|
|
into "$buildDir/dependant-libs-${versions.scala}"
|
|
|
|
duplicatesStrategy 'exclude'
|
|
|
|
}
|
|
|
|
|
|
|
|
jar {
|
|
|
|
dependsOn 'copyDependantLibs'
|
|
|
|
}
|
|
|
|
|
|
|
|
systemTestLibs {
|
|
|
|
dependsOn testJar
|
|
|
|
}
|
|
|
|
|
|
|
|
task genStreamsConfigDocs(type: JavaExec) {
|
|
|
|
classpath = sourceSets.main.runtimeClasspath
|
|
|
|
main = 'org.apache.kafka.streams.StreamsConfig'
|
|
|
|
if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
|
|
|
|
standardOutput = new File(generatedDocsDir, "streams_config.html").newOutputStream()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
project(':streams:streams-scala') {
|
|
|
|
println "Building project 'streams-scala' with Scala version ${versions.scala}"
|
|
|
|
apply plugin: 'scala'
|
|
|
|
archivesBaseName = "kafka-streams-scala_${versions.baseScala}"
|
|
|
|
|
|
|
|
dependencies {
|
|
|
|
compile project(':streams')
|
|
|
|
|
|
|
|
compile libs.scalaLibrary
|
|
|
|
|
|
|
|
testCompile project(':core')
|
|
|
|
testCompile project(':core').sourceSets.test.output
|
|
|
|
testCompile project(':streams').sourceSets.test.output
|
|
|
|
testCompile project(':clients').sourceSets.test.output
|
|
|
|
testCompile project(':streams:test-utils')
|
|
|
|
|
|
|
|
testCompile libs.junit
|
|
|
|
testCompile libs.scalatest
|
|
|
|
testCompile libs.easymock
|
|
|
|
|
|
|
|
testRuntime libs.slf4jlog4j
|
|
|
|
}
|
|
|
|
|
|
|
|
javadoc {
|
|
|
|
include "**/org/apache/kafka/streams/scala/**"
|
|
|
|
}
|
|
|
|
|
|
|
|
tasks.create(name: "copyDependantLibs", type: Copy) {
|
|
|
|
from (configurations.runtime) {
|
|
|
|
exclude('kafka-streams*')
|
|
|
|
}
|
|
|
|
into "$buildDir/dependant-libs-${versions.scala}"
|
|
|
|
duplicatesStrategy 'exclude'
|
|
|
|
}
|
|
|
|
|
|
|
|
jar {
|
|
|
|
dependsOn 'copyDependantLibs'
|
|
|
|
}
|
|
|
|
|
|
|
|
test.dependsOn(':spotlessScalaCheck')
|
|
|
|
}
|
|
|
|
|
|
|
|
project(':streams:test-utils') {
|
|
|
|
archivesBaseName = "kafka-streams-test-utils"
|
|
|
|
|
|
|
|
dependencies {
|
|
|
|
compile project(':streams')
|
|
|
|
compile project(':clients')
|
|
|
|
|
|
|
|
testCompile project(':clients').sourceSets.test.output
|
|
|
|
testCompile libs.junit
|
|
|
|
testCompile libs.easymock
|
|
|
|
|
|
|
|
testRuntime libs.slf4jlog4j
|
|
|
|
}
|
|
|
|
|
|
|
|
javadoc {
|
|
|
|
include "**/org/apache/kafka/streams/test/**"
|
|
|
|
exclude "**/internals/**"
|
|
|
|
}
|
|
|
|
|
|
|
|
tasks.create(name: "copyDependantLibs", type: Copy) {
|
|
|
|
from (configurations.runtime) {
|
|
|
|
exclude('kafka-streams*')
|
|
|
|
}
|
|
|
|
into "$buildDir/dependant-libs-${versions.scala}"
|
|
|
|
duplicatesStrategy 'exclude'
|
|
|
|
}
|
|
|
|
|
|
|
|
jar {
|
|
|
|
dependsOn 'copyDependantLibs'
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
project(':streams:examples') {
|
|
|
|
archivesBaseName = "kafka-streams-examples"
|
|
|
|
|
|
|
|
dependencies {
|
|
|
|
compile project(':streams')
|
|
|
|
compile project(':connect:json') // this dependency should be removed after we unify data API
|
|
|
|
compile libs.slf4jlog4j
|
|
|
|
|
|
|
|
testCompile project(':streams:test-utils')
|
|
|
|
testCompile project(':clients').sourceSets.test.output // for org.apache.kafka.test.IntegrationTest
|
|
|
|
testCompile libs.junit
|
|
|
|
}
|
|
|
|
|
|
|
|
javadoc {
|
|
|
|
enabled = false
|
|
|
|
}
|
|
|
|
|
|
|
|
tasks.create(name: "copyDependantLibs", type: Copy) {
|
|
|
|
from (configurations.runtime) {
|
|
|
|
exclude('kafka-streams*')
|
|
|
|
}
|
|
|
|
into "$buildDir/dependant-libs-${versions.scala}"
|
|
|
|
duplicatesStrategy 'exclude'
|
|
|
|
}
|
|
|
|
|
|
|
|
jar {
|
|
|
|
dependsOn 'copyDependantLibs'
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
project(':streams:upgrade-system-tests-0100') {
|
|
|
|
archivesBaseName = "kafka-streams-upgrade-system-tests-0100"
|
|
|
|
|
|
|
|
dependencies {
|
|
|
|
testCompile libs.kafkaStreams_0100
|
|
|
|
}
|
|
|
|
|
|
|
|
systemTestLibs {
|
|
|
|
dependsOn testJar
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
project(':streams:upgrade-system-tests-0101') {
|
|
|
|
archivesBaseName = "kafka-streams-upgrade-system-tests-0101"
|
|
|
|
|
|
|
|
dependencies {
|
|
|
|
testCompile libs.kafkaStreams_0101
|
|
|
|
}
|
|
|
|
|
|
|
|
systemTestLibs {
|
|
|
|
dependsOn testJar
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
project(':streams:upgrade-system-tests-0102') {
|
|
|
|
archivesBaseName = "kafka-streams-upgrade-system-tests-0102"
|
|
|
|
|
|
|
|
dependencies {
|
|
|
|
testCompile libs.kafkaStreams_0102
|
|
|
|
}
|
|
|
|
|
|
|
|
systemTestLibs {
|
|
|
|
dependsOn testJar
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
project(':streams:upgrade-system-tests-0110') {
|
|
|
|
archivesBaseName = "kafka-streams-upgrade-system-tests-0110"
|
|
|
|
|
|
|
|
dependencies {
|
|
|
|
testCompile libs.kafkaStreams_0110
|
|
|
|
}
|
|
|
|
|
|
|
|
systemTestLibs {
|
|
|
|
dependsOn testJar
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
project(':streams:upgrade-system-tests-10') {
|
|
|
|
archivesBaseName = "kafka-streams-upgrade-system-tests-10"
|
|
|
|
|
|
|
|
dependencies {
|
|
|
|
testCompile libs.kafkaStreams_10
|
|
|
|
}
|
|
|
|
|
|
|
|
systemTestLibs {
|
|
|
|
dependsOn testJar
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
project(':streams:upgrade-system-tests-11') {
|
|
|
|
archivesBaseName = "kafka-streams-upgrade-system-tests-11"
|
|
|
|
|
|
|
|
dependencies {
|
|
|
|
testCompile libs.kafkaStreams_11
|
|
|
|
}
|
|
|
|
|
|
|
|
systemTestLibs {
|
|
|
|
dependsOn testJar
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
MINOR: Enable ignored upgrade system tests - trunk (#5605)
Removed ignore annotations from the upgrade tests. This PR includes the following changes for updating the upgrade tests:
* Uploaded new versions 0.10.2.2, 0.11.0.3, 1.0.2, 1.1.1, and 2.0.0 (in the associated scala versions) to kafka-packages
* Update versions in version.py, Dockerfile, base.sh
* Added new versions to StreamsUpgradeTest.test_upgrade_downgrade_brokers including version 2.0.0
* Added new versions StreamsUpgradeTest.test_simple_upgrade_downgrade test excluding version 2.0.0
* Version 2.0.0 is excluded from the streams upgrade/downgrade test as StreamsConfig needs an update for the new version, requiring a KIP. Once the community votes the KIP in, a minor follow-up PR can be pushed to add the 2.0.0 version to the upgrade test.
* Fixed minor bug in kafka-run-class.sh for classpath in upgrade/downgrade tests across versions.
* Follow on PRs for 0.10.2x, 0.11.0x, 1.0.x, 1.1.x, and 2.0.x will be pushed soon with the same updates required for the specific version.
Reviewers: Eno Thereska <eno.thereska@gmail.com>, John Roesler <vvcephei@users.noreply.github.com>, Guozhang Wang <wangguoz@gmail.com>, Matthias J. Sax <matthias@confluent.io>
6 years ago
|
|
|
project(':streams:upgrade-system-tests-20') {
|
|
|
|
archivesBaseName = "kafka-streams-upgrade-system-tests-20"
|
|
|
|
|
|
|
|
dependencies {
|
|
|
|
testCompile libs.kafkaStreams_20
|
|
|
|
}
|
|
|
|
|
|
|
|
systemTestLibs {
|
|
|
|
dependsOn testJar
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
project(':jmh-benchmarks') {
|
|
|
|
|
|
|
|
apply plugin: 'com.github.johnrengelman.shadow'
|
|
|
|
|
|
|
|
shadowJar {
|
|
|
|
baseName = 'kafka-jmh-benchmarks-all'
|
|
|
|
classifier = null
|
|
|
|
version = null
|
|
|
|
}
|
|
|
|
|
|
|
|
dependencies {
|
|
|
|
compile project(':clients')
|
|
|
|
compile project(':streams')
|
|
|
|
compile libs.jmhCore
|
|
|
|
annotationProcessor libs.jmhGeneratorAnnProcess
|
|
|
|
compile libs.jmhCoreBenchmarks
|
|
|
|
}
|
|
|
|
|
|
|
|
jar {
|
|
|
|
manifest {
|
|
|
|
attributes "Main-Class": "org.openjdk.jmh.Main"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
task jmh(type: JavaExec, dependsOn: [':jmh-benchmarks:clean', ':jmh-benchmarks:shadowJar']) {
|
|
|
|
|
|
|
|
main="-jar"
|
|
|
|
|
|
|
|
doFirst {
|
|
|
|
if (System.getProperty("jmhArgs")) {
|
|
|
|
args System.getProperty("jmhArgs").split(',')
|
|
|
|
}
|
|
|
|
args = [shadowJar.archivePath, *args]
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
javadoc {
|
|
|
|
enabled = false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
project(':log4j-appender') {
|
|
|
|
archivesBaseName = "kafka-log4j-appender"
|
|
|
|
|
|
|
|
dependencies {
|
|
|
|
compile project(':clients')
|
|
|
|
compile libs.slf4jlog4j
|
|
|
|
|
|
|
|
testCompile project(':clients').sourceSets.test.output
|
|
|
|
testCompile libs.junit
|
|
|
|
testCompile libs.easymock
|
|
|
|
}
|
|
|
|
|
|
|
|
javadoc {
|
|
|
|
enabled = false
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
KAFKA-2366; Initial patch for Copycat
This is an initial patch implementing the basics of Copycat for KIP-26.
The intent here is to start a review of the key pieces of the core API and get a reasonably functional, baseline, non-distributed implementation of Copycat in place to get things rolling. The current patch has a number of known issues that need to be addressed before a final version:
* Some build-related issues. Specifically, requires some locally-installed dependencies (see below), ignores checkstyle for the runtime data library because it's lifted from Avro currently and likely won't last in its current form, and some Gradle task dependencies aren't quite right because I haven't gotten rid of the dependency on `core` (which should now be an easy patch since new consumer groups are in a much better state).
* This patch currently depends on some Confluent trunk code because I prototyped with our Avro serializers w/ schema-registry support. We need to figure out what we want to provide as an example built-in set of serializers. Unlike core Kafka where we could ignore the issue, providing only ByteArray or String serializers, this is pretty central to how Copycat works.
* This patch uses a hacked up version of Avro as its runtime data format. Not sure if we want to go through the entire API discussion just to get some basic code committed, so I filed KAFKA-2367 to handle that separately. The core connector APIs and the runtime data APIs are entirely orthogonal.
* This patch needs some updates to get aligned with recent new consumer changes (specifically, I'm aware of the ConcurrentModificationException issue on exit). More generally, the new consumer is in flux but Copycat depends on it, so there are likely to be some negative interactions.
* The layout feels a bit awkward to me right now because I ported it from a Maven layout. We don't have nearly the same level of granularity in Kafka currently (core and clients, plus the mostly ignored examples, log4j-appender, and a couple of contribs). We might want to reorganize, although keeping data+api separate from runtime and connector plugins is useful for minimizing dependencies.
* There are a variety of other things (e.g., I'm not happy with the exception hierarchy/how they are currently handled, TopicPartition doesn't really need to be duplicated unless we want Copycat entirely isolated from the Kafka APIs, etc), but I expect those we'll cover in the review.
Before commenting on the patch, it's probably worth reviewing https://issues.apache.org/jira/browse/KAFKA-2365 and https://issues.apache.org/jira/browse/KAFKA-2366 to get an idea of what I had in mind for a) what we ultimately want with all the Copycat patches and b) what we aim to cover in this initial patch. My hope is that we can use a WIP patch (after the current obvious deficiencies are addressed) while recognizing that we want to make iterative progress with a bunch of subsequent PRs.
Author: Ewen Cheslack-Postava <me@ewencp.org>
Reviewers: Ismael Juma, Gwen Shapira
Closes #99 from ewencp/copycat and squashes the following commits:
a3a47a6 [Ewen Cheslack-Postava] Simplify Copycat exceptions, make them a subclass of KafkaException.
8c108b0 [Ewen Cheslack-Postava] Rename Coordinator to Herder to avoid confusion with the consumer coordinator.
7bf8075 [Ewen Cheslack-Postava] Make Copycat CLI speific to standalone mode, clean up some config and get rid of config storage in standalone mode.
656a003 [Ewen Cheslack-Postava] Clarify and expand the explanation of the Copycat Coordinator interface.
c0e5fdc [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
0fa7a36 [Ewen Cheslack-Postava] Mark Copycat classes as unstable and reduce visibility of some classes where possible.
d55d31e [Ewen Cheslack-Postava] Reorganize Copycat code to put it all under one top-level directory.
b29cb2c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
d713a21 [Ewen Cheslack-Postava] Address Gwen's review comments.
6787a85 [Ewen Cheslack-Postava] Make Converter generic to match serializers since some serialization formats do not require a base class of Object; update many other classes to have generic key and value class type parameters to match this change.
b194c73 [Ewen Cheslack-Postava] Split Copycat converter option into two options for key and value.
0b5a1a0 [Ewen Cheslack-Postava] Normalize naming to use partition for both source and Kafka, adjusting naming in CopycatRecord classes to clearly differentiate.
e345142 [Ewen Cheslack-Postava] Remove Copycat reflection utils, use existing Utils and ConfigDef functionality from clients package.
be5c387 [Ewen Cheslack-Postava] Minor cleanup
122423e [Ewen Cheslack-Postava] Style cleanup
6ba87de [Ewen Cheslack-Postava] Remove most of the Avro-based mock runtime data API, only preserving enough schema functionality to support basic primitive types for an initial patch.
4674d13 [Ewen Cheslack-Postava] Address review comments, clean up some code styling.
25b5739 [Ewen Cheslack-Postava] Fix sink task offset commit concurrency issue by moving it to the worker thread and waking up the consumer to ensure it exits promptly.
0aefe21 [Ewen Cheslack-Postava] Add log4j settings for Copycat.
220e42d [Ewen Cheslack-Postava] Replace Avro serializer with JSON serializer.
1243a7c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
5a618c6 [Ewen Cheslack-Postava] Remove offset serializers, instead reusing the existing serializers and removing schema projection support.
e849e10 [Ewen Cheslack-Postava] Remove duplicated TopicPartition implementation.
dec1379 [Ewen Cheslack-Postava] Switch to using new consumer coordinator instead of manually assigning partitions. Remove dependency of copycat-runtime on core.
4a9b4f3 [Ewen Cheslack-Postava] Add some helpful Copycat-specific build and test targets that cover all Copycat packages.
31cd1ca [Ewen Cheslack-Postava] Add CLI tools for Copycat.
e14942c [Ewen Cheslack-Postava] Add Copycat file connector.
0233456 [Ewen Cheslack-Postava] Add copycat-avro and copycat-runtime
11981d2 [Ewen Cheslack-Postava] Add copycat-data and copycat-api
9 years ago
|
|
|
|
|
|
|
project(':connect:api') {
|
|
|
|
archivesBaseName = "connect-api"
|
KAFKA-2366; Initial patch for Copycat
This is an initial patch implementing the basics of Copycat for KIP-26.
The intent here is to start a review of the key pieces of the core API and get a reasonably functional, baseline, non-distributed implementation of Copycat in place to get things rolling. The current patch has a number of known issues that need to be addressed before a final version:
* Some build-related issues. Specifically, requires some locally-installed dependencies (see below), ignores checkstyle for the runtime data library because it's lifted from Avro currently and likely won't last in its current form, and some Gradle task dependencies aren't quite right because I haven't gotten rid of the dependency on `core` (which should now be an easy patch since new consumer groups are in a much better state).
* This patch currently depends on some Confluent trunk code because I prototyped with our Avro serializers w/ schema-registry support. We need to figure out what we want to provide as an example built-in set of serializers. Unlike core Kafka where we could ignore the issue, providing only ByteArray or String serializers, this is pretty central to how Copycat works.
* This patch uses a hacked up version of Avro as its runtime data format. Not sure if we want to go through the entire API discussion just to get some basic code committed, so I filed KAFKA-2367 to handle that separately. The core connector APIs and the runtime data APIs are entirely orthogonal.
* This patch needs some updates to get aligned with recent new consumer changes (specifically, I'm aware of the ConcurrentModificationException issue on exit). More generally, the new consumer is in flux but Copycat depends on it, so there are likely to be some negative interactions.
* The layout feels a bit awkward to me right now because I ported it from a Maven layout. We don't have nearly the same level of granularity in Kafka currently (core and clients, plus the mostly ignored examples, log4j-appender, and a couple of contribs). We might want to reorganize, although keeping data+api separate from runtime and connector plugins is useful for minimizing dependencies.
* There are a variety of other things (e.g., I'm not happy with the exception hierarchy/how they are currently handled, TopicPartition doesn't really need to be duplicated unless we want Copycat entirely isolated from the Kafka APIs, etc), but I expect those we'll cover in the review.
Before commenting on the patch, it's probably worth reviewing https://issues.apache.org/jira/browse/KAFKA-2365 and https://issues.apache.org/jira/browse/KAFKA-2366 to get an idea of what I had in mind for a) what we ultimately want with all the Copycat patches and b) what we aim to cover in this initial patch. My hope is that we can use a WIP patch (after the current obvious deficiencies are addressed) while recognizing that we want to make iterative progress with a bunch of subsequent PRs.
Author: Ewen Cheslack-Postava <me@ewencp.org>
Reviewers: Ismael Juma, Gwen Shapira
Closes #99 from ewencp/copycat and squashes the following commits:
a3a47a6 [Ewen Cheslack-Postava] Simplify Copycat exceptions, make them a subclass of KafkaException.
8c108b0 [Ewen Cheslack-Postava] Rename Coordinator to Herder to avoid confusion with the consumer coordinator.
7bf8075 [Ewen Cheslack-Postava] Make Copycat CLI speific to standalone mode, clean up some config and get rid of config storage in standalone mode.
656a003 [Ewen Cheslack-Postava] Clarify and expand the explanation of the Copycat Coordinator interface.
c0e5fdc [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
0fa7a36 [Ewen Cheslack-Postava] Mark Copycat classes as unstable and reduce visibility of some classes where possible.
d55d31e [Ewen Cheslack-Postava] Reorganize Copycat code to put it all under one top-level directory.
b29cb2c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
d713a21 [Ewen Cheslack-Postava] Address Gwen's review comments.
6787a85 [Ewen Cheslack-Postava] Make Converter generic to match serializers since some serialization formats do not require a base class of Object; update many other classes to have generic key and value class type parameters to match this change.
b194c73 [Ewen Cheslack-Postava] Split Copycat converter option into two options for key and value.
0b5a1a0 [Ewen Cheslack-Postava] Normalize naming to use partition for both source and Kafka, adjusting naming in CopycatRecord classes to clearly differentiate.
e345142 [Ewen Cheslack-Postava] Remove Copycat reflection utils, use existing Utils and ConfigDef functionality from clients package.
be5c387 [Ewen Cheslack-Postava] Minor cleanup
122423e [Ewen Cheslack-Postava] Style cleanup
6ba87de [Ewen Cheslack-Postava] Remove most of the Avro-based mock runtime data API, only preserving enough schema functionality to support basic primitive types for an initial patch.
4674d13 [Ewen Cheslack-Postava] Address review comments, clean up some code styling.
25b5739 [Ewen Cheslack-Postava] Fix sink task offset commit concurrency issue by moving it to the worker thread and waking up the consumer to ensure it exits promptly.
0aefe21 [Ewen Cheslack-Postava] Add log4j settings for Copycat.
220e42d [Ewen Cheslack-Postava] Replace Avro serializer with JSON serializer.
1243a7c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
5a618c6 [Ewen Cheslack-Postava] Remove offset serializers, instead reusing the existing serializers and removing schema projection support.
e849e10 [Ewen Cheslack-Postava] Remove duplicated TopicPartition implementation.
dec1379 [Ewen Cheslack-Postava] Switch to using new consumer coordinator instead of manually assigning partitions. Remove dependency of copycat-runtime on core.
4a9b4f3 [Ewen Cheslack-Postava] Add some helpful Copycat-specific build and test targets that cover all Copycat packages.
31cd1ca [Ewen Cheslack-Postava] Add CLI tools for Copycat.
e14942c [Ewen Cheslack-Postava] Add Copycat file connector.
0233456 [Ewen Cheslack-Postava] Add copycat-avro and copycat-runtime
11981d2 [Ewen Cheslack-Postava] Add copycat-data and copycat-api
9 years ago
|
|
|
|
|
|
|
dependencies {
|
|
|
|
compile project(':clients')
|
|
|
|
compile libs.slf4jApi
|
|
|
|
compile libs.jaxrsApi
|
|
|
|
|
|
|
|
testCompile libs.junit
|
KAFKA-2366; Initial patch for Copycat
This is an initial patch implementing the basics of Copycat for KIP-26.
The intent here is to start a review of the key pieces of the core API and get a reasonably functional, baseline, non-distributed implementation of Copycat in place to get things rolling. The current patch has a number of known issues that need to be addressed before a final version:
* Some build-related issues. Specifically, requires some locally-installed dependencies (see below), ignores checkstyle for the runtime data library because it's lifted from Avro currently and likely won't last in its current form, and some Gradle task dependencies aren't quite right because I haven't gotten rid of the dependency on `core` (which should now be an easy patch since new consumer groups are in a much better state).
* This patch currently depends on some Confluent trunk code because I prototyped with our Avro serializers w/ schema-registry support. We need to figure out what we want to provide as an example built-in set of serializers. Unlike core Kafka where we could ignore the issue, providing only ByteArray or String serializers, this is pretty central to how Copycat works.
* This patch uses a hacked up version of Avro as its runtime data format. Not sure if we want to go through the entire API discussion just to get some basic code committed, so I filed KAFKA-2367 to handle that separately. The core connector APIs and the runtime data APIs are entirely orthogonal.
* This patch needs some updates to get aligned with recent new consumer changes (specifically, I'm aware of the ConcurrentModificationException issue on exit). More generally, the new consumer is in flux but Copycat depends on it, so there are likely to be some negative interactions.
* The layout feels a bit awkward to me right now because I ported it from a Maven layout. We don't have nearly the same level of granularity in Kafka currently (core and clients, plus the mostly ignored examples, log4j-appender, and a couple of contribs). We might want to reorganize, although keeping data+api separate from runtime and connector plugins is useful for minimizing dependencies.
* There are a variety of other things (e.g., I'm not happy with the exception hierarchy/how they are currently handled, TopicPartition doesn't really need to be duplicated unless we want Copycat entirely isolated from the Kafka APIs, etc), but I expect those we'll cover in the review.
Before commenting on the patch, it's probably worth reviewing https://issues.apache.org/jira/browse/KAFKA-2365 and https://issues.apache.org/jira/browse/KAFKA-2366 to get an idea of what I had in mind for a) what we ultimately want with all the Copycat patches and b) what we aim to cover in this initial patch. My hope is that we can use a WIP patch (after the current obvious deficiencies are addressed) while recognizing that we want to make iterative progress with a bunch of subsequent PRs.
Author: Ewen Cheslack-Postava <me@ewencp.org>
Reviewers: Ismael Juma, Gwen Shapira
Closes #99 from ewencp/copycat and squashes the following commits:
a3a47a6 [Ewen Cheslack-Postava] Simplify Copycat exceptions, make them a subclass of KafkaException.
8c108b0 [Ewen Cheslack-Postava] Rename Coordinator to Herder to avoid confusion with the consumer coordinator.
7bf8075 [Ewen Cheslack-Postava] Make Copycat CLI speific to standalone mode, clean up some config and get rid of config storage in standalone mode.
656a003 [Ewen Cheslack-Postava] Clarify and expand the explanation of the Copycat Coordinator interface.
c0e5fdc [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
0fa7a36 [Ewen Cheslack-Postava] Mark Copycat classes as unstable and reduce visibility of some classes where possible.
d55d31e [Ewen Cheslack-Postava] Reorganize Copycat code to put it all under one top-level directory.
b29cb2c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
d713a21 [Ewen Cheslack-Postava] Address Gwen's review comments.
6787a85 [Ewen Cheslack-Postava] Make Converter generic to match serializers since some serialization formats do not require a base class of Object; update many other classes to have generic key and value class type parameters to match this change.
b194c73 [Ewen Cheslack-Postava] Split Copycat converter option into two options for key and value.
0b5a1a0 [Ewen Cheslack-Postava] Normalize naming to use partition for both source and Kafka, adjusting naming in CopycatRecord classes to clearly differentiate.
e345142 [Ewen Cheslack-Postava] Remove Copycat reflection utils, use existing Utils and ConfigDef functionality from clients package.
be5c387 [Ewen Cheslack-Postava] Minor cleanup
122423e [Ewen Cheslack-Postava] Style cleanup
6ba87de [Ewen Cheslack-Postava] Remove most of the Avro-based mock runtime data API, only preserving enough schema functionality to support basic primitive types for an initial patch.
4674d13 [Ewen Cheslack-Postava] Address review comments, clean up some code styling.
25b5739 [Ewen Cheslack-Postava] Fix sink task offset commit concurrency issue by moving it to the worker thread and waking up the consumer to ensure it exits promptly.
0aefe21 [Ewen Cheslack-Postava] Add log4j settings for Copycat.
220e42d [Ewen Cheslack-Postava] Replace Avro serializer with JSON serializer.
1243a7c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
5a618c6 [Ewen Cheslack-Postava] Remove offset serializers, instead reusing the existing serializers and removing schema projection support.
e849e10 [Ewen Cheslack-Postava] Remove duplicated TopicPartition implementation.
dec1379 [Ewen Cheslack-Postava] Switch to using new consumer coordinator instead of manually assigning partitions. Remove dependency of copycat-runtime on core.
4a9b4f3 [Ewen Cheslack-Postava] Add some helpful Copycat-specific build and test targets that cover all Copycat packages.
31cd1ca [Ewen Cheslack-Postava] Add CLI tools for Copycat.
e14942c [Ewen Cheslack-Postava] Add Copycat file connector.
0233456 [Ewen Cheslack-Postava] Add copycat-avro and copycat-runtime
11981d2 [Ewen Cheslack-Postava] Add copycat-data and copycat-api
9 years ago
|
|
|
|
|
|
|
testRuntime libs.slf4jlog4j
|
|
|
|
testCompile project(':clients').sourceSets.test.output
|
KAFKA-2366; Initial patch for Copycat
This is an initial patch implementing the basics of Copycat for KIP-26.
The intent here is to start a review of the key pieces of the core API and get a reasonably functional, baseline, non-distributed implementation of Copycat in place to get things rolling. The current patch has a number of known issues that need to be addressed before a final version:
* Some build-related issues. Specifically, requires some locally-installed dependencies (see below), ignores checkstyle for the runtime data library because it's lifted from Avro currently and likely won't last in its current form, and some Gradle task dependencies aren't quite right because I haven't gotten rid of the dependency on `core` (which should now be an easy patch since new consumer groups are in a much better state).
* This patch currently depends on some Confluent trunk code because I prototyped with our Avro serializers w/ schema-registry support. We need to figure out what we want to provide as an example built-in set of serializers. Unlike core Kafka where we could ignore the issue, providing only ByteArray or String serializers, this is pretty central to how Copycat works.
* This patch uses a hacked up version of Avro as its runtime data format. Not sure if we want to go through the entire API discussion just to get some basic code committed, so I filed KAFKA-2367 to handle that separately. The core connector APIs and the runtime data APIs are entirely orthogonal.
* This patch needs some updates to get aligned with recent new consumer changes (specifically, I'm aware of the ConcurrentModificationException issue on exit). More generally, the new consumer is in flux but Copycat depends on it, so there are likely to be some negative interactions.
* The layout feels a bit awkward to me right now because I ported it from a Maven layout. We don't have nearly the same level of granularity in Kafka currently (core and clients, plus the mostly ignored examples, log4j-appender, and a couple of contribs). We might want to reorganize, although keeping data+api separate from runtime and connector plugins is useful for minimizing dependencies.
* There are a variety of other things (e.g., I'm not happy with the exception hierarchy/how they are currently handled, TopicPartition doesn't really need to be duplicated unless we want Copycat entirely isolated from the Kafka APIs, etc), but I expect those we'll cover in the review.
Before commenting on the patch, it's probably worth reviewing https://issues.apache.org/jira/browse/KAFKA-2365 and https://issues.apache.org/jira/browse/KAFKA-2366 to get an idea of what I had in mind for a) what we ultimately want with all the Copycat patches and b) what we aim to cover in this initial patch. My hope is that we can use a WIP patch (after the current obvious deficiencies are addressed) while recognizing that we want to make iterative progress with a bunch of subsequent PRs.
Author: Ewen Cheslack-Postava <me@ewencp.org>
Reviewers: Ismael Juma, Gwen Shapira
Closes #99 from ewencp/copycat and squashes the following commits:
a3a47a6 [Ewen Cheslack-Postava] Simplify Copycat exceptions, make them a subclass of KafkaException.
8c108b0 [Ewen Cheslack-Postava] Rename Coordinator to Herder to avoid confusion with the consumer coordinator.
7bf8075 [Ewen Cheslack-Postava] Make Copycat CLI speific to standalone mode, clean up some config and get rid of config storage in standalone mode.
656a003 [Ewen Cheslack-Postava] Clarify and expand the explanation of the Copycat Coordinator interface.
c0e5fdc [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
0fa7a36 [Ewen Cheslack-Postava] Mark Copycat classes as unstable and reduce visibility of some classes where possible.
d55d31e [Ewen Cheslack-Postava] Reorganize Copycat code to put it all under one top-level directory.
b29cb2c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
d713a21 [Ewen Cheslack-Postava] Address Gwen's review comments.
6787a85 [Ewen Cheslack-Postava] Make Converter generic to match serializers since some serialization formats do not require a base class of Object; update many other classes to have generic key and value class type parameters to match this change.
b194c73 [Ewen Cheslack-Postava] Split Copycat converter option into two options for key and value.
0b5a1a0 [Ewen Cheslack-Postava] Normalize naming to use partition for both source and Kafka, adjusting naming in CopycatRecord classes to clearly differentiate.
e345142 [Ewen Cheslack-Postava] Remove Copycat reflection utils, use existing Utils and ConfigDef functionality from clients package.
be5c387 [Ewen Cheslack-Postava] Minor cleanup
122423e [Ewen Cheslack-Postava] Style cleanup
6ba87de [Ewen Cheslack-Postava] Remove most of the Avro-based mock runtime data API, only preserving enough schema functionality to support basic primitive types for an initial patch.
4674d13 [Ewen Cheslack-Postava] Address review comments, clean up some code styling.
25b5739 [Ewen Cheslack-Postava] Fix sink task offset commit concurrency issue by moving it to the worker thread and waking up the consumer to ensure it exits promptly.
0aefe21 [Ewen Cheslack-Postava] Add log4j settings for Copycat.
220e42d [Ewen Cheslack-Postava] Replace Avro serializer with JSON serializer.
1243a7c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
5a618c6 [Ewen Cheslack-Postava] Remove offset serializers, instead reusing the existing serializers and removing schema projection support.
e849e10 [Ewen Cheslack-Postava] Remove duplicated TopicPartition implementation.
dec1379 [Ewen Cheslack-Postava] Switch to using new consumer coordinator instead of manually assigning partitions. Remove dependency of copycat-runtime on core.
4a9b4f3 [Ewen Cheslack-Postava] Add some helpful Copycat-specific build and test targets that cover all Copycat packages.
31cd1ca [Ewen Cheslack-Postava] Add CLI tools for Copycat.
e14942c [Ewen Cheslack-Postava] Add Copycat file connector.
0233456 [Ewen Cheslack-Postava] Add copycat-avro and copycat-runtime
11981d2 [Ewen Cheslack-Postava] Add copycat-data and copycat-api
9 years ago
|
|
|
}
|
|
|
|
|
|
|
|
javadoc {
|
|
|
|
include "**/org/apache/kafka/connect/**" // needed for the `javadocAll` task
|
|
|
|
options.links "http://docs.oracle.com/javase/7/docs/api/"
|
KAFKA-2366; Initial patch for Copycat
This is an initial patch implementing the basics of Copycat for KIP-26.
The intent here is to start a review of the key pieces of the core API and get a reasonably functional, baseline, non-distributed implementation of Copycat in place to get things rolling. The current patch has a number of known issues that need to be addressed before a final version:
* Some build-related issues. Specifically, requires some locally-installed dependencies (see below), ignores checkstyle for the runtime data library because it's lifted from Avro currently and likely won't last in its current form, and some Gradle task dependencies aren't quite right because I haven't gotten rid of the dependency on `core` (which should now be an easy patch since new consumer groups are in a much better state).
* This patch currently depends on some Confluent trunk code because I prototyped with our Avro serializers w/ schema-registry support. We need to figure out what we want to provide as an example built-in set of serializers. Unlike core Kafka where we could ignore the issue, providing only ByteArray or String serializers, this is pretty central to how Copycat works.
* This patch uses a hacked up version of Avro as its runtime data format. Not sure if we want to go through the entire API discussion just to get some basic code committed, so I filed KAFKA-2367 to handle that separately. The core connector APIs and the runtime data APIs are entirely orthogonal.
* This patch needs some updates to get aligned with recent new consumer changes (specifically, I'm aware of the ConcurrentModificationException issue on exit). More generally, the new consumer is in flux but Copycat depends on it, so there are likely to be some negative interactions.
* The layout feels a bit awkward to me right now because I ported it from a Maven layout. We don't have nearly the same level of granularity in Kafka currently (core and clients, plus the mostly ignored examples, log4j-appender, and a couple of contribs). We might want to reorganize, although keeping data+api separate from runtime and connector plugins is useful for minimizing dependencies.
* There are a variety of other things (e.g., I'm not happy with the exception hierarchy/how they are currently handled, TopicPartition doesn't really need to be duplicated unless we want Copycat entirely isolated from the Kafka APIs, etc), but I expect those we'll cover in the review.
Before commenting on the patch, it's probably worth reviewing https://issues.apache.org/jira/browse/KAFKA-2365 and https://issues.apache.org/jira/browse/KAFKA-2366 to get an idea of what I had in mind for a) what we ultimately want with all the Copycat patches and b) what we aim to cover in this initial patch. My hope is that we can use a WIP patch (after the current obvious deficiencies are addressed) while recognizing that we want to make iterative progress with a bunch of subsequent PRs.
Author: Ewen Cheslack-Postava <me@ewencp.org>
Reviewers: Ismael Juma, Gwen Shapira
Closes #99 from ewencp/copycat and squashes the following commits:
a3a47a6 [Ewen Cheslack-Postava] Simplify Copycat exceptions, make them a subclass of KafkaException.
8c108b0 [Ewen Cheslack-Postava] Rename Coordinator to Herder to avoid confusion with the consumer coordinator.
7bf8075 [Ewen Cheslack-Postava] Make Copycat CLI speific to standalone mode, clean up some config and get rid of config storage in standalone mode.
656a003 [Ewen Cheslack-Postava] Clarify and expand the explanation of the Copycat Coordinator interface.
c0e5fdc [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
0fa7a36 [Ewen Cheslack-Postava] Mark Copycat classes as unstable and reduce visibility of some classes where possible.
d55d31e [Ewen Cheslack-Postava] Reorganize Copycat code to put it all under one top-level directory.
b29cb2c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
d713a21 [Ewen Cheslack-Postava] Address Gwen's review comments.
6787a85 [Ewen Cheslack-Postava] Make Converter generic to match serializers since some serialization formats do not require a base class of Object; update many other classes to have generic key and value class type parameters to match this change.
b194c73 [Ewen Cheslack-Postava] Split Copycat converter option into two options for key and value.
0b5a1a0 [Ewen Cheslack-Postava] Normalize naming to use partition for both source and Kafka, adjusting naming in CopycatRecord classes to clearly differentiate.
e345142 [Ewen Cheslack-Postava] Remove Copycat reflection utils, use existing Utils and ConfigDef functionality from clients package.
be5c387 [Ewen Cheslack-Postava] Minor cleanup
122423e [Ewen Cheslack-Postava] Style cleanup
6ba87de [Ewen Cheslack-Postava] Remove most of the Avro-based mock runtime data API, only preserving enough schema functionality to support basic primitive types for an initial patch.
4674d13 [Ewen Cheslack-Postava] Address review comments, clean up some code styling.
25b5739 [Ewen Cheslack-Postava] Fix sink task offset commit concurrency issue by moving it to the worker thread and waking up the consumer to ensure it exits promptly.
0aefe21 [Ewen Cheslack-Postava] Add log4j settings for Copycat.
220e42d [Ewen Cheslack-Postava] Replace Avro serializer with JSON serializer.
1243a7c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
5a618c6 [Ewen Cheslack-Postava] Remove offset serializers, instead reusing the existing serializers and removing schema projection support.
e849e10 [Ewen Cheslack-Postava] Remove duplicated TopicPartition implementation.
dec1379 [Ewen Cheslack-Postava] Switch to using new consumer coordinator instead of manually assigning partitions. Remove dependency of copycat-runtime on core.
4a9b4f3 [Ewen Cheslack-Postava] Add some helpful Copycat-specific build and test targets that cover all Copycat packages.
31cd1ca [Ewen Cheslack-Postava] Add CLI tools for Copycat.
e14942c [Ewen Cheslack-Postava] Add Copycat file connector.
0233456 [Ewen Cheslack-Postava] Add copycat-avro and copycat-runtime
11981d2 [Ewen Cheslack-Postava] Add copycat-data and copycat-api
9 years ago
|
|
|
}
|
|
|
|
|
|
|
|
tasks.create(name: "copyDependantLibs", type: Copy) {
|
|
|
|
from (configurations.testRuntime) {
|
|
|
|
include('slf4j-log4j12*')
|
|
|
|
include('log4j*jar')
|
|
|
|
}
|
|
|
|
from (configurations.runtime) {
|
|
|
|
exclude('kafka-clients*')
|
|
|
|
exclude('connect-*')
|
|
|
|
}
|
|
|
|
into "$buildDir/dependant-libs"
|
|
|
|
duplicatesStrategy 'exclude'
|
|
|
|
}
|
|
|
|
|
|
|
|
jar {
|
|
|
|
dependsOn copyDependantLibs
|
|
|
|
}
|
KAFKA-2366; Initial patch for Copycat
This is an initial patch implementing the basics of Copycat for KIP-26.
The intent here is to start a review of the key pieces of the core API and get a reasonably functional, baseline, non-distributed implementation of Copycat in place to get things rolling. The current patch has a number of known issues that need to be addressed before a final version:
* Some build-related issues. Specifically, requires some locally-installed dependencies (see below), ignores checkstyle for the runtime data library because it's lifted from Avro currently and likely won't last in its current form, and some Gradle task dependencies aren't quite right because I haven't gotten rid of the dependency on `core` (which should now be an easy patch since new consumer groups are in a much better state).
* This patch currently depends on some Confluent trunk code because I prototyped with our Avro serializers w/ schema-registry support. We need to figure out what we want to provide as an example built-in set of serializers. Unlike core Kafka where we could ignore the issue, providing only ByteArray or String serializers, this is pretty central to how Copycat works.
* This patch uses a hacked up version of Avro as its runtime data format. Not sure if we want to go through the entire API discussion just to get some basic code committed, so I filed KAFKA-2367 to handle that separately. The core connector APIs and the runtime data APIs are entirely orthogonal.
* This patch needs some updates to get aligned with recent new consumer changes (specifically, I'm aware of the ConcurrentModificationException issue on exit). More generally, the new consumer is in flux but Copycat depends on it, so there are likely to be some negative interactions.
* The layout feels a bit awkward to me right now because I ported it from a Maven layout. We don't have nearly the same level of granularity in Kafka currently (core and clients, plus the mostly ignored examples, log4j-appender, and a couple of contribs). We might want to reorganize, although keeping data+api separate from runtime and connector plugins is useful for minimizing dependencies.
* There are a variety of other things (e.g., I'm not happy with the exception hierarchy/how they are currently handled, TopicPartition doesn't really need to be duplicated unless we want Copycat entirely isolated from the Kafka APIs, etc), but I expect those we'll cover in the review.
Before commenting on the patch, it's probably worth reviewing https://issues.apache.org/jira/browse/KAFKA-2365 and https://issues.apache.org/jira/browse/KAFKA-2366 to get an idea of what I had in mind for a) what we ultimately want with all the Copycat patches and b) what we aim to cover in this initial patch. My hope is that we can use a WIP patch (after the current obvious deficiencies are addressed) while recognizing that we want to make iterative progress with a bunch of subsequent PRs.
Author: Ewen Cheslack-Postava <me@ewencp.org>
Reviewers: Ismael Juma, Gwen Shapira
Closes #99 from ewencp/copycat and squashes the following commits:
a3a47a6 [Ewen Cheslack-Postava] Simplify Copycat exceptions, make them a subclass of KafkaException.
8c108b0 [Ewen Cheslack-Postava] Rename Coordinator to Herder to avoid confusion with the consumer coordinator.
7bf8075 [Ewen Cheslack-Postava] Make Copycat CLI speific to standalone mode, clean up some config and get rid of config storage in standalone mode.
656a003 [Ewen Cheslack-Postava] Clarify and expand the explanation of the Copycat Coordinator interface.
c0e5fdc [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
0fa7a36 [Ewen Cheslack-Postava] Mark Copycat classes as unstable and reduce visibility of some classes where possible.
d55d31e [Ewen Cheslack-Postava] Reorganize Copycat code to put it all under one top-level directory.
b29cb2c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
d713a21 [Ewen Cheslack-Postava] Address Gwen's review comments.
6787a85 [Ewen Cheslack-Postava] Make Converter generic to match serializers since some serialization formats do not require a base class of Object; update many other classes to have generic key and value class type parameters to match this change.
b194c73 [Ewen Cheslack-Postava] Split Copycat converter option into two options for key and value.
0b5a1a0 [Ewen Cheslack-Postava] Normalize naming to use partition for both source and Kafka, adjusting naming in CopycatRecord classes to clearly differentiate.
e345142 [Ewen Cheslack-Postava] Remove Copycat reflection utils, use existing Utils and ConfigDef functionality from clients package.
be5c387 [Ewen Cheslack-Postava] Minor cleanup
122423e [Ewen Cheslack-Postava] Style cleanup
6ba87de [Ewen Cheslack-Postava] Remove most of the Avro-based mock runtime data API, only preserving enough schema functionality to support basic primitive types for an initial patch.
4674d13 [Ewen Cheslack-Postava] Address review comments, clean up some code styling.
25b5739 [Ewen Cheslack-Postava] Fix sink task offset commit concurrency issue by moving it to the worker thread and waking up the consumer to ensure it exits promptly.
0aefe21 [Ewen Cheslack-Postava] Add log4j settings for Copycat.
220e42d [Ewen Cheslack-Postava] Replace Avro serializer with JSON serializer.
1243a7c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
5a618c6 [Ewen Cheslack-Postava] Remove offset serializers, instead reusing the existing serializers and removing schema projection support.
e849e10 [Ewen Cheslack-Postava] Remove duplicated TopicPartition implementation.
dec1379 [Ewen Cheslack-Postava] Switch to using new consumer coordinator instead of manually assigning partitions. Remove dependency of copycat-runtime on core.
4a9b4f3 [Ewen Cheslack-Postava] Add some helpful Copycat-specific build and test targets that cover all Copycat packages.
31cd1ca [Ewen Cheslack-Postava] Add CLI tools for Copycat.
e14942c [Ewen Cheslack-Postava] Add Copycat file connector.
0233456 [Ewen Cheslack-Postava] Add copycat-avro and copycat-runtime
11981d2 [Ewen Cheslack-Postava] Add copycat-data and copycat-api
9 years ago
|
|
|
}
|
|
|
|
|
|
|
|
project(':connect:transforms') {
|
|
|
|
archivesBaseName = "connect-transforms"
|
|
|
|
|
|
|
|
dependencies {
|
|
|
|
compile project(':connect:api')
|
|
|
|
compile libs.slf4jApi
|
|
|
|
|
|
|
|
testCompile libs.easymock
|
|
|
|
testCompile libs.junit
|
|
|
|
testCompile libs.powermockJunit4
|
|
|
|
testCompile libs.powermockEasymock
|
|
|
|
|
|
|
|
testRuntime libs.slf4jlog4j
|
|
|
|
testCompile project(':clients').sourceSets.test.output
|
|
|
|
}
|
|
|
|
|
|
|
|
javadoc {
|
|
|
|
enabled = false
|
|
|
|
}
|
|
|
|
|
|
|
|
tasks.create(name: "copyDependantLibs", type: Copy) {
|
|
|
|
from (configurations.testRuntime) {
|
|
|
|
include('slf4j-log4j12*')
|
|
|
|
include('log4j*jar')
|
|
|
|
}
|
|
|
|
from (configurations.runtime) {
|
|
|
|
exclude('kafka-clients*')
|
|
|
|
exclude('connect-*')
|
|
|
|
}
|
|
|
|
into "$buildDir/dependant-libs"
|
|
|
|
duplicatesStrategy 'exclude'
|
|
|
|
}
|
|
|
|
|
|
|
|
jar {
|
|
|
|
dependsOn copyDependantLibs
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
project(':connect:json') {
|
|
|
|
archivesBaseName = "connect-json"
|
KAFKA-2366; Initial patch for Copycat
This is an initial patch implementing the basics of Copycat for KIP-26.
The intent here is to start a review of the key pieces of the core API and get a reasonably functional, baseline, non-distributed implementation of Copycat in place to get things rolling. The current patch has a number of known issues that need to be addressed before a final version:
* Some build-related issues. Specifically, requires some locally-installed dependencies (see below), ignores checkstyle for the runtime data library because it's lifted from Avro currently and likely won't last in its current form, and some Gradle task dependencies aren't quite right because I haven't gotten rid of the dependency on `core` (which should now be an easy patch since new consumer groups are in a much better state).
* This patch currently depends on some Confluent trunk code because I prototyped with our Avro serializers w/ schema-registry support. We need to figure out what we want to provide as an example built-in set of serializers. Unlike core Kafka where we could ignore the issue, providing only ByteArray or String serializers, this is pretty central to how Copycat works.
* This patch uses a hacked up version of Avro as its runtime data format. Not sure if we want to go through the entire API discussion just to get some basic code committed, so I filed KAFKA-2367 to handle that separately. The core connector APIs and the runtime data APIs are entirely orthogonal.
* This patch needs some updates to get aligned with recent new consumer changes (specifically, I'm aware of the ConcurrentModificationException issue on exit). More generally, the new consumer is in flux but Copycat depends on it, so there are likely to be some negative interactions.
* The layout feels a bit awkward to me right now because I ported it from a Maven layout. We don't have nearly the same level of granularity in Kafka currently (core and clients, plus the mostly ignored examples, log4j-appender, and a couple of contribs). We might want to reorganize, although keeping data+api separate from runtime and connector plugins is useful for minimizing dependencies.
* There are a variety of other things (e.g., I'm not happy with the exception hierarchy/how they are currently handled, TopicPartition doesn't really need to be duplicated unless we want Copycat entirely isolated from the Kafka APIs, etc), but I expect those we'll cover in the review.
Before commenting on the patch, it's probably worth reviewing https://issues.apache.org/jira/browse/KAFKA-2365 and https://issues.apache.org/jira/browse/KAFKA-2366 to get an idea of what I had in mind for a) what we ultimately want with all the Copycat patches and b) what we aim to cover in this initial patch. My hope is that we can use a WIP patch (after the current obvious deficiencies are addressed) while recognizing that we want to make iterative progress with a bunch of subsequent PRs.
Author: Ewen Cheslack-Postava <me@ewencp.org>
Reviewers: Ismael Juma, Gwen Shapira
Closes #99 from ewencp/copycat and squashes the following commits:
a3a47a6 [Ewen Cheslack-Postava] Simplify Copycat exceptions, make them a subclass of KafkaException.
8c108b0 [Ewen Cheslack-Postava] Rename Coordinator to Herder to avoid confusion with the consumer coordinator.
7bf8075 [Ewen Cheslack-Postava] Make Copycat CLI speific to standalone mode, clean up some config and get rid of config storage in standalone mode.
656a003 [Ewen Cheslack-Postava] Clarify and expand the explanation of the Copycat Coordinator interface.
c0e5fdc [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
0fa7a36 [Ewen Cheslack-Postava] Mark Copycat classes as unstable and reduce visibility of some classes where possible.
d55d31e [Ewen Cheslack-Postava] Reorganize Copycat code to put it all under one top-level directory.
b29cb2c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
d713a21 [Ewen Cheslack-Postava] Address Gwen's review comments.
6787a85 [Ewen Cheslack-Postava] Make Converter generic to match serializers since some serialization formats do not require a base class of Object; update many other classes to have generic key and value class type parameters to match this change.
b194c73 [Ewen Cheslack-Postava] Split Copycat converter option into two options for key and value.
0b5a1a0 [Ewen Cheslack-Postava] Normalize naming to use partition for both source and Kafka, adjusting naming in CopycatRecord classes to clearly differentiate.
e345142 [Ewen Cheslack-Postava] Remove Copycat reflection utils, use existing Utils and ConfigDef functionality from clients package.
be5c387 [Ewen Cheslack-Postava] Minor cleanup
122423e [Ewen Cheslack-Postava] Style cleanup
6ba87de [Ewen Cheslack-Postava] Remove most of the Avro-based mock runtime data API, only preserving enough schema functionality to support basic primitive types for an initial patch.
4674d13 [Ewen Cheslack-Postava] Address review comments, clean up some code styling.
25b5739 [Ewen Cheslack-Postava] Fix sink task offset commit concurrency issue by moving it to the worker thread and waking up the consumer to ensure it exits promptly.
0aefe21 [Ewen Cheslack-Postava] Add log4j settings for Copycat.
220e42d [Ewen Cheslack-Postava] Replace Avro serializer with JSON serializer.
1243a7c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
5a618c6 [Ewen Cheslack-Postava] Remove offset serializers, instead reusing the existing serializers and removing schema projection support.
e849e10 [Ewen Cheslack-Postava] Remove duplicated TopicPartition implementation.
dec1379 [Ewen Cheslack-Postava] Switch to using new consumer coordinator instead of manually assigning partitions. Remove dependency of copycat-runtime on core.
4a9b4f3 [Ewen Cheslack-Postava] Add some helpful Copycat-specific build and test targets that cover all Copycat packages.
31cd1ca [Ewen Cheslack-Postava] Add CLI tools for Copycat.
e14942c [Ewen Cheslack-Postava] Add Copycat file connector.
0233456 [Ewen Cheslack-Postava] Add copycat-avro and copycat-runtime
11981d2 [Ewen Cheslack-Postava] Add copycat-data and copycat-api
9 years ago
|
|
|
|
|
|
|
dependencies {
|
|
|
|
compile project(':connect:api')
|
|
|
|
compile libs.jacksonDatabind
|
|
|
|
compile libs.slf4jApi
|
KAFKA-2366; Initial patch for Copycat
This is an initial patch implementing the basics of Copycat for KIP-26.
The intent here is to start a review of the key pieces of the core API and get a reasonably functional, baseline, non-distributed implementation of Copycat in place to get things rolling. The current patch has a number of known issues that need to be addressed before a final version:
* Some build-related issues. Specifically, requires some locally-installed dependencies (see below), ignores checkstyle for the runtime data library because it's lifted from Avro currently and likely won't last in its current form, and some Gradle task dependencies aren't quite right because I haven't gotten rid of the dependency on `core` (which should now be an easy patch since new consumer groups are in a much better state).
* This patch currently depends on some Confluent trunk code because I prototyped with our Avro serializers w/ schema-registry support. We need to figure out what we want to provide as an example built-in set of serializers. Unlike core Kafka where we could ignore the issue, providing only ByteArray or String serializers, this is pretty central to how Copycat works.
* This patch uses a hacked up version of Avro as its runtime data format. Not sure if we want to go through the entire API discussion just to get some basic code committed, so I filed KAFKA-2367 to handle that separately. The core connector APIs and the runtime data APIs are entirely orthogonal.
* This patch needs some updates to get aligned with recent new consumer changes (specifically, I'm aware of the ConcurrentModificationException issue on exit). More generally, the new consumer is in flux but Copycat depends on it, so there are likely to be some negative interactions.
* The layout feels a bit awkward to me right now because I ported it from a Maven layout. We don't have nearly the same level of granularity in Kafka currently (core and clients, plus the mostly ignored examples, log4j-appender, and a couple of contribs). We might want to reorganize, although keeping data+api separate from runtime and connector plugins is useful for minimizing dependencies.
* There are a variety of other things (e.g., I'm not happy with the exception hierarchy/how they are currently handled, TopicPartition doesn't really need to be duplicated unless we want Copycat entirely isolated from the Kafka APIs, etc), but I expect those we'll cover in the review.
Before commenting on the patch, it's probably worth reviewing https://issues.apache.org/jira/browse/KAFKA-2365 and https://issues.apache.org/jira/browse/KAFKA-2366 to get an idea of what I had in mind for a) what we ultimately want with all the Copycat patches and b) what we aim to cover in this initial patch. My hope is that we can use a WIP patch (after the current obvious deficiencies are addressed) while recognizing that we want to make iterative progress with a bunch of subsequent PRs.
Author: Ewen Cheslack-Postava <me@ewencp.org>
Reviewers: Ismael Juma, Gwen Shapira
Closes #99 from ewencp/copycat and squashes the following commits:
a3a47a6 [Ewen Cheslack-Postava] Simplify Copycat exceptions, make them a subclass of KafkaException.
8c108b0 [Ewen Cheslack-Postava] Rename Coordinator to Herder to avoid confusion with the consumer coordinator.
7bf8075 [Ewen Cheslack-Postava] Make Copycat CLI speific to standalone mode, clean up some config and get rid of config storage in standalone mode.
656a003 [Ewen Cheslack-Postava] Clarify and expand the explanation of the Copycat Coordinator interface.
c0e5fdc [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
0fa7a36 [Ewen Cheslack-Postava] Mark Copycat classes as unstable and reduce visibility of some classes where possible.
d55d31e [Ewen Cheslack-Postava] Reorganize Copycat code to put it all under one top-level directory.
b29cb2c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
d713a21 [Ewen Cheslack-Postava] Address Gwen's review comments.
6787a85 [Ewen Cheslack-Postava] Make Converter generic to match serializers since some serialization formats do not require a base class of Object; update many other classes to have generic key and value class type parameters to match this change.
b194c73 [Ewen Cheslack-Postava] Split Copycat converter option into two options for key and value.
0b5a1a0 [Ewen Cheslack-Postava] Normalize naming to use partition for both source and Kafka, adjusting naming in CopycatRecord classes to clearly differentiate.
e345142 [Ewen Cheslack-Postava] Remove Copycat reflection utils, use existing Utils and ConfigDef functionality from clients package.
be5c387 [Ewen Cheslack-Postava] Minor cleanup
122423e [Ewen Cheslack-Postava] Style cleanup
6ba87de [Ewen Cheslack-Postava] Remove most of the Avro-based mock runtime data API, only preserving enough schema functionality to support basic primitive types for an initial patch.
4674d13 [Ewen Cheslack-Postava] Address review comments, clean up some code styling.
25b5739 [Ewen Cheslack-Postava] Fix sink task offset commit concurrency issue by moving it to the worker thread and waking up the consumer to ensure it exits promptly.
0aefe21 [Ewen Cheslack-Postava] Add log4j settings for Copycat.
220e42d [Ewen Cheslack-Postava] Replace Avro serializer with JSON serializer.
1243a7c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
5a618c6 [Ewen Cheslack-Postava] Remove offset serializers, instead reusing the existing serializers and removing schema projection support.
e849e10 [Ewen Cheslack-Postava] Remove duplicated TopicPartition implementation.
dec1379 [Ewen Cheslack-Postava] Switch to using new consumer coordinator instead of manually assigning partitions. Remove dependency of copycat-runtime on core.
4a9b4f3 [Ewen Cheslack-Postava] Add some helpful Copycat-specific build and test targets that cover all Copycat packages.
31cd1ca [Ewen Cheslack-Postava] Add CLI tools for Copycat.
e14942c [Ewen Cheslack-Postava] Add Copycat file connector.
0233456 [Ewen Cheslack-Postava] Add copycat-avro and copycat-runtime
11981d2 [Ewen Cheslack-Postava] Add copycat-data and copycat-api
9 years ago
|
|
|
|
|
|
|
testCompile libs.easymock
|
|
|
|
testCompile libs.junit
|
|
|
|
testCompile libs.powermockJunit4
|
|
|
|
testCompile libs.powermockEasymock
|
|
|
|
|
|
|
|
testRuntime libs.slf4jlog4j
|
|
|
|
testCompile project(':clients').sourceSets.test.output
|
KAFKA-2366; Initial patch for Copycat
This is an initial patch implementing the basics of Copycat for KIP-26.
The intent here is to start a review of the key pieces of the core API and get a reasonably functional, baseline, non-distributed implementation of Copycat in place to get things rolling. The current patch has a number of known issues that need to be addressed before a final version:
* Some build-related issues. Specifically, requires some locally-installed dependencies (see below), ignores checkstyle for the runtime data library because it's lifted from Avro currently and likely won't last in its current form, and some Gradle task dependencies aren't quite right because I haven't gotten rid of the dependency on `core` (which should now be an easy patch since new consumer groups are in a much better state).
* This patch currently depends on some Confluent trunk code because I prototyped with our Avro serializers w/ schema-registry support. We need to figure out what we want to provide as an example built-in set of serializers. Unlike core Kafka where we could ignore the issue, providing only ByteArray or String serializers, this is pretty central to how Copycat works.
* This patch uses a hacked up version of Avro as its runtime data format. Not sure if we want to go through the entire API discussion just to get some basic code committed, so I filed KAFKA-2367 to handle that separately. The core connector APIs and the runtime data APIs are entirely orthogonal.
* This patch needs some updates to get aligned with recent new consumer changes (specifically, I'm aware of the ConcurrentModificationException issue on exit). More generally, the new consumer is in flux but Copycat depends on it, so there are likely to be some negative interactions.
* The layout feels a bit awkward to me right now because I ported it from a Maven layout. We don't have nearly the same level of granularity in Kafka currently (core and clients, plus the mostly ignored examples, log4j-appender, and a couple of contribs). We might want to reorganize, although keeping data+api separate from runtime and connector plugins is useful for minimizing dependencies.
* There are a variety of other things (e.g., I'm not happy with the exception hierarchy/how they are currently handled, TopicPartition doesn't really need to be duplicated unless we want Copycat entirely isolated from the Kafka APIs, etc), but I expect those we'll cover in the review.
Before commenting on the patch, it's probably worth reviewing https://issues.apache.org/jira/browse/KAFKA-2365 and https://issues.apache.org/jira/browse/KAFKA-2366 to get an idea of what I had in mind for a) what we ultimately want with all the Copycat patches and b) what we aim to cover in this initial patch. My hope is that we can use a WIP patch (after the current obvious deficiencies are addressed) while recognizing that we want to make iterative progress with a bunch of subsequent PRs.
Author: Ewen Cheslack-Postava <me@ewencp.org>
Reviewers: Ismael Juma, Gwen Shapira
Closes #99 from ewencp/copycat and squashes the following commits:
a3a47a6 [Ewen Cheslack-Postava] Simplify Copycat exceptions, make them a subclass of KafkaException.
8c108b0 [Ewen Cheslack-Postava] Rename Coordinator to Herder to avoid confusion with the consumer coordinator.
7bf8075 [Ewen Cheslack-Postava] Make Copycat CLI speific to standalone mode, clean up some config and get rid of config storage in standalone mode.
656a003 [Ewen Cheslack-Postava] Clarify and expand the explanation of the Copycat Coordinator interface.
c0e5fdc [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
0fa7a36 [Ewen Cheslack-Postava] Mark Copycat classes as unstable and reduce visibility of some classes where possible.
d55d31e [Ewen Cheslack-Postava] Reorganize Copycat code to put it all under one top-level directory.
b29cb2c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
d713a21 [Ewen Cheslack-Postava] Address Gwen's review comments.
6787a85 [Ewen Cheslack-Postava] Make Converter generic to match serializers since some serialization formats do not require a base class of Object; update many other classes to have generic key and value class type parameters to match this change.
b194c73 [Ewen Cheslack-Postava] Split Copycat converter option into two options for key and value.
0b5a1a0 [Ewen Cheslack-Postava] Normalize naming to use partition for both source and Kafka, adjusting naming in CopycatRecord classes to clearly differentiate.
e345142 [Ewen Cheslack-Postava] Remove Copycat reflection utils, use existing Utils and ConfigDef functionality from clients package.
be5c387 [Ewen Cheslack-Postava] Minor cleanup
122423e [Ewen Cheslack-Postava] Style cleanup
6ba87de [Ewen Cheslack-Postava] Remove most of the Avro-based mock runtime data API, only preserving enough schema functionality to support basic primitive types for an initial patch.
4674d13 [Ewen Cheslack-Postava] Address review comments, clean up some code styling.
25b5739 [Ewen Cheslack-Postava] Fix sink task offset commit concurrency issue by moving it to the worker thread and waking up the consumer to ensure it exits promptly.
0aefe21 [Ewen Cheslack-Postava] Add log4j settings for Copycat.
220e42d [Ewen Cheslack-Postava] Replace Avro serializer with JSON serializer.
1243a7c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
5a618c6 [Ewen Cheslack-Postava] Remove offset serializers, instead reusing the existing serializers and removing schema projection support.
e849e10 [Ewen Cheslack-Postava] Remove duplicated TopicPartition implementation.
dec1379 [Ewen Cheslack-Postava] Switch to using new consumer coordinator instead of manually assigning partitions. Remove dependency of copycat-runtime on core.
4a9b4f3 [Ewen Cheslack-Postava] Add some helpful Copycat-specific build and test targets that cover all Copycat packages.
31cd1ca [Ewen Cheslack-Postava] Add CLI tools for Copycat.
e14942c [Ewen Cheslack-Postava] Add Copycat file connector.
0233456 [Ewen Cheslack-Postava] Add copycat-avro and copycat-runtime
11981d2 [Ewen Cheslack-Postava] Add copycat-data and copycat-api
9 years ago
|
|
|
}
|
|
|
|
|
|
|
|
javadoc {
|
|
|
|
enabled = false
|
KAFKA-2366; Initial patch for Copycat
This is an initial patch implementing the basics of Copycat for KIP-26.
The intent here is to start a review of the key pieces of the core API and get a reasonably functional, baseline, non-distributed implementation of Copycat in place to get things rolling. The current patch has a number of known issues that need to be addressed before a final version:
* Some build-related issues. Specifically, requires some locally-installed dependencies (see below), ignores checkstyle for the runtime data library because it's lifted from Avro currently and likely won't last in its current form, and some Gradle task dependencies aren't quite right because I haven't gotten rid of the dependency on `core` (which should now be an easy patch since new consumer groups are in a much better state).
* This patch currently depends on some Confluent trunk code because I prototyped with our Avro serializers w/ schema-registry support. We need to figure out what we want to provide as an example built-in set of serializers. Unlike core Kafka where we could ignore the issue, providing only ByteArray or String serializers, this is pretty central to how Copycat works.
* This patch uses a hacked up version of Avro as its runtime data format. Not sure if we want to go through the entire API discussion just to get some basic code committed, so I filed KAFKA-2367 to handle that separately. The core connector APIs and the runtime data APIs are entirely orthogonal.
* This patch needs some updates to get aligned with recent new consumer changes (specifically, I'm aware of the ConcurrentModificationException issue on exit). More generally, the new consumer is in flux but Copycat depends on it, so there are likely to be some negative interactions.
* The layout feels a bit awkward to me right now because I ported it from a Maven layout. We don't have nearly the same level of granularity in Kafka currently (core and clients, plus the mostly ignored examples, log4j-appender, and a couple of contribs). We might want to reorganize, although keeping data+api separate from runtime and connector plugins is useful for minimizing dependencies.
* There are a variety of other things (e.g., I'm not happy with the exception hierarchy/how they are currently handled, TopicPartition doesn't really need to be duplicated unless we want Copycat entirely isolated from the Kafka APIs, etc), but I expect those we'll cover in the review.
Before commenting on the patch, it's probably worth reviewing https://issues.apache.org/jira/browse/KAFKA-2365 and https://issues.apache.org/jira/browse/KAFKA-2366 to get an idea of what I had in mind for a) what we ultimately want with all the Copycat patches and b) what we aim to cover in this initial patch. My hope is that we can use a WIP patch (after the current obvious deficiencies are addressed) while recognizing that we want to make iterative progress with a bunch of subsequent PRs.
Author: Ewen Cheslack-Postava <me@ewencp.org>
Reviewers: Ismael Juma, Gwen Shapira
Closes #99 from ewencp/copycat and squashes the following commits:
a3a47a6 [Ewen Cheslack-Postava] Simplify Copycat exceptions, make them a subclass of KafkaException.
8c108b0 [Ewen Cheslack-Postava] Rename Coordinator to Herder to avoid confusion with the consumer coordinator.
7bf8075 [Ewen Cheslack-Postava] Make Copycat CLI speific to standalone mode, clean up some config and get rid of config storage in standalone mode.
656a003 [Ewen Cheslack-Postava] Clarify and expand the explanation of the Copycat Coordinator interface.
c0e5fdc [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
0fa7a36 [Ewen Cheslack-Postava] Mark Copycat classes as unstable and reduce visibility of some classes where possible.
d55d31e [Ewen Cheslack-Postava] Reorganize Copycat code to put it all under one top-level directory.
b29cb2c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
d713a21 [Ewen Cheslack-Postava] Address Gwen's review comments.
6787a85 [Ewen Cheslack-Postava] Make Converter generic to match serializers since some serialization formats do not require a base class of Object; update many other classes to have generic key and value class type parameters to match this change.
b194c73 [Ewen Cheslack-Postava] Split Copycat converter option into two options for key and value.
0b5a1a0 [Ewen Cheslack-Postava] Normalize naming to use partition for both source and Kafka, adjusting naming in CopycatRecord classes to clearly differentiate.
e345142 [Ewen Cheslack-Postava] Remove Copycat reflection utils, use existing Utils and ConfigDef functionality from clients package.
be5c387 [Ewen Cheslack-Postava] Minor cleanup
122423e [Ewen Cheslack-Postava] Style cleanup
6ba87de [Ewen Cheslack-Postava] Remove most of the Avro-based mock runtime data API, only preserving enough schema functionality to support basic primitive types for an initial patch.
4674d13 [Ewen Cheslack-Postava] Address review comments, clean up some code styling.
25b5739 [Ewen Cheslack-Postava] Fix sink task offset commit concurrency issue by moving it to the worker thread and waking up the consumer to ensure it exits promptly.
0aefe21 [Ewen Cheslack-Postava] Add log4j settings for Copycat.
220e42d [Ewen Cheslack-Postava] Replace Avro serializer with JSON serializer.
1243a7c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
5a618c6 [Ewen Cheslack-Postava] Remove offset serializers, instead reusing the existing serializers and removing schema projection support.
e849e10 [Ewen Cheslack-Postava] Remove duplicated TopicPartition implementation.
dec1379 [Ewen Cheslack-Postava] Switch to using new consumer coordinator instead of manually assigning partitions. Remove dependency of copycat-runtime on core.
4a9b4f3 [Ewen Cheslack-Postava] Add some helpful Copycat-specific build and test targets that cover all Copycat packages.
31cd1ca [Ewen Cheslack-Postava] Add CLI tools for Copycat.
e14942c [Ewen Cheslack-Postava] Add Copycat file connector.
0233456 [Ewen Cheslack-Postava] Add copycat-avro and copycat-runtime
11981d2 [Ewen Cheslack-Postava] Add copycat-data and copycat-api
9 years ago
|
|
|
}
|
|
|
|
|
|
|
|
tasks.create(name: "copyDependantLibs", type: Copy) {
|
|
|
|
from (configurations.testRuntime) {
|
|
|
|
include('slf4j-log4j12*')
|
|
|
|
include('log4j*jar')
|
|
|
|
}
|
|
|
|
from (configurations.runtime) {
|
|
|
|
exclude('kafka-clients*')
|
|
|
|
exclude('connect-*')
|
|
|
|
}
|
|
|
|
into "$buildDir/dependant-libs"
|
|
|
|
duplicatesStrategy 'exclude'
|
|
|
|
}
|
|
|
|
|
|
|
|
jar {
|
|
|
|
dependsOn copyDependantLibs
|
|
|
|
}
|
KAFKA-2366; Initial patch for Copycat
This is an initial patch implementing the basics of Copycat for KIP-26.
The intent here is to start a review of the key pieces of the core API and get a reasonably functional, baseline, non-distributed implementation of Copycat in place to get things rolling. The current patch has a number of known issues that need to be addressed before a final version:
* Some build-related issues. Specifically, requires some locally-installed dependencies (see below), ignores checkstyle for the runtime data library because it's lifted from Avro currently and likely won't last in its current form, and some Gradle task dependencies aren't quite right because I haven't gotten rid of the dependency on `core` (which should now be an easy patch since new consumer groups are in a much better state).
* This patch currently depends on some Confluent trunk code because I prototyped with our Avro serializers w/ schema-registry support. We need to figure out what we want to provide as an example built-in set of serializers. Unlike core Kafka where we could ignore the issue, providing only ByteArray or String serializers, this is pretty central to how Copycat works.
* This patch uses a hacked up version of Avro as its runtime data format. Not sure if we want to go through the entire API discussion just to get some basic code committed, so I filed KAFKA-2367 to handle that separately. The core connector APIs and the runtime data APIs are entirely orthogonal.
* This patch needs some updates to get aligned with recent new consumer changes (specifically, I'm aware of the ConcurrentModificationException issue on exit). More generally, the new consumer is in flux but Copycat depends on it, so there are likely to be some negative interactions.
* The layout feels a bit awkward to me right now because I ported it from a Maven layout. We don't have nearly the same level of granularity in Kafka currently (core and clients, plus the mostly ignored examples, log4j-appender, and a couple of contribs). We might want to reorganize, although keeping data+api separate from runtime and connector plugins is useful for minimizing dependencies.
* There are a variety of other things (e.g., I'm not happy with the exception hierarchy/how they are currently handled, TopicPartition doesn't really need to be duplicated unless we want Copycat entirely isolated from the Kafka APIs, etc), but I expect those we'll cover in the review.
Before commenting on the patch, it's probably worth reviewing https://issues.apache.org/jira/browse/KAFKA-2365 and https://issues.apache.org/jira/browse/KAFKA-2366 to get an idea of what I had in mind for a) what we ultimately want with all the Copycat patches and b) what we aim to cover in this initial patch. My hope is that we can use a WIP patch (after the current obvious deficiencies are addressed) while recognizing that we want to make iterative progress with a bunch of subsequent PRs.
Author: Ewen Cheslack-Postava <me@ewencp.org>
Reviewers: Ismael Juma, Gwen Shapira
Closes #99 from ewencp/copycat and squashes the following commits:
a3a47a6 [Ewen Cheslack-Postava] Simplify Copycat exceptions, make them a subclass of KafkaException.
8c108b0 [Ewen Cheslack-Postava] Rename Coordinator to Herder to avoid confusion with the consumer coordinator.
7bf8075 [Ewen Cheslack-Postava] Make Copycat CLI speific to standalone mode, clean up some config and get rid of config storage in standalone mode.
656a003 [Ewen Cheslack-Postava] Clarify and expand the explanation of the Copycat Coordinator interface.
c0e5fdc [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
0fa7a36 [Ewen Cheslack-Postava] Mark Copycat classes as unstable and reduce visibility of some classes where possible.
d55d31e [Ewen Cheslack-Postava] Reorganize Copycat code to put it all under one top-level directory.
b29cb2c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
d713a21 [Ewen Cheslack-Postava] Address Gwen's review comments.
6787a85 [Ewen Cheslack-Postava] Make Converter generic to match serializers since some serialization formats do not require a base class of Object; update many other classes to have generic key and value class type parameters to match this change.
b194c73 [Ewen Cheslack-Postava] Split Copycat converter option into two options for key and value.
0b5a1a0 [Ewen Cheslack-Postava] Normalize naming to use partition for both source and Kafka, adjusting naming in CopycatRecord classes to clearly differentiate.
e345142 [Ewen Cheslack-Postava] Remove Copycat reflection utils, use existing Utils and ConfigDef functionality from clients package.
be5c387 [Ewen Cheslack-Postava] Minor cleanup
122423e [Ewen Cheslack-Postava] Style cleanup
6ba87de [Ewen Cheslack-Postava] Remove most of the Avro-based mock runtime data API, only preserving enough schema functionality to support basic primitive types for an initial patch.
4674d13 [Ewen Cheslack-Postava] Address review comments, clean up some code styling.
25b5739 [Ewen Cheslack-Postava] Fix sink task offset commit concurrency issue by moving it to the worker thread and waking up the consumer to ensure it exits promptly.
0aefe21 [Ewen Cheslack-Postava] Add log4j settings for Copycat.
220e42d [Ewen Cheslack-Postava] Replace Avro serializer with JSON serializer.
1243a7c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
5a618c6 [Ewen Cheslack-Postava] Remove offset serializers, instead reusing the existing serializers and removing schema projection support.
e849e10 [Ewen Cheslack-Postava] Remove duplicated TopicPartition implementation.
dec1379 [Ewen Cheslack-Postava] Switch to using new consumer coordinator instead of manually assigning partitions. Remove dependency of copycat-runtime on core.
4a9b4f3 [Ewen Cheslack-Postava] Add some helpful Copycat-specific build and test targets that cover all Copycat packages.
31cd1ca [Ewen Cheslack-Postava] Add CLI tools for Copycat.
e14942c [Ewen Cheslack-Postava] Add Copycat file connector.
0233456 [Ewen Cheslack-Postava] Add copycat-avro and copycat-runtime
11981d2 [Ewen Cheslack-Postava] Add copycat-data and copycat-api
9 years ago
|
|
|
}
|
|
|
|
|
|
|
|
project(':connect:runtime') {
|
|
|
|
archivesBaseName = "connect-runtime"
|
KAFKA-2366; Initial patch for Copycat
This is an initial patch implementing the basics of Copycat for KIP-26.
The intent here is to start a review of the key pieces of the core API and get a reasonably functional, baseline, non-distributed implementation of Copycat in place to get things rolling. The current patch has a number of known issues that need to be addressed before a final version:
* Some build-related issues. Specifically, requires some locally-installed dependencies (see below), ignores checkstyle for the runtime data library because it's lifted from Avro currently and likely won't last in its current form, and some Gradle task dependencies aren't quite right because I haven't gotten rid of the dependency on `core` (which should now be an easy patch since new consumer groups are in a much better state).
* This patch currently depends on some Confluent trunk code because I prototyped with our Avro serializers w/ schema-registry support. We need to figure out what we want to provide as an example built-in set of serializers. Unlike core Kafka where we could ignore the issue, providing only ByteArray or String serializers, this is pretty central to how Copycat works.
* This patch uses a hacked up version of Avro as its runtime data format. Not sure if we want to go through the entire API discussion just to get some basic code committed, so I filed KAFKA-2367 to handle that separately. The core connector APIs and the runtime data APIs are entirely orthogonal.
* This patch needs some updates to get aligned with recent new consumer changes (specifically, I'm aware of the ConcurrentModificationException issue on exit). More generally, the new consumer is in flux but Copycat depends on it, so there are likely to be some negative interactions.
* The layout feels a bit awkward to me right now because I ported it from a Maven layout. We don't have nearly the same level of granularity in Kafka currently (core and clients, plus the mostly ignored examples, log4j-appender, and a couple of contribs). We might want to reorganize, although keeping data+api separate from runtime and connector plugins is useful for minimizing dependencies.
* There are a variety of other things (e.g., I'm not happy with the exception hierarchy/how they are currently handled, TopicPartition doesn't really need to be duplicated unless we want Copycat entirely isolated from the Kafka APIs, etc), but I expect those we'll cover in the review.
Before commenting on the patch, it's probably worth reviewing https://issues.apache.org/jira/browse/KAFKA-2365 and https://issues.apache.org/jira/browse/KAFKA-2366 to get an idea of what I had in mind for a) what we ultimately want with all the Copycat patches and b) what we aim to cover in this initial patch. My hope is that we can use a WIP patch (after the current obvious deficiencies are addressed) while recognizing that we want to make iterative progress with a bunch of subsequent PRs.
Author: Ewen Cheslack-Postava <me@ewencp.org>
Reviewers: Ismael Juma, Gwen Shapira
Closes #99 from ewencp/copycat and squashes the following commits:
a3a47a6 [Ewen Cheslack-Postava] Simplify Copycat exceptions, make them a subclass of KafkaException.
8c108b0 [Ewen Cheslack-Postava] Rename Coordinator to Herder to avoid confusion with the consumer coordinator.
7bf8075 [Ewen Cheslack-Postava] Make Copycat CLI speific to standalone mode, clean up some config and get rid of config storage in standalone mode.
656a003 [Ewen Cheslack-Postava] Clarify and expand the explanation of the Copycat Coordinator interface.
c0e5fdc [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
0fa7a36 [Ewen Cheslack-Postava] Mark Copycat classes as unstable and reduce visibility of some classes where possible.
d55d31e [Ewen Cheslack-Postava] Reorganize Copycat code to put it all under one top-level directory.
b29cb2c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
d713a21 [Ewen Cheslack-Postava] Address Gwen's review comments.
6787a85 [Ewen Cheslack-Postava] Make Converter generic to match serializers since some serialization formats do not require a base class of Object; update many other classes to have generic key and value class type parameters to match this change.
b194c73 [Ewen Cheslack-Postava] Split Copycat converter option into two options for key and value.
0b5a1a0 [Ewen Cheslack-Postava] Normalize naming to use partition for both source and Kafka, adjusting naming in CopycatRecord classes to clearly differentiate.
e345142 [Ewen Cheslack-Postava] Remove Copycat reflection utils, use existing Utils and ConfigDef functionality from clients package.
be5c387 [Ewen Cheslack-Postava] Minor cleanup
122423e [Ewen Cheslack-Postava] Style cleanup
6ba87de [Ewen Cheslack-Postava] Remove most of the Avro-based mock runtime data API, only preserving enough schema functionality to support basic primitive types for an initial patch.
4674d13 [Ewen Cheslack-Postava] Address review comments, clean up some code styling.
25b5739 [Ewen Cheslack-Postava] Fix sink task offset commit concurrency issue by moving it to the worker thread and waking up the consumer to ensure it exits promptly.
0aefe21 [Ewen Cheslack-Postava] Add log4j settings for Copycat.
220e42d [Ewen Cheslack-Postava] Replace Avro serializer with JSON serializer.
1243a7c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
5a618c6 [Ewen Cheslack-Postava] Remove offset serializers, instead reusing the existing serializers and removing schema projection support.
e849e10 [Ewen Cheslack-Postava] Remove duplicated TopicPartition implementation.
dec1379 [Ewen Cheslack-Postava] Switch to using new consumer coordinator instead of manually assigning partitions. Remove dependency of copycat-runtime on core.
4a9b4f3 [Ewen Cheslack-Postava] Add some helpful Copycat-specific build and test targets that cover all Copycat packages.
31cd1ca [Ewen Cheslack-Postava] Add CLI tools for Copycat.
e14942c [Ewen Cheslack-Postava] Add Copycat file connector.
0233456 [Ewen Cheslack-Postava] Add copycat-avro and copycat-runtime
11981d2 [Ewen Cheslack-Postava] Add copycat-data and copycat-api
9 years ago
|
|
|
|
|
|
|
dependencies {
|
|
|
|
|
|
|
|
compile project(':connect:api')
|
KAFKA-2366; Initial patch for Copycat
This is an initial patch implementing the basics of Copycat for KIP-26.
The intent here is to start a review of the key pieces of the core API and get a reasonably functional, baseline, non-distributed implementation of Copycat in place to get things rolling. The current patch has a number of known issues that need to be addressed before a final version:
* Some build-related issues. Specifically, requires some locally-installed dependencies (see below), ignores checkstyle for the runtime data library because it's lifted from Avro currently and likely won't last in its current form, and some Gradle task dependencies aren't quite right because I haven't gotten rid of the dependency on `core` (which should now be an easy patch since new consumer groups are in a much better state).
* This patch currently depends on some Confluent trunk code because I prototyped with our Avro serializers w/ schema-registry support. We need to figure out what we want to provide as an example built-in set of serializers. Unlike core Kafka where we could ignore the issue, providing only ByteArray or String serializers, this is pretty central to how Copycat works.
* This patch uses a hacked up version of Avro as its runtime data format. Not sure if we want to go through the entire API discussion just to get some basic code committed, so I filed KAFKA-2367 to handle that separately. The core connector APIs and the runtime data APIs are entirely orthogonal.
* This patch needs some updates to get aligned with recent new consumer changes (specifically, I'm aware of the ConcurrentModificationException issue on exit). More generally, the new consumer is in flux but Copycat depends on it, so there are likely to be some negative interactions.
* The layout feels a bit awkward to me right now because I ported it from a Maven layout. We don't have nearly the same level of granularity in Kafka currently (core and clients, plus the mostly ignored examples, log4j-appender, and a couple of contribs). We might want to reorganize, although keeping data+api separate from runtime and connector plugins is useful for minimizing dependencies.
* There are a variety of other things (e.g., I'm not happy with the exception hierarchy/how they are currently handled, TopicPartition doesn't really need to be duplicated unless we want Copycat entirely isolated from the Kafka APIs, etc), but I expect those we'll cover in the review.
Before commenting on the patch, it's probably worth reviewing https://issues.apache.org/jira/browse/KAFKA-2365 and https://issues.apache.org/jira/browse/KAFKA-2366 to get an idea of what I had in mind for a) what we ultimately want with all the Copycat patches and b) what we aim to cover in this initial patch. My hope is that we can use a WIP patch (after the current obvious deficiencies are addressed) while recognizing that we want to make iterative progress with a bunch of subsequent PRs.
Author: Ewen Cheslack-Postava <me@ewencp.org>
Reviewers: Ismael Juma, Gwen Shapira
Closes #99 from ewencp/copycat and squashes the following commits:
a3a47a6 [Ewen Cheslack-Postava] Simplify Copycat exceptions, make them a subclass of KafkaException.
8c108b0 [Ewen Cheslack-Postava] Rename Coordinator to Herder to avoid confusion with the consumer coordinator.
7bf8075 [Ewen Cheslack-Postava] Make Copycat CLI speific to standalone mode, clean up some config and get rid of config storage in standalone mode.
656a003 [Ewen Cheslack-Postava] Clarify and expand the explanation of the Copycat Coordinator interface.
c0e5fdc [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
0fa7a36 [Ewen Cheslack-Postava] Mark Copycat classes as unstable and reduce visibility of some classes where possible.
d55d31e [Ewen Cheslack-Postava] Reorganize Copycat code to put it all under one top-level directory.
b29cb2c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
d713a21 [Ewen Cheslack-Postava] Address Gwen's review comments.
6787a85 [Ewen Cheslack-Postava] Make Converter generic to match serializers since some serialization formats do not require a base class of Object; update many other classes to have generic key and value class type parameters to match this change.
b194c73 [Ewen Cheslack-Postava] Split Copycat converter option into two options for key and value.
0b5a1a0 [Ewen Cheslack-Postava] Normalize naming to use partition for both source and Kafka, adjusting naming in CopycatRecord classes to clearly differentiate.
e345142 [Ewen Cheslack-Postava] Remove Copycat reflection utils, use existing Utils and ConfigDef functionality from clients package.
be5c387 [Ewen Cheslack-Postava] Minor cleanup
122423e [Ewen Cheslack-Postava] Style cleanup
6ba87de [Ewen Cheslack-Postava] Remove most of the Avro-based mock runtime data API, only preserving enough schema functionality to support basic primitive types for an initial patch.
4674d13 [Ewen Cheslack-Postava] Address review comments, clean up some code styling.
25b5739 [Ewen Cheslack-Postava] Fix sink task offset commit concurrency issue by moving it to the worker thread and waking up the consumer to ensure it exits promptly.
0aefe21 [Ewen Cheslack-Postava] Add log4j settings for Copycat.
220e42d [Ewen Cheslack-Postava] Replace Avro serializer with JSON serializer.
1243a7c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
5a618c6 [Ewen Cheslack-Postava] Remove offset serializers, instead reusing the existing serializers and removing schema projection support.
e849e10 [Ewen Cheslack-Postava] Remove duplicated TopicPartition implementation.
dec1379 [Ewen Cheslack-Postava] Switch to using new consumer coordinator instead of manually assigning partitions. Remove dependency of copycat-runtime on core.
4a9b4f3 [Ewen Cheslack-Postava] Add some helpful Copycat-specific build and test targets that cover all Copycat packages.
31cd1ca [Ewen Cheslack-Postava] Add CLI tools for Copycat.
e14942c [Ewen Cheslack-Postava] Add Copycat file connector.
0233456 [Ewen Cheslack-Postava] Add copycat-avro and copycat-runtime
11981d2 [Ewen Cheslack-Postava] Add copycat-data and copycat-api
9 years ago
|
|
|
compile project(':clients')
|
|
|
|
compile project(':tools')
|
|
|
|
compile project(':connect:json')
|
|
|
|
compile project(':connect:transforms')
|
|
|
|
|
|
|
|
compile libs.slf4jApi
|
|
|
|
compile libs.jacksonJaxrsJsonProvider
|
|
|
|
compile libs.jerseyContainerServlet
|
|
|
|
compile libs.jerseyHk2
|
|
|
|
compile libs.jaxbApi // Jersey dependency that was available in the JDK before Java 9
|
|
|
|
compile libs.activation // Jersey dependency that was available in the JDK before Java 9
|
|
|
|
compile libs.jettyServer
|
|
|
|
compile libs.jettyServlet
|
|
|
|
compile libs.jettyServlets
|
|
|
|
compile libs.jettyClient
|
|
|
|
compile(libs.reflections)
|
|
|
|
compile(libs.mavenArtifact)
|
|
|
|
|
|
|
|
testCompile project(':clients').sourceSets.test.output
|
|
|
|
testCompile libs.easymock
|
|
|
|
testCompile libs.junit
|
|
|
|
testCompile libs.powermockJunit4
|
|
|
|
testCompile libs.powermockEasymock
|
|
|
|
|
|
|
|
testCompile project(':clients').sourceSets.test.output
|
|
|
|
|
|
|
|
testRuntime libs.slf4jlog4j
|
KAFKA-2366; Initial patch for Copycat
This is an initial patch implementing the basics of Copycat for KIP-26.
The intent here is to start a review of the key pieces of the core API and get a reasonably functional, baseline, non-distributed implementation of Copycat in place to get things rolling. The current patch has a number of known issues that need to be addressed before a final version:
* Some build-related issues. Specifically, requires some locally-installed dependencies (see below), ignores checkstyle for the runtime data library because it's lifted from Avro currently and likely won't last in its current form, and some Gradle task dependencies aren't quite right because I haven't gotten rid of the dependency on `core` (which should now be an easy patch since new consumer groups are in a much better state).
* This patch currently depends on some Confluent trunk code because I prototyped with our Avro serializers w/ schema-registry support. We need to figure out what we want to provide as an example built-in set of serializers. Unlike core Kafka where we could ignore the issue, providing only ByteArray or String serializers, this is pretty central to how Copycat works.
* This patch uses a hacked up version of Avro as its runtime data format. Not sure if we want to go through the entire API discussion just to get some basic code committed, so I filed KAFKA-2367 to handle that separately. The core connector APIs and the runtime data APIs are entirely orthogonal.
* This patch needs some updates to get aligned with recent new consumer changes (specifically, I'm aware of the ConcurrentModificationException issue on exit). More generally, the new consumer is in flux but Copycat depends on it, so there are likely to be some negative interactions.
* The layout feels a bit awkward to me right now because I ported it from a Maven layout. We don't have nearly the same level of granularity in Kafka currently (core and clients, plus the mostly ignored examples, log4j-appender, and a couple of contribs). We might want to reorganize, although keeping data+api separate from runtime and connector plugins is useful for minimizing dependencies.
* There are a variety of other things (e.g., I'm not happy with the exception hierarchy/how they are currently handled, TopicPartition doesn't really need to be duplicated unless we want Copycat entirely isolated from the Kafka APIs, etc), but I expect those we'll cover in the review.
Before commenting on the patch, it's probably worth reviewing https://issues.apache.org/jira/browse/KAFKA-2365 and https://issues.apache.org/jira/browse/KAFKA-2366 to get an idea of what I had in mind for a) what we ultimately want with all the Copycat patches and b) what we aim to cover in this initial patch. My hope is that we can use a WIP patch (after the current obvious deficiencies are addressed) while recognizing that we want to make iterative progress with a bunch of subsequent PRs.
Author: Ewen Cheslack-Postava <me@ewencp.org>
Reviewers: Ismael Juma, Gwen Shapira
Closes #99 from ewencp/copycat and squashes the following commits:
a3a47a6 [Ewen Cheslack-Postava] Simplify Copycat exceptions, make them a subclass of KafkaException.
8c108b0 [Ewen Cheslack-Postava] Rename Coordinator to Herder to avoid confusion with the consumer coordinator.
7bf8075 [Ewen Cheslack-Postava] Make Copycat CLI speific to standalone mode, clean up some config and get rid of config storage in standalone mode.
656a003 [Ewen Cheslack-Postava] Clarify and expand the explanation of the Copycat Coordinator interface.
c0e5fdc [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
0fa7a36 [Ewen Cheslack-Postava] Mark Copycat classes as unstable and reduce visibility of some classes where possible.
d55d31e [Ewen Cheslack-Postava] Reorganize Copycat code to put it all under one top-level directory.
b29cb2c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
d713a21 [Ewen Cheslack-Postava] Address Gwen's review comments.
6787a85 [Ewen Cheslack-Postava] Make Converter generic to match serializers since some serialization formats do not require a base class of Object; update many other classes to have generic key and value class type parameters to match this change.
b194c73 [Ewen Cheslack-Postava] Split Copycat converter option into two options for key and value.
0b5a1a0 [Ewen Cheslack-Postava] Normalize naming to use partition for both source and Kafka, adjusting naming in CopycatRecord classes to clearly differentiate.
e345142 [Ewen Cheslack-Postava] Remove Copycat reflection utils, use existing Utils and ConfigDef functionality from clients package.
be5c387 [Ewen Cheslack-Postava] Minor cleanup
122423e [Ewen Cheslack-Postava] Style cleanup
6ba87de [Ewen Cheslack-Postava] Remove most of the Avro-based mock runtime data API, only preserving enough schema functionality to support basic primitive types for an initial patch.
4674d13 [Ewen Cheslack-Postava] Address review comments, clean up some code styling.
25b5739 [Ewen Cheslack-Postava] Fix sink task offset commit concurrency issue by moving it to the worker thread and waking up the consumer to ensure it exits promptly.
0aefe21 [Ewen Cheslack-Postava] Add log4j settings for Copycat.
220e42d [Ewen Cheslack-Postava] Replace Avro serializer with JSON serializer.
1243a7c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
5a618c6 [Ewen Cheslack-Postava] Remove offset serializers, instead reusing the existing serializers and removing schema projection support.
e849e10 [Ewen Cheslack-Postava] Remove duplicated TopicPartition implementation.
dec1379 [Ewen Cheslack-Postava] Switch to using new consumer coordinator instead of manually assigning partitions. Remove dependency of copycat-runtime on core.
4a9b4f3 [Ewen Cheslack-Postava] Add some helpful Copycat-specific build and test targets that cover all Copycat packages.
31cd1ca [Ewen Cheslack-Postava] Add CLI tools for Copycat.
e14942c [Ewen Cheslack-Postava] Add Copycat file connector.
0233456 [Ewen Cheslack-Postava] Add copycat-avro and copycat-runtime
11981d2 [Ewen Cheslack-Postava] Add copycat-data and copycat-api
9 years ago
|
|
|
}
|
|
|
|
|
|
|
|
javadoc {
|
|
|
|
enabled = false
|
KAFKA-2366; Initial patch for Copycat
This is an initial patch implementing the basics of Copycat for KIP-26.
The intent here is to start a review of the key pieces of the core API and get a reasonably functional, baseline, non-distributed implementation of Copycat in place to get things rolling. The current patch has a number of known issues that need to be addressed before a final version:
* Some build-related issues. Specifically, requires some locally-installed dependencies (see below), ignores checkstyle for the runtime data library because it's lifted from Avro currently and likely won't last in its current form, and some Gradle task dependencies aren't quite right because I haven't gotten rid of the dependency on `core` (which should now be an easy patch since new consumer groups are in a much better state).
* This patch currently depends on some Confluent trunk code because I prototyped with our Avro serializers w/ schema-registry support. We need to figure out what we want to provide as an example built-in set of serializers. Unlike core Kafka where we could ignore the issue, providing only ByteArray or String serializers, this is pretty central to how Copycat works.
* This patch uses a hacked up version of Avro as its runtime data format. Not sure if we want to go through the entire API discussion just to get some basic code committed, so I filed KAFKA-2367 to handle that separately. The core connector APIs and the runtime data APIs are entirely orthogonal.
* This patch needs some updates to get aligned with recent new consumer changes (specifically, I'm aware of the ConcurrentModificationException issue on exit). More generally, the new consumer is in flux but Copycat depends on it, so there are likely to be some negative interactions.
* The layout feels a bit awkward to me right now because I ported it from a Maven layout. We don't have nearly the same level of granularity in Kafka currently (core and clients, plus the mostly ignored examples, log4j-appender, and a couple of contribs). We might want to reorganize, although keeping data+api separate from runtime and connector plugins is useful for minimizing dependencies.
* There are a variety of other things (e.g., I'm not happy with the exception hierarchy/how they are currently handled, TopicPartition doesn't really need to be duplicated unless we want Copycat entirely isolated from the Kafka APIs, etc), but I expect those we'll cover in the review.
Before commenting on the patch, it's probably worth reviewing https://issues.apache.org/jira/browse/KAFKA-2365 and https://issues.apache.org/jira/browse/KAFKA-2366 to get an idea of what I had in mind for a) what we ultimately want with all the Copycat patches and b) what we aim to cover in this initial patch. My hope is that we can use a WIP patch (after the current obvious deficiencies are addressed) while recognizing that we want to make iterative progress with a bunch of subsequent PRs.
Author: Ewen Cheslack-Postava <me@ewencp.org>
Reviewers: Ismael Juma, Gwen Shapira
Closes #99 from ewencp/copycat and squashes the following commits:
a3a47a6 [Ewen Cheslack-Postava] Simplify Copycat exceptions, make them a subclass of KafkaException.
8c108b0 [Ewen Cheslack-Postava] Rename Coordinator to Herder to avoid confusion with the consumer coordinator.
7bf8075 [Ewen Cheslack-Postava] Make Copycat CLI speific to standalone mode, clean up some config and get rid of config storage in standalone mode.
656a003 [Ewen Cheslack-Postava] Clarify and expand the explanation of the Copycat Coordinator interface.
c0e5fdc [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
0fa7a36 [Ewen Cheslack-Postava] Mark Copycat classes as unstable and reduce visibility of some classes where possible.
d55d31e [Ewen Cheslack-Postava] Reorganize Copycat code to put it all under one top-level directory.
b29cb2c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
d713a21 [Ewen Cheslack-Postava] Address Gwen's review comments.
6787a85 [Ewen Cheslack-Postava] Make Converter generic to match serializers since some serialization formats do not require a base class of Object; update many other classes to have generic key and value class type parameters to match this change.
b194c73 [Ewen Cheslack-Postava] Split Copycat converter option into two options for key and value.
0b5a1a0 [Ewen Cheslack-Postava] Normalize naming to use partition for both source and Kafka, adjusting naming in CopycatRecord classes to clearly differentiate.
e345142 [Ewen Cheslack-Postava] Remove Copycat reflection utils, use existing Utils and ConfigDef functionality from clients package.
be5c387 [Ewen Cheslack-Postava] Minor cleanup
122423e [Ewen Cheslack-Postava] Style cleanup
6ba87de [Ewen Cheslack-Postava] Remove most of the Avro-based mock runtime data API, only preserving enough schema functionality to support basic primitive types for an initial patch.
4674d13 [Ewen Cheslack-Postava] Address review comments, clean up some code styling.
25b5739 [Ewen Cheslack-Postava] Fix sink task offset commit concurrency issue by moving it to the worker thread and waking up the consumer to ensure it exits promptly.
0aefe21 [Ewen Cheslack-Postava] Add log4j settings for Copycat.
220e42d [Ewen Cheslack-Postava] Replace Avro serializer with JSON serializer.
1243a7c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
5a618c6 [Ewen Cheslack-Postava] Remove offset serializers, instead reusing the existing serializers and removing schema projection support.
e849e10 [Ewen Cheslack-Postava] Remove duplicated TopicPartition implementation.
dec1379 [Ewen Cheslack-Postava] Switch to using new consumer coordinator instead of manually assigning partitions. Remove dependency of copycat-runtime on core.
4a9b4f3 [Ewen Cheslack-Postava] Add some helpful Copycat-specific build and test targets that cover all Copycat packages.
31cd1ca [Ewen Cheslack-Postava] Add CLI tools for Copycat.
e14942c [Ewen Cheslack-Postava] Add Copycat file connector.
0233456 [Ewen Cheslack-Postava] Add copycat-avro and copycat-runtime
11981d2 [Ewen Cheslack-Postava] Add copycat-data and copycat-api
9 years ago
|
|
|
}
|
|
|
|
|
|
|
|
tasks.create(name: "copyDependantLibs", type: Copy) {
|
|
|
|
from (configurations.testRuntime) {
|
|
|
|
include('slf4j-log4j12*')
|
|
|
|
include('log4j*jar')
|
|
|
|
}
|
|
|
|
from (configurations.runtime) {
|
|
|
|
exclude('kafka-clients*')
|
|
|
|
exclude('connect-*')
|
|
|
|
}
|
|
|
|
into "$buildDir/dependant-libs"
|
|
|
|
duplicatesStrategy 'exclude'
|
|
|
|
}
|
|
|
|
|
|
|
|
jar {
|
|
|
|
dependsOn copyDependantLibs
|
|
|
|
}
|
|
|
|
|
|
|
|
task genConnectConfigDocs(type: JavaExec) {
|
|
|
|
classpath = sourceSets.main.runtimeClasspath
|
|
|
|
main = 'org.apache.kafka.connect.runtime.distributed.DistributedConfig'
|
|
|
|
if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
|
|
|
|
standardOutput = new File(generatedDocsDir, "connect_config.html").newOutputStream()
|
|
|
|
}
|
|
|
|
|
|
|
|
task genSinkConnectorConfigDocs(type: JavaExec) {
|
|
|
|
classpath = sourceSets.main.runtimeClasspath
|
|
|
|
main = 'org.apache.kafka.connect.runtime.SinkConnectorConfig'
|
|
|
|
if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
|
|
|
|
standardOutput = new File(generatedDocsDir, "sink_connector_config.html").newOutputStream()
|
|
|
|
}
|
|
|
|
|
|
|
|
task genSourceConnectorConfigDocs(type: JavaExec) {
|
|
|
|
classpath = sourceSets.main.runtimeClasspath
|
|
|
|
main = 'org.apache.kafka.connect.runtime.SourceConnectorConfig'
|
|
|
|
if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
|
|
|
|
standardOutput = new File(generatedDocsDir, "source_connector_config.html").newOutputStream()
|
|
|
|
}
|
|
|
|
|
|
|
|
task genConnectTransformationDocs(type: JavaExec) {
|
|
|
|
classpath = sourceSets.main.runtimeClasspath
|
|
|
|
main = 'org.apache.kafka.connect.tools.TransformationDoc'
|
|
|
|
if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
|
|
|
|
standardOutput = new File(generatedDocsDir, "connect_transforms.html").newOutputStream()
|
|
|
|
}
|
|
|
|
|
|
|
|
task genConnectMetricsDocs(type: JavaExec) {
|
|
|
|
classpath = sourceSets.test.runtimeClasspath
|
|
|
|
main = 'org.apache.kafka.connect.runtime.ConnectMetrics'
|
|
|
|
if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
|
|
|
|
standardOutput = new File(generatedDocsDir, "connect_metrics.html").newOutputStream()
|
|
|
|
}
|
|
|
|
|
KAFKA-2366; Initial patch for Copycat
This is an initial patch implementing the basics of Copycat for KIP-26.
The intent here is to start a review of the key pieces of the core API and get a reasonably functional, baseline, non-distributed implementation of Copycat in place to get things rolling. The current patch has a number of known issues that need to be addressed before a final version:
* Some build-related issues. Specifically, requires some locally-installed dependencies (see below), ignores checkstyle for the runtime data library because it's lifted from Avro currently and likely won't last in its current form, and some Gradle task dependencies aren't quite right because I haven't gotten rid of the dependency on `core` (which should now be an easy patch since new consumer groups are in a much better state).
* This patch currently depends on some Confluent trunk code because I prototyped with our Avro serializers w/ schema-registry support. We need to figure out what we want to provide as an example built-in set of serializers. Unlike core Kafka where we could ignore the issue, providing only ByteArray or String serializers, this is pretty central to how Copycat works.
* This patch uses a hacked up version of Avro as its runtime data format. Not sure if we want to go through the entire API discussion just to get some basic code committed, so I filed KAFKA-2367 to handle that separately. The core connector APIs and the runtime data APIs are entirely orthogonal.
* This patch needs some updates to get aligned with recent new consumer changes (specifically, I'm aware of the ConcurrentModificationException issue on exit). More generally, the new consumer is in flux but Copycat depends on it, so there are likely to be some negative interactions.
* The layout feels a bit awkward to me right now because I ported it from a Maven layout. We don't have nearly the same level of granularity in Kafka currently (core and clients, plus the mostly ignored examples, log4j-appender, and a couple of contribs). We might want to reorganize, although keeping data+api separate from runtime and connector plugins is useful for minimizing dependencies.
* There are a variety of other things (e.g., I'm not happy with the exception hierarchy/how they are currently handled, TopicPartition doesn't really need to be duplicated unless we want Copycat entirely isolated from the Kafka APIs, etc), but I expect those we'll cover in the review.
Before commenting on the patch, it's probably worth reviewing https://issues.apache.org/jira/browse/KAFKA-2365 and https://issues.apache.org/jira/browse/KAFKA-2366 to get an idea of what I had in mind for a) what we ultimately want with all the Copycat patches and b) what we aim to cover in this initial patch. My hope is that we can use a WIP patch (after the current obvious deficiencies are addressed) while recognizing that we want to make iterative progress with a bunch of subsequent PRs.
Author: Ewen Cheslack-Postava <me@ewencp.org>
Reviewers: Ismael Juma, Gwen Shapira
Closes #99 from ewencp/copycat and squashes the following commits:
a3a47a6 [Ewen Cheslack-Postava] Simplify Copycat exceptions, make them a subclass of KafkaException.
8c108b0 [Ewen Cheslack-Postava] Rename Coordinator to Herder to avoid confusion with the consumer coordinator.
7bf8075 [Ewen Cheslack-Postava] Make Copycat CLI speific to standalone mode, clean up some config and get rid of config storage in standalone mode.
656a003 [Ewen Cheslack-Postava] Clarify and expand the explanation of the Copycat Coordinator interface.
c0e5fdc [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
0fa7a36 [Ewen Cheslack-Postava] Mark Copycat classes as unstable and reduce visibility of some classes where possible.
d55d31e [Ewen Cheslack-Postava] Reorganize Copycat code to put it all under one top-level directory.
b29cb2c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
d713a21 [Ewen Cheslack-Postava] Address Gwen's review comments.
6787a85 [Ewen Cheslack-Postava] Make Converter generic to match serializers since some serialization formats do not require a base class of Object; update many other classes to have generic key and value class type parameters to match this change.
b194c73 [Ewen Cheslack-Postava] Split Copycat converter option into two options for key and value.
0b5a1a0 [Ewen Cheslack-Postava] Normalize naming to use partition for both source and Kafka, adjusting naming in CopycatRecord classes to clearly differentiate.
e345142 [Ewen Cheslack-Postava] Remove Copycat reflection utils, use existing Utils and ConfigDef functionality from clients package.
be5c387 [Ewen Cheslack-Postava] Minor cleanup
122423e [Ewen Cheslack-Postava] Style cleanup
6ba87de [Ewen Cheslack-Postava] Remove most of the Avro-based mock runtime data API, only preserving enough schema functionality to support basic primitive types for an initial patch.
4674d13 [Ewen Cheslack-Postava] Address review comments, clean up some code styling.
25b5739 [Ewen Cheslack-Postava] Fix sink task offset commit concurrency issue by moving it to the worker thread and waking up the consumer to ensure it exits promptly.
0aefe21 [Ewen Cheslack-Postava] Add log4j settings for Copycat.
220e42d [Ewen Cheslack-Postava] Replace Avro serializer with JSON serializer.
1243a7c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
5a618c6 [Ewen Cheslack-Postava] Remove offset serializers, instead reusing the existing serializers and removing schema projection support.
e849e10 [Ewen Cheslack-Postava] Remove duplicated TopicPartition implementation.
dec1379 [Ewen Cheslack-Postava] Switch to using new consumer coordinator instead of manually assigning partitions. Remove dependency of copycat-runtime on core.
4a9b4f3 [Ewen Cheslack-Postava] Add some helpful Copycat-specific build and test targets that cover all Copycat packages.
31cd1ca [Ewen Cheslack-Postava] Add CLI tools for Copycat.
e14942c [Ewen Cheslack-Postava] Add Copycat file connector.
0233456 [Ewen Cheslack-Postava] Add copycat-avro and copycat-runtime
11981d2 [Ewen Cheslack-Postava] Add copycat-data and copycat-api
9 years ago
|
|
|
}
|
|
|
|
|
|
|
|
project(':connect:file') {
|
|
|
|
archivesBaseName = "connect-file"
|
KAFKA-2366; Initial patch for Copycat
This is an initial patch implementing the basics of Copycat for KIP-26.
The intent here is to start a review of the key pieces of the core API and get a reasonably functional, baseline, non-distributed implementation of Copycat in place to get things rolling. The current patch has a number of known issues that need to be addressed before a final version:
* Some build-related issues. Specifically, requires some locally-installed dependencies (see below), ignores checkstyle for the runtime data library because it's lifted from Avro currently and likely won't last in its current form, and some Gradle task dependencies aren't quite right because I haven't gotten rid of the dependency on `core` (which should now be an easy patch since new consumer groups are in a much better state).
* This patch currently depends on some Confluent trunk code because I prototyped with our Avro serializers w/ schema-registry support. We need to figure out what we want to provide as an example built-in set of serializers. Unlike core Kafka where we could ignore the issue, providing only ByteArray or String serializers, this is pretty central to how Copycat works.
* This patch uses a hacked up version of Avro as its runtime data format. Not sure if we want to go through the entire API discussion just to get some basic code committed, so I filed KAFKA-2367 to handle that separately. The core connector APIs and the runtime data APIs are entirely orthogonal.
* This patch needs some updates to get aligned with recent new consumer changes (specifically, I'm aware of the ConcurrentModificationException issue on exit). More generally, the new consumer is in flux but Copycat depends on it, so there are likely to be some negative interactions.
* The layout feels a bit awkward to me right now because I ported it from a Maven layout. We don't have nearly the same level of granularity in Kafka currently (core and clients, plus the mostly ignored examples, log4j-appender, and a couple of contribs). We might want to reorganize, although keeping data+api separate from runtime and connector plugins is useful for minimizing dependencies.
* There are a variety of other things (e.g., I'm not happy with the exception hierarchy/how they are currently handled, TopicPartition doesn't really need to be duplicated unless we want Copycat entirely isolated from the Kafka APIs, etc), but I expect those we'll cover in the review.
Before commenting on the patch, it's probably worth reviewing https://issues.apache.org/jira/browse/KAFKA-2365 and https://issues.apache.org/jira/browse/KAFKA-2366 to get an idea of what I had in mind for a) what we ultimately want with all the Copycat patches and b) what we aim to cover in this initial patch. My hope is that we can use a WIP patch (after the current obvious deficiencies are addressed) while recognizing that we want to make iterative progress with a bunch of subsequent PRs.
Author: Ewen Cheslack-Postava <me@ewencp.org>
Reviewers: Ismael Juma, Gwen Shapira
Closes #99 from ewencp/copycat and squashes the following commits:
a3a47a6 [Ewen Cheslack-Postava] Simplify Copycat exceptions, make them a subclass of KafkaException.
8c108b0 [Ewen Cheslack-Postava] Rename Coordinator to Herder to avoid confusion with the consumer coordinator.
7bf8075 [Ewen Cheslack-Postava] Make Copycat CLI speific to standalone mode, clean up some config and get rid of config storage in standalone mode.
656a003 [Ewen Cheslack-Postava] Clarify and expand the explanation of the Copycat Coordinator interface.
c0e5fdc [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
0fa7a36 [Ewen Cheslack-Postava] Mark Copycat classes as unstable and reduce visibility of some classes where possible.
d55d31e [Ewen Cheslack-Postava] Reorganize Copycat code to put it all under one top-level directory.
b29cb2c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
d713a21 [Ewen Cheslack-Postava] Address Gwen's review comments.
6787a85 [Ewen Cheslack-Postava] Make Converter generic to match serializers since some serialization formats do not require a base class of Object; update many other classes to have generic key and value class type parameters to match this change.
b194c73 [Ewen Cheslack-Postava] Split Copycat converter option into two options for key and value.
0b5a1a0 [Ewen Cheslack-Postava] Normalize naming to use partition for both source and Kafka, adjusting naming in CopycatRecord classes to clearly differentiate.
e345142 [Ewen Cheslack-Postava] Remove Copycat reflection utils, use existing Utils and ConfigDef functionality from clients package.
be5c387 [Ewen Cheslack-Postava] Minor cleanup
122423e [Ewen Cheslack-Postava] Style cleanup
6ba87de [Ewen Cheslack-Postava] Remove most of the Avro-based mock runtime data API, only preserving enough schema functionality to support basic primitive types for an initial patch.
4674d13 [Ewen Cheslack-Postava] Address review comments, clean up some code styling.
25b5739 [Ewen Cheslack-Postava] Fix sink task offset commit concurrency issue by moving it to the worker thread and waking up the consumer to ensure it exits promptly.
0aefe21 [Ewen Cheslack-Postava] Add log4j settings for Copycat.
220e42d [Ewen Cheslack-Postava] Replace Avro serializer with JSON serializer.
1243a7c [Ewen Cheslack-Postava] Merge remote-tracking branch 'origin/trunk' into copycat
5a618c6 [Ewen Cheslack-Postava] Remove offset serializers, instead reusing the existing serializers and removing schema projection support.
e849e10 [Ewen Cheslack-Postava] Remove duplicated TopicPartition implementation.
dec1379 [Ewen Cheslack-Postava] Switch to using new consumer coordinator instead of manually assigning partitions. Remove dependency of copycat-runtime on core.
4a9b4f3 [Ewen Cheslack-Postava] Add some helpful Copycat-specific build and test targets that cover all Copycat packages.
31cd1ca [Ewen Cheslack-Postava] Add CLI tools for Copycat.
e14942c [Ewen Cheslack-Postava] Add Copycat file connector.
0233456 [Ewen Cheslack-Postava] Add copycat-avro and copycat-runtime
11981d2 [Ewen Cheslack-Postava] Add copycat-data and copycat-api
9 years ago
|
|
|
|
|
|
|
dependencies {
|
|
|
|
compile project(':connect:api')
|
|
|
|
compile libs.slf4jApi
|
|
|
|
|
|
|
|
testCompile libs.easymock
|
|
|
|
testCompile libs.junit
|
|
|
|
testCompile libs.powermockJunit4
|
|
|
|
testCompile libs.powermockEasymock
|
|
|
|
|
|
|
|
testRuntime libs.slf4jlog4j
|
|
|
|
testCompile project(':clients').sourceSets.test.output
|
|
|
|
}
|
|
|
|
|
|
|
|
javadoc {
|
|
|
|
enabled = false
|
|
|
|
}
|
|
|
|
|
|
|
|
tasks.create(name: "copyDependantLibs", type: Copy) {
|
|
|
|
from (configurations.testRuntime) {
|
|
|
|
include('slf4j-log4j12*')
|
|
|
|
include('log4j*jar')
|
|
|
|
}
|
|
|
|
from (configurations.runtime) {
|
|
|
|
exclude('kafka-clients*')
|
|
|
|
exclude('connect-*')
|
|
|
|
}
|
|
|
|
into "$buildDir/dependant-libs"
|
|
|
|
duplicatesStrategy 'exclude'
|
|
|
|
}
|
|
|
|
|
|
|
|
jar {
|
|
|
|
dependsOn copyDependantLibs
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
project(':connect:basic-auth-extension') {
|
|
|
|
archivesBaseName = "connect-basic-auth-extension"
|
|
|
|
|
|
|
|
dependencies {
|
|
|
|
compile project(':connect:api')
|
|
|
|
compile libs.slf4jApi
|
|
|
|
|
|
|
|
testCompile libs.bcpkix
|
|
|
|
testCompile libs.easymock
|
|
|
|
testCompile libs.junit
|
|
|
|
testCompile libs.powermockJunit4
|
|
|
|
testCompile libs.powermockEasymock
|
|
|
|
testCompile project(':clients').sourceSets.test.output
|
|
|
|
|
|
|
|
testRuntime libs.slf4jlog4j
|
|
|
|
testRuntime libs.jerseyContainerServlet
|
|
|
|
}
|
|
|
|
|
|
|
|
javadoc {
|
|
|
|
enabled = false
|
|
|
|
}
|
|
|
|
|
|
|
|
tasks.create(name: "copyDependantLibs", type: Copy) {
|
|
|
|
from (configurations.testRuntime) {
|
|
|
|
include('slf4j-log4j12*')
|
|
|
|
include('log4j*jar')
|
|
|
|
}
|
|
|
|
from (configurations.runtime) {
|
|
|
|
exclude('kafka-clients*')
|
|
|
|
exclude('connect-*')
|
|
|
|
}
|
|
|
|
into "$buildDir/dependant-libs"
|
|
|
|
duplicatesStrategy 'exclude'
|
|
|
|
}
|
|
|
|
|
|
|
|
jar {
|
|
|
|
dependsOn copyDependantLibs
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
task aggregatedJavadoc(type: Javadoc) {
|
|
|
|
def projectsWithJavadoc = subprojects.findAll { it.javadoc.enabled }
|
|
|
|
source = projectsWithJavadoc.collect { it.sourceSets.main.allJava }
|
|
|
|
classpath = files(projectsWithJavadoc.collect { it.sourceSets.main.compileClasspath })
|
|
|
|
includes = projectsWithJavadoc.collectMany { it.javadoc.getIncludes() }
|
|
|
|
excludes = projectsWithJavadoc.collectMany { it.javadoc.getExcludes() }
|
|
|
|
options.links "http://docs.oracle.com/javase/7/docs/api/"
|
|
|
|
}
|