Browse Source

KAFKA-7844: Use regular subproject for generator to fix *All targets (#6182)

The presence of the buildSrc subproject is causing problems when we try
to run installAll, jarAll, and the other "all" targets. It's easier
just to make the generator code a regular subproject and use the
JavaExec gradle task to run the code. This also makes it more
straightforward to run the generator unit tests.

Reviewers: David Arthur <mumrah@gmail.com>, Ismael Juma <ismael@juma.me.uk>

Co-authored-by: Colin P. Mccabe <cmccabe@confluent.io>
Co-authored-by: Stanislav Kozlovski <stanislav_kozlovski@outlook.com>
pull/6296/head
Stanislav Kozlovski 6 years ago committed by Ismael Juma
parent
commit
fb0db7602a
  1. 36
      build.gradle
  2. 26
      buildSrc/build.gradle
  3. 68
      buildSrc/src/main/java/org/apache/kafka/task/ProcessMessagesTask.java
  4. 0
      generator/src/main/java/org/apache/kafka/message/ApiMessageFactoryGenerator.java
  5. 0
      generator/src/main/java/org/apache/kafka/message/CodeBuffer.java
  6. 0
      generator/src/main/java/org/apache/kafka/message/FieldSpec.java
  7. 0
      generator/src/main/java/org/apache/kafka/message/FieldType.java
  8. 0
      generator/src/main/java/org/apache/kafka/message/HeaderGenerator.java
  9. 0
      generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java
  10. 62
      generator/src/main/java/org/apache/kafka/message/MessageGenerator.java
  11. 0
      generator/src/main/java/org/apache/kafka/message/MessageSpec.java
  12. 0
      generator/src/main/java/org/apache/kafka/message/MessageSpecType.java
  13. 0
      generator/src/main/java/org/apache/kafka/message/SchemaGenerator.java
  14. 0
      generator/src/main/java/org/apache/kafka/message/StructSpec.java
  15. 0
      generator/src/main/java/org/apache/kafka/message/Versions.java
  16. 0
      generator/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java
  17. 0
      generator/src/test/java/org/apache/kafka/message/VersionsTest.java
  18. 2
      settings.gradle

36
build.gradle

@ -249,10 +249,11 @@ subprojects { @@ -249,10 +249,11 @@ subprojects {
exceptionFormat = testExceptionFormat
}
useJUnit {
excludeCategories 'org.apache.kafka.test.IntegrationTest'
if (it.project.name != 'generator') {
useJUnit {
excludeCategories 'org.apache.kafka.test.IntegrationTest'
}
}
}
jar {
@ -823,6 +824,23 @@ project(':examples') { @@ -823,6 +824,23 @@ project(':examples') {
}
}
project(':generator') {
dependencies {
compile libs.jacksonDatabind
compile libs.jacksonJDK8Datatypes
compile libs.jacksonJaxrsJsonProvider
testCompile libs.junit
}
integrationTest {
enabled = false
}
javadoc {
enabled = false
}
}
project(':clients') {
archivesBaseName = "kafka-clients"
@ -899,9 +917,12 @@ project(':clients') { @@ -899,9 +917,12 @@ project(':clients') {
delete "$buildDir/kafka/"
}
task processMessages(type:org.apache.kafka.task.ProcessMessagesTask) {
inputDirectory = file("src/main/resources/common/message")
outputDirectory = file("src/generated/java/org/apache/kafka/common/message")
task processMessages(type:JavaExec) {
main = "org.apache.kafka.message.MessageGenerator"
classpath = project(':generator').sourceSets.main.runtimeClasspath
args = [ "src/generated/java/org/apache/kafka/common/message", "src/main/resources/common/message" ]
inputs.dir("src/main/resources/common/message")
outputs.dir("src/generated/java/org/apache/kafka/common/message")
}
sourceSets {
@ -912,8 +933,7 @@ project(':clients') { @@ -912,8 +933,7 @@ project(':clients') {
}
test {
java {
srcDirs = ["src/generated/java", "src/test/java",
"$rootDir/buildSrc/src/main/java/org/apache/kafka/message/"]
srcDirs = ["src/generated/java", "src/test/java"]
}
}
}

26
buildSrc/build.gradle

@ -1,26 +0,0 @@ @@ -1,26 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
repositories {
mavenCentral()
}
dependencies {
compile "com.fasterxml.jackson.core:jackson-databind:2.9.6"
compile "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.9.6"
compile "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.9.6"
}
test.enabled=false

68
buildSrc/src/main/java/org/apache/kafka/task/ProcessMessagesTask.java

@ -1,68 +0,0 @@ @@ -1,68 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kafka.task;
import org.apache.kafka.message.MessageGenerator;
import org.gradle.api.DefaultTask;
import org.gradle.api.tasks.InputDirectory;
import org.gradle.api.tasks.OutputFiles;
import org.gradle.api.tasks.TaskAction;
import java.io.File;
import java.util.Map;
/**
* A gradle task which processes a directory full of JSON files into an output directory.
*/
public class ProcessMessagesTask extends DefaultTask {
/**
* The directory where we should read the input JSON from.
*/
public File inputDirectory;
/**
* The directory that we should write output JSON to.
*/
public File outputDirectory;
@InputDirectory
public File getInputDirectory() {
return inputDirectory;
}
/**
* Define the task outputs.
*
* Gradle consults this to see if the task is up-to-date.
*/
@OutputFiles
public Map<String, File> getOutputFiles() throws Exception {
return MessageGenerator.getOutputFiles(
outputDirectory.toString(), inputDirectory.toString());
}
@TaskAction
public void run() {
try {
MessageGenerator.processDirectories(
outputDirectory.toString(), inputDirectory.toString());
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}

0
buildSrc/src/main/java/org/apache/kafka/message/ApiMessageFactoryGenerator.java → generator/src/main/java/org/apache/kafka/message/ApiMessageFactoryGenerator.java

0
buildSrc/src/main/java/org/apache/kafka/message/CodeBuffer.java → generator/src/main/java/org/apache/kafka/message/CodeBuffer.java

0
buildSrc/src/main/java/org/apache/kafka/message/FieldSpec.java → generator/src/main/java/org/apache/kafka/message/FieldSpec.java

0
buildSrc/src/main/java/org/apache/kafka/message/FieldType.java → generator/src/main/java/org/apache/kafka/message/FieldType.java

0
buildSrc/src/main/java/org/apache/kafka/message/HeaderGenerator.java → generator/src/main/java/org/apache/kafka/message/HeaderGenerator.java

0
buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java → generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java

62
buildSrc/src/main/java/org/apache/kafka/message/MessageGenerator.java → generator/src/main/java/org/apache/kafka/message/MessageGenerator.java

@ -24,14 +24,12 @@ import com.fasterxml.jackson.databind.ObjectMapper; @@ -24,14 +24,12 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import java.io.BufferedWriter;
import java.io.File;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Locale;
import java.util.Map;
/**
* The Kafka message generator.
@ -94,40 +92,30 @@ public final class MessageGenerator { @@ -94,40 +92,30 @@ public final class MessageGenerator {
JSON_SERDE.setSerializationInclusion(JsonInclude.Include.NON_EMPTY);
}
public static Map<String, File> getOutputFiles(String outputDir, String inputDir) throws Exception {
HashMap<String, File> outputFiles = new HashMap<>();
for (Path inputPath : Files.newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) {
String jsonName = inputPath.getFileName().toString();
String javaName = jsonName.substring(0, jsonName.length() - JSON_SUFFIX.length()) + "Data.java";
File outputFile = new File(outputDir, javaName);
outputFiles.put(outputFile.toString(), outputFile);
}
File factoryFile = new File(outputDir, API_MESSAGE_FACTORY_JAVA);
outputFiles.put(factoryFile.toString(), factoryFile);
return outputFiles;
}
public static void processDirectories(String outputDir, String inputDir) throws Exception {
Files.createDirectories(Paths.get(outputDir));
int numProcessed = 0;
ApiMessageFactoryGenerator messageFactoryGenerator = new ApiMessageFactoryGenerator();
HashSet<String> outputFileNames = new HashSet<>();
for (Path inputPath : Files.newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) {
try {
MessageSpec spec = JSON_SERDE.
readValue(inputPath.toFile(), MessageSpec.class);
String javaName = spec.generatedClassName() + JAVA_SUFFIX;
outputFileNames.add(javaName);
Path outputPath = Paths.get(outputDir, javaName);
try (BufferedWriter writer = Files.newBufferedWriter(outputPath)) {
MessageDataGenerator generator = new MessageDataGenerator();
generator.generate(spec);
generator.write(writer);
try (DirectoryStream<Path> directoryStream = Files
.newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) {
for (Path inputPath : directoryStream) {
try {
MessageSpec spec = JSON_SERDE.
readValue(inputPath.toFile(), MessageSpec.class);
String javaName = spec.generatedClassName() + JAVA_SUFFIX;
outputFileNames.add(javaName);
Path outputPath = Paths.get(outputDir, javaName);
try (BufferedWriter writer = Files.newBufferedWriter(outputPath)) {
MessageDataGenerator generator = new MessageDataGenerator();
generator.generate(spec);
generator.write(writer);
}
numProcessed++;
messageFactoryGenerator.registerMessageType(spec);
} catch (Exception e) {
throw new RuntimeException("Exception while processing " + inputPath.toString(), e);
}
numProcessed++;
messageFactoryGenerator.registerMessageType(spec);
} catch (Exception e) {
throw new RuntimeException("Exception while processing " + inputPath.toString(), e);
}
}
Path factoryOutputPath = Paths.get(outputDir, API_MESSAGE_FACTORY_JAVA);
@ -137,9 +125,15 @@ public final class MessageGenerator { @@ -137,9 +125,15 @@ public final class MessageGenerator {
messageFactoryGenerator.write(writer);
}
numProcessed++;
for (Path outputPath : Files.newDirectoryStream(Paths.get(outputDir))) {
if (!outputFileNames.contains(outputPath.getFileName().toString())) {
Files.delete(outputPath);
try (DirectoryStream<Path> directoryStream = Files.
newDirectoryStream(Paths.get(outputDir))) {
for (Path outputPath : directoryStream) {
Path fileName = outputPath.getFileName();
if (fileName != null) {
if (!outputFileNames.contains(fileName.toString())) {
Files.delete(outputPath);
}
}
}
}
System.out.printf("MessageGenerator: processed %d Kafka message JSON files(s).%n", numProcessed);

0
buildSrc/src/main/java/org/apache/kafka/message/MessageSpec.java → generator/src/main/java/org/apache/kafka/message/MessageSpec.java

0
buildSrc/src/main/java/org/apache/kafka/message/MessageSpecType.java → generator/src/main/java/org/apache/kafka/message/MessageSpecType.java

0
buildSrc/src/main/java/org/apache/kafka/message/SchemaGenerator.java → generator/src/main/java/org/apache/kafka/message/SchemaGenerator.java

0
buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java → generator/src/main/java/org/apache/kafka/message/StructSpec.java

0
buildSrc/src/main/java/org/apache/kafka/message/Versions.java → generator/src/main/java/org/apache/kafka/message/Versions.java

0
clients/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java → generator/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java

0
clients/src/test/java/org/apache/kafka/message/VersionsTest.java → generator/src/test/java/org/apache/kafka/message/VersionsTest.java

2
settings.gradle

@ -17,4 +17,4 @@ include 'core', 'examples', 'clients', 'tools', 'streams', 'streams:streams-scal @@ -17,4 +17,4 @@ include 'core', 'examples', 'clients', 'tools', 'streams', 'streams:streams-scal
'streams:upgrade-system-tests-0100', 'streams:upgrade-system-tests-0101', 'streams:upgrade-system-tests-0102',
'streams:upgrade-system-tests-0110', 'streams:upgrade-system-tests-10', 'streams:upgrade-system-tests-11', 'streams:upgrade-system-tests-20',
'streams:upgrade-system-tests-21' , 'log4j-appender', 'connect:api', 'connect:transforms', 'connect:runtime', 'connect:json', 'connect:file',
'connect:basic-auth-extension', 'jmh-benchmarks'
'connect:basic-auth-extension', 'jmh-benchmarks', 'generator'

Loading…
Cancel
Save