// Licensed to the Apache Software Foundation (ASF) under one or more // contributor license agreements. See the NOTICE file distributed with // this work for additional information regarding copyright ownership. // The ASF licenses this file to You under the Apache License, Version 2.0 // (the "License"); you may not use this file except in compliance with // the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import org.ajoberstar.grgit.Grgit import java.nio.charset.StandardCharsets buildscript { repositories { mavenCentral() } apply from: "$rootDir/gradle/dependencies.gradle" dependencies { // For Apache Rat plugin to ignore non-Git files classpath "org.ajoberstar.grgit:grgit-core:$versions.grgit" } } plugins { id 'com.github.ben-manes.versions' version '0.53.0' id 'idea' id 'jacoco' id 'java-library' id 'org.owasp.dependencycheck' version '12.1.8' id 'org.nosphere.apache.rat' version "0.8.1" id "io.swagger.core.v3.swagger-gradle-plugin" version "${swaggerVersion}" id "com.github.spotbugs" version '6.4.4' apply false id 'org.scoverage' version '8.1' apply false id 'com.gradleup.shadow' version '8.3.9' apply false id 'com.diffplug.spotless' version "8.0.0" } ext { minClientJavaVersion = 11 minNonClientJavaVersion = 17 modulesNeedingJava11 = [":clients", ":generator", ":streams", ":streams:test-utils", ":streams:examples", ":streams:streams-scala", ":test-common:test-common-util"] buildVersionFileName = "kafka-version.properties" defaultMaxHeapSize = "2g" defaultJvmArgs = ["-Xss4m", "-XX:+UseParallelGC"] // "JEP 403: Strongly Encapsulate JDK Internals" causes some tests to fail when they try // to access internals (often via mocking libraries). We use `--add-opens` as a workaround // for now and we'll fix it properly (where possible) via KAFKA-13275. if (JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_16)) defaultJvmArgs.addAll( "--add-opens=java.base/java.io=ALL-UNNAMED", "--add-opens=java.base/java.lang=ALL-UNNAMED", "--add-opens=java.base/java.nio=ALL-UNNAMED", "--add-opens=java.base/java.nio.file=ALL-UNNAMED", "--add-opens=java.base/java.util=ALL-UNNAMED", "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", "--add-opens=java.base/java.util.regex=ALL-UNNAMED", "--add-opens=java.base/java.util.stream=ALL-UNNAMED", "--add-opens=java.base/java.text=ALL-UNNAMED", "--add-opens=java.base/java.time=ALL-UNNAMED", "--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED" ) maxTestForks = project.hasProperty('maxParallelForks') ? maxParallelForks.toInteger() : Runtime.runtime.availableProcessors() maxScalacThreads = project.hasProperty('maxScalacThreads') ? maxScalacThreads.toInteger() : Math.min(Runtime.runtime.availableProcessors(), 8) userIgnoreFailures = project.hasProperty('ignoreFailures') ? ignoreFailures.toBoolean() : false userMaxTestRetries = project.hasProperty('maxTestRetries') ? maxTestRetries.toInteger() : 0 userMaxTestRetryFailures = project.hasProperty('maxTestRetryFailures') ? maxTestRetryFailures.toInteger() : 0 skipSigning = project.hasProperty('skipSigning') && skipSigning.toBoolean() shouldSign = !skipSigning && !version.endsWith("SNAPSHOT") mavenUrl = project.hasProperty('mavenUrl') ? project.mavenUrl : '' mavenUsername = project.hasProperty('mavenUsername') ? project.mavenUsername : '' mavenPassword = project.hasProperty('mavenPassword') ? project.mavenPassword : '' userShowStandardStreams = project.hasProperty("showStandardStreams") ? showStandardStreams : null userTestLoggingEvents = project.hasProperty("testLoggingEvents") ? Arrays.asList(testLoggingEvents.split(",")) : null userEnableTestCoverage = project.hasProperty("enableTestCoverage") ? enableTestCoverage : false userKeepAliveModeString = project.hasProperty("keepAliveMode") ? keepAliveMode : "daemon" userKeepAliveMode = KeepAliveMode.values().find(m -> m.name().toLowerCase().equals(userKeepAliveModeString)) if (userKeepAliveMode == null) { def keepAliveValues = KeepAliveMode.values().collect(m -> m.name().toLowerCase()) throw new GradleException("Unexpected value for keepAliveMode property. Expected one of $keepAliveValues, but received: $userKeepAliveModeString") } // Used by :test task isGithubActions = System.getenv('GITHUB_ACTIONS') != null // See README.md for details on this option and the reasoning for the default userScalaOptimizerMode = project.hasProperty("scalaOptimizerMode") ? scalaOptimizerMode : "inline-kafka" def scalaOptimizerValues = ["none", "method", "inline-kafka", "inline-scala"] if (!scalaOptimizerValues.contains(userScalaOptimizerMode)) throw new GradleException("Unexpected value for scalaOptimizerMode property. Expected one of $scalaOptimizerValues, but received: $userScalaOptimizerMode") generatedDocsDir = new File("${project.rootDir}/docs/generated") repo = file("$rootDir/.git").isDirectory() ? Grgit.open(currentDir: project.getRootDir()) : null commitId = determineCommitId() configureJavaCompiler = { name, options, projectPath -> // -parameters generates arguments with parameter names in TestInfo#getDisplayName. // ref: https://github.com/junit-team/junit5/blob/4c0dddad1b96d4a20e92a2cd583954643ac56ac0/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ParameterizedTest.java#L161-L164 def releaseVersion = modulesNeedingJava11.any { projectPath == it } ? minClientJavaVersion : minNonClientJavaVersion options.compilerArgs << "-encoding" << "UTF-8" options.release = releaseVersion if (name in ["compileTestJava", "compileTestScala"]) { options.compilerArgs << "-parameters" } else if (name in ["compileJava", "compileScala"]) { options.compilerArgs << "-Xlint:all" options.compilerArgs << "-Xlint:-serial" options.compilerArgs << "-Xlint:-try" options.compilerArgs << "-Werror" if (userKeepAliveMode == KeepAliveMode.SESSION){ // Suppress warning triggered under keepAliveMode=SESSION during joint compilation // Unexpected javac output: warning: [path] bad path element ".../core/build/classes/java/main": no such file or directory options.compilerArgs << "-Xlint:-path" } } } runtimeTestLibs = [ libs.slf4jLog4j2, libs.junitPlatformLanucher, libs.jacksonDatabindYaml, project(":test-common:test-common-util") ] log4jReleaseLibs = [ libs.slf4jLog4j2, libs.log4j1Bridge2Api, libs.jacksonDatabindYaml ] log4j2Libs = [ libs.log4j2Api, libs.log4j2Core ] testLog4j2Libs = [ libs.slf4jApi, libs.slf4jLog4j2, libs.log4j2Api, libs.log4j2Core ] } allprojects { repositories { mavenCentral() } dependencyUpdates { revision="release" resolutionStrategy { componentSelection { rules -> rules.all { ComponentSelection selection -> boolean rejected = ['snap', 'alpha', 'beta', 'rc', 'cr', 'm'].any { qualifier -> selection.candidate.version ==~ /(?i).*[.-]${qualifier}[.\d-]*/ } if (rejected) { selection.reject('Release candidate') } } } } } configurations.all { // zinc is the Scala incremental compiler, it has a configuration for its own dependencies // that are unrelated to the project dependencies, we should not change them if (name != "zinc") { resolutionStrategy { force( // be explicit about the javassist dependency version instead of relying on the transitive version libs.javassist, // ensure we have a single version in the classpath despite transitive dependencies libs.scalaLibrary, libs.scalaReflect, libs.jacksonAnnotations, libs.commonsLang ) } } } task printAllDependencies(type: DependencyReportTask) {} tasks.withType(Javadoc) { options.charSet = 'UTF-8' options.docEncoding = 'UTF-8' options.encoding = 'UTF-8' options.memberLevel = JavadocMemberLevel.PUBLIC // Document only public members/API // Turn off doclint for now, see https://blog.joda.org/2014/02/turning-off-doclint-in-jdk-8-javadoc.html for rationale options.addStringOption('Xdoclint:none', '-quiet') // Javadoc warnings should fail the build in JDK 15+ https://bugs.openjdk.org/browse/JDK-8200363 options.addBooleanOption('Werror', JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_15)) options.links "https://docs.oracle.com/en/java/javase/${JavaVersion.current().majorVersion}/docs/api/" } tasks.withType(Checkstyle) { minHeapSize = "200m" maxHeapSize = "1g" } clean { delete "${projectDir}/src/generated" delete "${projectDir}/src/generated-test" } } def determineCommitId() { def takeFromHash = 16 if (project.hasProperty('commitId')) { commitId.take(takeFromHash) } else if (repo != null) { repo.head().id.take(takeFromHash) } else { "unknown" } } /** * For a given Project, compute a nice dash separated directory name * to store the JUnit XML files in. E.g., Project ":connect:api" -> "connect-api" */ static def projectToJUnitXmlPath(project) { var p = project var projectNames = [] while (p != null) { projectNames.push(p.name) p = p.parent if (p.name == "kafka") { break } } return projectNames.join("/") } apply from: file('wrapper.gradle') if (repo != null) { rat { dependsOn subprojects.collect { it.tasks.matching { it.name == "processMessages" || it.name == "processTestMessages" } } verbose.set(true) reportDir.set(project.file('build/rat')) stylesheet.set(file('gradle/resources/rat-output-to-html.xsl')) // Exclude everything under the directory that git should be ignoring via .gitignore or that isn't checked in. These // restrict us only to files that are checked in or are staged. excludes = new ArrayList(repo.clean(ignore: false, directories: true, dryRun: true)) // And some of the files that we have checked in should also be excluded from this check excludes.addAll([ '**/.git/**', '**/build/**', '.github/pull_request_template.md', 'CONTRIBUTING.md', 'gradlew', 'gradlew.bat', 'gradle/wrapper/gradle-wrapper.properties', 'trogdor/README.md', '**/README.md', '**/id_rsa', '**/id_rsa.pub', 'checkstyle/suppressions.xml', 'streams/quickstart/java/src/test/resources/projects/basic/goal.txt', 'streams/streams-scala/logs/*', 'licenses/*', '**/generated/**', 'clients/src/test/resources/serializedData/*', 'docker/test/fixtures/secrets/*', 'docker/examples/fixtures/secrets/*', 'docker/docker_official_images/.gitkeep' ]) } } else { rat.enabled = false } println("Starting build with version $version (commit id ${commitId == null ? "null" : commitId.take(8)}) using Gradle $versions.gradle, Java ${JavaVersion.current()} and Scala ${versions.scala}") println("Build properties: ignoreFailures=$userIgnoreFailures, maxParallelForks=$maxTestForks, maxScalacThreads=$maxScalacThreads, maxTestRetries=$userMaxTestRetries") subprojects { // enable running :dependencies task recursively on all subprojects // eg: ./gradlew allDeps task allDeps(type: DependencyReportTask) {} // enable running :dependencyInsight task recursively on all subprojects // eg: ./gradlew allDepInsight --configuration runtime --dependency com.fasterxml.jackson.core:jackson-databind task allDepInsight(type: DependencyInsightReportTask) {showingAllVariants = false} doLast {} apply plugin: 'java-library' apply plugin: 'checkstyle' apply plugin: "com.github.spotbugs" // We use the shadow plugin for the jmh-benchmarks module and the `-all` jar can get pretty large, so // don't publish it def shouldPublish = !project.name.equals('jmh-benchmarks') def shouldPublishWithShadow = (['clients'].contains(project.name)) if (shouldPublish) { apply plugin: 'maven-publish' apply plugin: 'signing' // Add aliases for the task names used by the maven plugin for backwards compatibility // The maven plugin was replaced by the maven-publish plugin in Gradle 7.0 tasks.register('install').configure { dependsOn(publishToMavenLocal) } tasks.register('uploadArchives').configure { dependsOn(publish) } } tasks.withType(AbstractArchiveTask).configureEach { reproducibleFileOrder = false preserveFileTimestamps = true useFileSystemPermissions() } tasks.withType(AbstractTestTask).configureEach { failOnNoDiscoveredTests = false } // apply the eclipse plugin only to subprojects that hold code. 'connect' is just a folder. if (!project.name.equals('connect')) { apply plugin: 'eclipse' fineTuneEclipseClasspathFile(eclipse, project) } java { consistentResolution { // resolve the compileClasspath and then "inject" the result of resolution as strict constraints into the runtimeClasspath useCompileClasspathVersions() } } tasks.withType(JavaCompile) { configureJavaCompiler(name, options, project.path) } if (shouldPublish) { publishing { repositories { // To test locally, invoke gradlew with `-PmavenUrl=file:///some/local/path` maven { url = mavenUrl credentials { username = mavenUsername password = mavenPassword } } } publications { mavenJava(MavenPublication) { if (!shouldPublishWithShadow) { from components.java } else { apply plugin: 'com.gradleup.shadow' from components.shadow // Fix for avoiding inclusion of runtime dependencies marked as 'shadow' in MANIFEST Class-Path. // https://github.com/GradleUp/shadow/issues/324 pom.withXml { xml -> def dependenciesNode = xml.asNode().get('dependencies') ?: xml.asNode().appendNode('dependencies') project.configurations.shadowed.allDependencies.each { def dependencyNode = dependenciesNode.appendNode('dependency') dependencyNode.appendNode('groupId', it.group) dependencyNode.appendNode('artifactId', it.name) dependencyNode.appendNode('version', it.version) dependencyNode.appendNode('scope', 'runtime') } } } afterEvaluate { ["srcJar", "javadocJar", "scaladocJar", "testJar", "testSrcJar"].forEach { taskName -> def task = tasks.findByName(taskName) if (task != null) artifact task } artifactId = base.archivesName.get() pom { name = 'Apache Kafka' url = 'https://kafka.apache.org' licenses { license { name = 'The Apache License, Version 2.0' url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' distribution = 'repo' } } } } } } } if (shouldSign) { signing { sign publishing.publications.mavenJava } } } def testLoggingEvents = ["passed", "skipped", "failed"] def testShowStandardStreams = false def testExceptionFormat = 'full' // Gradle built-in logging only supports sending test output to stdout, which generates a lot // of noise, especially for passing tests. We really only want output for failed tests. This // hooks into the output and logs it (so we don't have to buffer it all in memory) and only // saves the output for failing tests. Directory and filenames are such that you can, e.g., // create a Jenkins rule to collect failed test output. def logTestStdout = { def testId = { TestDescriptor descriptor -> "${descriptor.className}.${descriptor.name}".toString() } def logFiles = new HashMap() def logStreams = new HashMap() beforeTest { TestDescriptor td -> def tid = testId(td) // truncate the file name if it's too long def logFile = new File( "${projectDir}/build/reports/testOutput/${tid.substring(0, Math.min(tid.size(),240))}.test.stdout" ) logFile.parentFile.mkdirs() logFiles.put(tid, logFile) logStreams.put(tid, new FileOutputStream(logFile)) } onOutput { TestDescriptor td, TestOutputEvent toe -> def tid = testId(td) // Some output can happen outside the context of a specific test (e.g. at the class level) // and beforeTest/afterTest seems to not be invoked for these cases (and similarly, there's // a TestDescriptor hierarchy that includes the thread executing the test, Gradle tasks, // etc). We see some of these in practice and it seems like something buggy in the Gradle // test runner since we see it *before* any tests and it is frequently not related to any // code in the test (best guess is that it is tail output from last test). We won't have // an output file for these, so simply ignore them. If they become critical for debugging, // they can be seen with showStandardStreams. if (td.name == td.className || td.className == null) { // silently ignore output unrelated to specific test methods return } else if (logStreams.get(tid) == null) { println "WARNING: unexpectedly got output for a test [${tid}]" + " that we didn't previously see in the beforeTest hook." + " Message for debugging: [" + toe.message + "]." return } try { logStreams.get(tid).write(toe.message.getBytes(StandardCharsets.UTF_8)) } catch (Exception e) { println "ERROR: Failed to write output for test ${tid}" e.printStackTrace() } } afterTest { TestDescriptor td, TestResult tr -> def tid = testId(td) try { logStreams.get(tid).close() if (tr.resultType != TestResult.ResultType.FAILURE) { logFiles.get(tid).delete() } else { def file = logFiles.get(tid) println "${tid} failed, log available in ${file}" } } catch (Exception e) { println "ERROR: Failed to close stdout file for ${tid}" e.printStackTrace() } finally { logFiles.remove(tid) logStreams.remove(tid) } } } // Workaround for Mockito Java Agent restrictions in Java 21+ // Starting with Java 21, the JDK restricts libraries from attaching a Java agent // to their own JVM. As a result, Mockito’s inline mock maker (mockito-core) // fails without explicit instrumentation, and the JVM consistently emits warnings. // See also: https://javadoc.io/doc/org.mockito/mockito-core/latest/org.mockito/org/mockito/Mockito.html#mockito-instrumentation afterEvaluate { subproject -> def hasMockitoCore = subproject.configurations.findAll { it.canBeResolved }.any { config -> config.incoming.dependencies.any { dependency -> "$dependency" == libs.mockitoCore } } if (hasMockitoCore) { subproject.configurations { mockitoAgent { transitive = false } } subproject.dependencies { mockitoAgent libs.mockitoCore } } } // The suites are for running sets of tests in IDEs. // Gradle will run each test class, so we exclude the suites to avoid redundantly running the tests twice. def testsToExclude = ['**/*Suite.class'] // This task will copy JUnit XML files out of the sub-project's build directory and into // a top-level build/junit-xml directory. This is necessary to avoid reporting on tests which // were not run, but instead were restored via FROM-CACHE. See KAFKA-17479 for more details. def copyTestXml = tasks.register('copyTestXml') { onlyIf("Environment GITHUB_ACTIONS is set") { isGithubActions } onlyIf("Project '${project.name}:test' has sources") { ! test.state.noSource } onlyIf("Task '${project.name}:test' did work") { test.state.didWork } ext { output = project.findProperty("kafka.test.xml.output.dir") } // Never cache this task outputs.cacheIf { false } outputs.upToDateWhen { false } doLast { def moduleDirPath = projectToJUnitXmlPath(project) def dest = rootProject.layout.buildDirectory.dir("junit-xml/${moduleDirPath}/${output}").get().asFile println "Copy JUnit XML for ${project.name} to $dest" ant.copy(todir: "$dest") { ant.fileset(dir: "${test.reports.junitXml.entryPoint}") { ant.include(name: "**/*.xml") } } } } test { doFirst { def mockitoAgentConfig = configurations.findByName('mockitoAgent') if (mockitoAgentConfig) { jvmArgs("-javaagent:${mockitoAgentConfig.asPath}") } } maxParallelForks = maxTestForks ignoreFailures = userIgnoreFailures maxHeapSize = "3g" jvmArgs = defaultJvmArgs // KAFKA-17433 Used by deflake.yml github action to repeat individual tests systemProperty("kafka.cluster.test.repeat", project.findProperty("kafka.cluster.test.repeat")) systemProperty("kafka.test.catalog.file", project.findProperty("kafka.test.catalog.file")) systemProperty("kafka.test.run.new", project.findProperty("kafka.test.run.new")) systemProperty("kafka.test.run.flaky", project.findProperty("kafka.test.run.flaky")) systemProperty("kafka.test.verbose", project.findProperty("kafka.test.verbose")) testLogging { events = userTestLoggingEvents ?: testLoggingEvents showStandardStreams = userShowStandardStreams ?: testShowStandardStreams exceptionFormat = testExceptionFormat displayGranularity = 0 } logTestStdout.rehydrate(delegate, owner, this)() exclude testsToExclude useJUnitPlatform { includeEngines 'junit-jupiter' } develocity { testRetry { maxRetries = userMaxTestRetries maxFailures = userMaxTestRetryFailures } } finalizedBy("copyTestXml") } task integrationTest(type: Test, dependsOn: compileJava) { testClassesDirs = sourceSets.test.output.classesDirs classpath = sourceSets.test.runtimeClasspath maxParallelForks = maxTestForks ignoreFailures = userIgnoreFailures // Increase heap size for integration tests maxHeapSize = "2560m" jvmArgs = defaultJvmArgs testLogging { events = userTestLoggingEvents ?: testLoggingEvents showStandardStreams = userShowStandardStreams ?: testShowStandardStreams exceptionFormat = testExceptionFormat displayGranularity = 0 } logTestStdout.rehydrate(delegate, owner, this)() exclude testsToExclude useJUnitPlatform { includeTags "integration" includeEngines 'junit-jupiter' } develocity { testRetry { maxRetries = userMaxTestRetries maxFailures = userMaxTestRetryFailures } } } task unitTest(type: Test, dependsOn: compileJava) { testClassesDirs = sourceSets.test.output.classesDirs classpath = sourceSets.test.runtimeClasspath maxParallelForks = maxTestForks ignoreFailures = userIgnoreFailures maxHeapSize = defaultMaxHeapSize jvmArgs = defaultJvmArgs testLogging { events = userTestLoggingEvents ?: testLoggingEvents showStandardStreams = userShowStandardStreams ?: testShowStandardStreams exceptionFormat = testExceptionFormat displayGranularity = 0 } logTestStdout.rehydrate(delegate, owner, this)() exclude testsToExclude useJUnitPlatform { excludeTags "integration" includeEngines 'junit-jupiter' } develocity { testRetry { maxRetries = userMaxTestRetries maxFailures = userMaxTestRetryFailures } } } // remove test output from all test types tasks.withType(Test).all { t -> cleanTest { delete t.reports.junitXml.outputLocation delete t.reports.html.outputLocation } } jar { from "$rootDir/LICENSE" from "$rootDir/NOTICE" } task srcJar(type: Jar) { archiveClassifier = 'sources' from "$rootDir/LICENSE" from "$rootDir/NOTICE" from sourceSets.main.allSource } task javadocJar(type: Jar, dependsOn: javadoc) { archiveClassifier = 'javadoc' from "$rootDir/LICENSE" from "$rootDir/NOTICE" from javadoc.destinationDir } task docsJar(dependsOn: javadocJar) check.dependsOn('javadoc') task systemTestLibs(dependsOn: jar) if (!sourceSets.test.allSource.isEmpty()) { task testJar(type: Jar) { archiveClassifier = 'test' from "$rootDir/LICENSE" from "$rootDir/NOTICE" from sourceSets.test.output // The junit-platform.properties file is used for configuring and customizing the behavior of the JUnit platform. // It should only apply to Kafka's own JUnit tests, and should not exist in the test JAR. // If we include it in the test JAR, it could lead to conflicts with user configurations. exclude 'junit-platform.properties' } task testSrcJar(type: Jar, dependsOn: testJar) { archiveClassifier = 'test-sources' from "$rootDir/LICENSE" from "$rootDir/NOTICE" from sourceSets.test.allSource } } plugins.withType(ScalaPlugin) { scala { zincVersion = versions.zinc } task scaladocJar(type:Jar, dependsOn: scaladoc) { archiveClassifier = 'scaladoc' from "$rootDir/LICENSE" from "$rootDir/NOTICE" from scaladoc.destinationDir } //documentation task should also trigger building scala doc jar docsJar.dependsOn scaladocJar } tasks.withType(ScalaCompile) { def releaseVersion = modulesNeedingJava11.any { project.path == it } ? minClientJavaVersion : minNonClientJavaVersion scalaCompileOptions.keepAliveMode = userKeepAliveMode scalaCompileOptions.additionalParameters = [ "-deprecation:false", "-unchecked", "-encoding", "utf8", "-Xlog-reflective-calls", "-feature", "-language:postfixOps", "-language:implicitConversions", "-language:existentials", "-Ybackend-parallelism", maxScalacThreads.toString(), "-Xlint:constant", "-Xlint:delayedinit-select", "-Xlint:doc-detached", "-Xlint:missing-interpolator", "-Xlint:nullary-unit", "-Xlint:option-implicit", "-Xlint:package-object-classes", "-Xlint:poly-implicit-overload", "-Xlint:private-shadow", "-Xlint:stars-align", "-Xlint:type-parameter-shadow", "-Xlint:unused" ] // See README.md for details on this option and the meaning of each value if (userScalaOptimizerMode.equals("method")) scalaCompileOptions.additionalParameters += ["-opt:l:method"] else if (userScalaOptimizerMode.startsWith("inline-")) { List inlineFrom = ["-opt-inline-from:org.apache.kafka.**"] if (project.name.equals('core')) inlineFrom.add("-opt-inline-from:kafka.**") if (userScalaOptimizerMode.equals("inline-scala")) inlineFrom.add("-opt-inline-from:scala.**") scalaCompileOptions.additionalParameters += ["-opt:l:inline"] scalaCompileOptions.additionalParameters += inlineFrom } scalaCompileOptions.additionalParameters += ["-opt-warnings", "-Xlint:strict-unsealed-patmat"] // Scala 2.13.2 introduces compiler warnings suppression, which is a pre-requisite for -Xfatal-warnings scalaCompileOptions.additionalParameters += ["-Xfatal-warnings"] scalaCompileOptions.additionalParameters += ["--release", String.valueOf(releaseVersion)] // Gradle does not support the `release` configuration when performing joint Java-Scala compilation. // For more details, refer to https://github.com/gradle/gradle/issues/13762. // As a result, we need to explicitly configure the Scala compiler with this setting. options.compilerArgs += ["--release", String.valueOf(releaseVersion)] configureJavaCompiler(name, options, project.path) configure(scalaCompileOptions.forkOptions) { memoryMaximumSize = defaultMaxHeapSize jvmArgs = defaultJvmArgs } } checkstyle { configDirectory = rootProject.layout.projectDirectory.dir("checkstyle") configProperties = checkstyleConfigProperties("import-control.xml") toolVersion = versions.checkstyle } configure(checkstyleMain) { group = 'Verification' description = 'Run checkstyle on all main Java sources' } configure(checkstyleTest) { group = 'Verification' description = 'Run checkstyle on all test Java sources' } test.dependsOn('checkstyleMain', 'checkstyleTest') spotbugs { toolVersion = versions.spotbugs excludeFilter = file("$rootDir/gradle/spotbugs-exclude.xml") ignoreFailures = false } test.dependsOn('spotbugsMain') tasks.withType(com.github.spotbugs.snom.SpotBugsTask).configureEach { reports.configure { // Continue supporting `xmlFindBugsReport` for compatibility xml.enabled(project.hasProperty('xmlSpotBugsReport') || project.hasProperty('xmlFindBugsReport')) html.enabled(!project.hasProperty('xmlSpotBugsReport') && !project.hasProperty('xmlFindBugsReport')) } maxHeapSize = defaultMaxHeapSize jvmArgs = defaultJvmArgs } // Ignore core since its a scala project if (it.path != ':core') { if (userEnableTestCoverage) { apply plugin: "jacoco" jacoco { toolVersion = versions.jacoco } jacocoTestReport { dependsOn tasks.test sourceSets sourceSets.main reports { html.required = true xml.required = true csv.required = false } } } } if (userEnableTestCoverage) { def coverageGen = it.path == ':core' ? 'reportTestScoverage' : 'jacocoTestReport' tasks.register('reportCoverage').configure { dependsOn(coverageGen) } } dependencyCheck { suppressionFile = "$rootDir/gradle/resources/dependencycheck-suppressions.xml" skipProjects = [ ":jmh-benchmarks", ":trogdor" ] skipConfigurations = [ "zinc" ] } apply plugin: 'com.diffplug.spotless' spotless { java { targetExclude('**/generated/**/*.java','**/generated-test/**/*.java') importOrder('kafka', 'org.apache.kafka', 'com', 'net', 'org', 'java', 'javax', '', '\\#') removeUnusedImports() } } } gradle.taskGraph.whenReady { taskGraph -> taskGraph.getAllTasks().findAll { it.name.contains('spotbugsScoverage') || it.name.contains('spotbugsTest') }.each { task -> task.enabled = false } } def fineTuneEclipseClasspathFile(eclipse, project) { eclipse.classpath.file { beforeMerged { cp -> cp.entries.clear() // for the core project add the directories defined under test/scala as separate source directories if (project.name.equals('core')) { cp.entries.add(new org.gradle.plugins.ide.eclipse.model.SourceFolder("src/test/scala/integration", null)) cp.entries.add(new org.gradle.plugins.ide.eclipse.model.SourceFolder("src/test/scala/other", null)) cp.entries.add(new org.gradle.plugins.ide.eclipse.model.SourceFolder("src/test/scala/unit", null)) } } whenMerged { cp -> // for the core project exclude the separate sub-directories defined under test/scala. These are added as source dirs above if (project.name.equals('core')) { cp.entries.findAll { it.kind == "src" && it.path.equals("src/test/scala") }*.excludes = ["integration/", "other/", "unit/"] } /* * Set all eclipse build output to go to 'build_eclipse' directory. This is to ensure that gradle and eclipse use different * build output directories, and also avoid using the eclipse default of 'bin' which clashes with some of our script directories. * https://discuss.gradle.org/t/eclipse-generated-files-should-be-put-in-the-same-place-as-the-gradle-generated-files/6986/2 */ cp.entries.findAll { it.kind == "output" }*.path = "build_eclipse" /* * Some projects have explicitly added test output dependencies. These are required for the gradle build but not required * in Eclipse since the dependent projects are added as dependencies. So clean up these from the generated classpath. */ cp.entries.removeAll { it.kind == "lib" && it.path.matches(".*/build/(classes|resources)/test") } } } } def checkstyleConfigProperties(configFileName) { [importControlFile: "$configFileName"] } if (userEnableTestCoverage) { tasks.register('reportCoverage').configure { dependsOn(subprojects.reportCoverage) } } def connectPkgs = [ 'connect:api', 'connect:basic-auth-extension', 'connect:file', 'connect:json', 'connect:runtime', 'connect:test-plugins', 'connect:transforms', 'connect:mirror', 'connect:mirror-client' ] tasks.create(name: "jarConnect", dependsOn: connectPkgs.collect { it + ":jar" }) {} tasks.create(name: "testConnect", dependsOn: connectPkgs.collect { it + ":test" }) {} project(':server') { base { archivesName = "kafka-server" } dependencies { compileOnly libs.bndlib compileOnly libs.spotbugs implementation project(':clients') implementation project(':metadata') implementation project(':server-common') implementation project(':storage') implementation project(':group-coordinator') implementation project(':transaction-coordinator') implementation project(':raft') implementation project(':share-coordinator') implementation project(':storage:storage-api') implementation libs.jacksonDatabind implementation libs.metrics implementation libs.slf4jApi implementation log4j2Libs testImplementation project(':clients').sourceSets.test.output testImplementation libs.mockitoCore testImplementation libs.junitJupiter testImplementation testLog4j2Libs testImplementation project(':test-common:test-common-internal-api') testImplementation project(':test-common:test-common-runtime') testImplementation project(':storage:storage-api').sourceSets.test.output testImplementation project(':server-common').sourceSets.test.output testRuntimeOnly runtimeTestLibs } task createVersionFile() { def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName") inputs.property "commitId", commitId inputs.property "version", version outputs.file receiptFile outputs.cacheIf { true } doLast { def data = [ commitId: commitId, version: version, ] receiptFile.parentFile.mkdirs() def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n") receiptFile.setText(content, "ISO-8859-1") } } jar { dependsOn createVersionFile from("${layout.buildDirectory.get().asFile.path}") { include "kafka/$buildVersionFileName" } } clean.doFirst { delete "${layout.buildDirectory.get().asFile.path}/kafka/" } checkstyle { configProperties = checkstyleConfigProperties("import-control-server.xml") } javadoc { enabled = false } } project(':core') { apply plugin: 'scala' // scaladoc generation is configured at the sub-module level with an artifacts // block (cf. see streams-scala). If scaladoc generation is invoked explicitly // for the `core` module, this ensures the generated jar doesn't include scaladoc // files since the `core` module doesn't include public APIs. scaladoc { enabled = false } if (userEnableTestCoverage) apply plugin: "org.scoverage" base { archivesName = "kafka_${versions.baseScala}" } configurations { // manually excludes some unnecessary dependencies implementation.exclude module: 'javax' implementation.exclude module: 'jline' implementation.exclude module: 'jms' implementation.exclude module: 'jmxri' implementation.exclude module: 'jmxtools' implementation.exclude module: 'mail' // To prevent a UniqueResourceException due the same resource existing in both // org.apache.directory.api/api-all and org.apache.directory.api/api-ldap-schema-data testImplementation.exclude module: 'api-ldap-schema-data' releaseOnly } dependencies { releaseOnly log4jReleaseLibs // `core` is often used in users' tests, define the following dependencies as `api` for backwards compatibility // even though the `core` module doesn't expose any public API api project(':clients') api libs.scalaLibrary compileOnly log4j2Libs implementation project(':server-common') implementation project(':group-coordinator:group-coordinator-api') implementation project(':group-coordinator') implementation project(':transaction-coordinator') implementation project(':metadata') implementation project(':storage:storage-api') // tools-api is automatically included in releaseTarGz via core's runtimeClasspath. // If removed from here, remember to explicitly add it back in the releaseTarGz task. implementation project(':tools:tools-api') implementation project(':raft') implementation project(':storage') implementation project(':server') implementation project(':coordinator-common') implementation project(':share-coordinator') implementation libs.argparse4j implementation libs.commonsValidator implementation libs.jacksonDatabind implementation libs.jacksonDataformatCsv implementation libs.jacksonJDK8Datatypes implementation libs.jacksonDatabindYaml implementation libs.joptSimple implementation libs.jose4j implementation libs.metrics // only needed transitively, but set it explicitly to ensure it has the same version as scala-library implementation libs.scalaReflect implementation libs.scalaLogging implementation libs.slf4jApi implementation libs.re2j testImplementation project(':clients').sourceSets.test.output testImplementation project(':group-coordinator').sourceSets.test.output testImplementation project(':share-coordinator').sourceSets.test.output testImplementation project(':metadata').sourceSets.test.output testImplementation project(':raft').sourceSets.test.output testImplementation project(':server-common').sourceSets.test.output testImplementation project(':storage:storage-api').sourceSets.test.output testImplementation project(':server').sourceSets.test.output testImplementation project(':test-common:test-common-runtime') testImplementation project(':test-common:test-common-internal-api') testImplementation project(':test-common:test-common-util') testImplementation libs.bcpkix testImplementation libs.mockitoCore testImplementation libs.jqwik testImplementation(libs.apacheda) { exclude group: 'xml-apis', module: 'xml-apis' // `mina-core` is a transitive dependency for `apacheds` and `apacheda`. // It is safer to use from `apacheds` since that is the implementation. exclude module: 'mina-core' } testImplementation libs.apachedsCoreApi testImplementation libs.apachedsInterceptorKerberos testImplementation libs.apachedsProtocolShared testImplementation libs.apachedsProtocolKerberos testImplementation libs.apachedsProtocolLdap testImplementation libs.apachedsLdifPartition testImplementation libs.apachedsMavibotPartition testImplementation libs.apachedsJdbmPartition testImplementation libs.junitJupiter testImplementation libs.caffeine testImplementation testLog4j2Libs testImplementation libs.mockOAuth2Server testRuntimeOnly runtimeTestLibs } if (userEnableTestCoverage) { scoverage { scoverageVersion = versions.scoverage scoverageScalaVersion = versions.scala reportDir = file("${layout.buildDirectory.get().asFile.path}/scoverage") highlighting = false minimumRate = 0.0 } } tasks.create(name: "copyDependantLibs", type: Copy) { from (configurations.runtimeClasspath) { exclude('kafka-clients*') } into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}" duplicatesStrategy 'exclude' } task genProtocolErrorDocs(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.common.protocol.Errors' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "protocol_errors.html").newOutputStream() } task genProtocolTypesDocs(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.common.protocol.types.Type' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "protocol_types.html").newOutputStream() } task genProtocolApiKeyDocs(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.common.protocol.ApiKeys' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "protocol_api_keys.html").newOutputStream() } task genProtocolMessageDocs(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.common.protocol.Protocol' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "protocol_messages.html").newOutputStream() } task genAdminClientConfigDocs(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.clients.admin.AdminClientConfig' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "admin_client_config.html").newOutputStream() } task genProducerConfigDocs(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.clients.producer.ProducerConfig' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "producer_config.html").newOutputStream() } task genConsumerConfigDocs(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.clients.consumer.ConsumerConfig' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "consumer_config.html").newOutputStream() } task genKafkaConfigDocs(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'kafka.server.KafkaConfig' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "kafka_config.html").newOutputStream() } task genTopicConfigDocs(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.storage.internals.log.LogConfig' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "topic_config.html").newOutputStream() } task genGroupConfigDocs(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.coordinator.group.GroupConfig' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "group_config.html").newOutputStream() } task genConsumerMetricsDocs(type: JavaExec) { classpath = sourceSets.test.runtimeClasspath mainClass = 'org.apache.kafka.clients.consumer.internals.ConsumerMetrics' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "consumer_metrics.html").newOutputStream() } task genProducerMetricsDocs(type: JavaExec) { classpath = sourceSets.test.runtimeClasspath mainClass = 'org.apache.kafka.clients.producer.internals.ProducerMetrics' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "producer_metrics.html").newOutputStream() } task siteDocsTar(dependsOn: ['genProtocolErrorDocs', 'genProtocolTypesDocs', 'genProtocolApiKeyDocs', 'genProtocolMessageDocs', 'genAdminClientConfigDocs', 'genProducerConfigDocs', 'genConsumerConfigDocs', 'genKafkaConfigDocs', 'genTopicConfigDocs', 'genGroupConfigDocs', ':connect:runtime:genConnectConfigDocs', ':connect:runtime:genConnectTransformationDocs', ':connect:runtime:genConnectPredicateDocs', ':connect:runtime:genSinkConnectorConfigDocs', ':connect:runtime:genSourceConnectorConfigDocs', ':streams:genStreamsConfigDocs', 'genConsumerMetricsDocs', 'genProducerMetricsDocs', ':connect:runtime:genConnectMetricsDocs', ':connect:runtime:genConnectOpenAPIDocs', ':connect:mirror:genMirrorSourceConfigDocs', ':connect:mirror:genMirrorCheckpointConfigDocs', ':connect:mirror:genMirrorHeartbeatConfigDocs', ':connect:mirror:genMirrorConnectorConfigDocs', ':storage:genRemoteLogManagerConfigDoc', ':storage:genRemoteLogMetadataManagerConfigDoc'], type: Tar) { archiveClassifier = 'site-docs' compression = Compression.GZIP from project.file("$rootDir/docs") into 'site-docs' duplicatesStrategy 'exclude' } tasks.create(name: "releaseTarGz", dependsOn: configurations.archives.artifacts, type: Tar) { into "kafka_${versions.baseScala}-${archiveVersion.get()}" compression = Compression.GZIP from(project.file("$rootDir/bin")) { into "bin/" } from(project.file("$rootDir/config")) { into "config/" } from(project.file("$rootDir/licenses")) { into "licenses/" } from "$rootDir/LICENSE-binary" rename {String filename -> filename.replace("-binary", "")} from "$rootDir/NOTICE-binary" rename {String filename -> filename.replace("-binary", "")} from(configurations.runtimeClasspath) { into("libs/") } from(configurations.archives.artifacts.files) { into("libs/") } from(configurations.releaseOnly) { into("libs/") } from(project.siteDocsTar) { into("site-docs/") } from(project(':tools').jar) { into("libs/") } from(project(':tools').configurations.runtimeClasspath) { into("libs/") } from(project(':trogdor').jar) { into("libs/") } from(project(':trogdor').configurations.runtimeClasspath) { into("libs/") } from(project(':shell').jar) { into("libs/") } from(project(':shell').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:api').jar) { into("libs/") } from(project(':connect:api').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:runtime').jar) { into("libs/") } from(project(':connect:runtime').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:transforms').jar) { into("libs/") } from(project(':connect:transforms').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:json').jar) { into("libs/") } from(project(':connect:json').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:file').jar) { into("libs/") } from(project(':connect:file').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:basic-auth-extension').jar) { into("libs/") } from(project(':connect:basic-auth-extension').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:mirror').jar) { into("libs/") } from(project(':connect:mirror').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:mirror-client').jar) { into("libs/") } from(project(':connect:mirror-client').configurations.runtimeClasspath) { into("libs/") } from(project(':streams').jar) { into("libs/") } from(project(':streams').configurations.runtimeClasspath) { into("libs/") } from(project(':streams:streams-scala').jar) { into("libs/") } from(project(':streams:streams-scala').configurations.runtimeClasspath) { into("libs/") } from(project(':streams:test-utils').jar) { into("libs/") } from(project(':streams:test-utils').configurations.runtimeClasspath) { into("libs/") } from(project(':streams:examples').jar) { into("libs/") } from(project(':streams:examples').configurations.runtimeClasspath) { into("libs/") } duplicatesStrategy 'exclude' } jar { dependsOn copyDependantLibs } jar.manifest { attributes( 'Version': "${version}" ) } test { useJUnitPlatform { includeEngines 'jqwik', 'junit-jupiter' } } tasks.create(name: "copyDependantTestLibs", type: Copy) { from (configurations.testRuntimeClasspath) { include('*.jar') } from (configurations.releaseOnly) into "${layout.buildDirectory.get().asFile.path}/dependant-testlibs" //By default gradle does not handle test dependencies between the sub-projects //This line is to include clients project test jar to dependant-testlibs from (project(':clients').testJar ) { "${layout.buildDirectory.get().asFile.path}/dependant-testlibs" } duplicatesStrategy 'exclude' } systemTestLibs.dependsOn('jar', 'testJar', 'copyDependantTestLibs') checkstyle { configProperties = checkstyleConfigProperties("import-control-core.xml") } sourceSets { // Set java/scala source folders in the `scala` block to enable joint compilation main { java { srcDirs = [] } scala { srcDirs = ["src/main/java", "src/main/scala"] } } test { java { srcDirs = [] } scala { srcDirs = ["src/test/java", "src/test/scala"] } } } } project(':metadata') { base { archivesName = "kafka-metadata" } configurations { generator } dependencies { implementation project(':server-common') implementation project(':clients') implementation project(':raft') implementation libs.jacksonDatabind implementation libs.jacksonJDK8Datatypes implementation libs.metrics implementation libs.slf4jApi testImplementation testLog4j2Libs testImplementation libs.junitJupiter testImplementation libs.jqwik testImplementation libs.mockitoCore testImplementation project(':clients').sourceSets.test.output testImplementation project(':raft').sourceSets.test.output testImplementation project(':server-common').sourceSets.test.output testImplementation project(':test-common:test-common-util') testRuntimeOnly runtimeTestLibs generator project(':generator') } task processMessages(type:JavaExec) { mainClass = "org.apache.kafka.message.MessageGenerator" classpath = configurations.generator args = [ "-p", "org.apache.kafka.common.metadata", "-o", "${projectDir}/build/generated/main/java/org/apache/kafka/common/metadata", "-i", "src/main/resources/common/metadata", "-m", "MessageDataGenerator", "JsonConverterGenerator", "-t", "MetadataRecordTypeGenerator", "MetadataJsonConvertersGenerator" ] inputs.dir("src/main/resources/common/metadata") .withPropertyName("messages") .withPathSensitivity(PathSensitivity.RELATIVE) outputs.cacheIf { true } outputs.dir("${projectDir}/build/generated/main/java/org/apache/kafka/common/metadata") } compileJava.dependsOn 'processMessages' srcJar.dependsOn 'processMessages' sourceSets { main { java { srcDirs = ["src/main/java", "${projectDir}/build/generated/main/java"] } } test { java { srcDirs = ["src/test/java"] } } } javadoc { enabled = false } checkstyle { configProperties = checkstyleConfigProperties("import-control-metadata.xml") } } project(':group-coordinator:group-coordinator-api') { base { archivesName = "kafka-group-coordinator-api" } dependencies { implementation project(':clients') } task createVersionFile() { def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName") inputs.property "commitId", commitId inputs.property "version", version outputs.file receiptFile outputs.cacheIf { true } doLast { def data = [ commitId: commitId, version: version, ] receiptFile.parentFile.mkdirs() def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n") receiptFile.setText(content, "ISO-8859-1") } } jar { dependsOn createVersionFile from("${layout.buildDirectory.get().asFile.path}") { include "kafka/$buildVersionFileName" } } clean.doFirst { delete "${layout.buildDirectory.get().asFile.path}/kafka/" } javadoc { include "**/org/apache/kafka/coordinator/group/api/**" } checkstyle { configProperties = checkstyleConfigProperties("import-control-group-coordinator.xml") } } project(':group-coordinator') { base { archivesName = "kafka-group-coordinator" } configurations { generator } dependencies { implementation project(':server-common') implementation project(':clients') implementation project(':metadata') implementation project(':group-coordinator:group-coordinator-api') implementation project(':storage') implementation project(':coordinator-common') implementation libs.jacksonDatabind implementation libs.jacksonJDK8Datatypes implementation libs.metrics implementation libs.hdrHistogram implementation libs.re2j implementation libs.slf4jApi implementation libs.hash4j testImplementation project(':clients').sourceSets.test.output testImplementation project(':server-common').sourceSets.test.output testImplementation project(':coordinator-common').sourceSets.test.output testImplementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testImplementation libs.mockitoCore testImplementation testLog4j2Libs testRuntimeOnly runtimeTestLibs generator project(':generator') } sourceSets { main { java { srcDirs = ["src/main/java", "${projectDir}/build/generated/main/java"] } } test { java { srcDirs = ["src/test/java"] } } } javadoc { enabled = false } checkstyle { configProperties = checkstyleConfigProperties("import-control-group-coordinator.xml") } task processMessages(type:JavaExec) { mainClass = "org.apache.kafka.message.MessageGenerator" classpath = configurations.generator args = [ "-p", "org.apache.kafka.coordinator.group.generated", "-o", "${projectDir}/build/generated/main/java/org/apache/kafka/coordinator/group/generated", "-i", "src/main/resources/common/message", "-m", "MessageDataGenerator", "JsonConverterGenerator", "-t", "CoordinatorRecordTypeGenerator", "CoordinatorRecordJsonConvertersGenerator" ] inputs.dir("src/main/resources/common/message") .withPropertyName("messages") .withPathSensitivity(PathSensitivity.RELATIVE) outputs.cacheIf { true } outputs.dir("${projectDir}/build/generated/main/java/org/apache/kafka/coordinator/group/generated") } compileJava.dependsOn 'processMessages' srcJar.dependsOn 'processMessages' } project(':test-common:test-common-internal-api') { // Interfaces, config classes, and other test APIs. Java 17 is the minimum Java version. base { archivesName = "kafka-test-common-internal-api" } dependencies { implementation project(':server-common') // Only project dependency allowed implementation libs.junitJupiterApi testImplementation libs.junitJupiter testImplementation libs.mockitoCore testImplementation testLog4j2Libs testRuntimeOnly runtimeTestLibs } checkstyle { configProperties = checkstyleConfigProperties("import-control-test-common-internal-api.xml") } javadoc { enabled = false } } project(':test-common:test-common-util') { // Runtime-only JUnit extensions for entire project. Java 11 is the minimum Java version required. base { archivesName = "kafka-test-common-util" } dependencies { implementation libs.junitPlatformLanucher implementation libs.junitJupiterApi implementation libs.junitJupiter implementation libs.slf4jApi testImplementation testLog4j2Libs } checkstyle { configProperties = checkstyleConfigProperties("import-control-test-common-util.xml") } javadoc { enabled = false } } project(':test-common:test-common-runtime') { // Runtime-only JUnit extensions for integration tests. Java 17 is the minimum Java version. base { archivesName = "kafka-test-common-runtime" } dependencies { api project(':core') api project(':clients') implementation project(':server') implementation project(':server-common') implementation project(':group-coordinator') implementation project(':test-common:test-common-internal-api') implementation project(':metadata') implementation project(':raft') implementation project(':storage') implementation libs.junitPlatformLanucher implementation libs.junitJupiter implementation libs.jacksonDatabindYaml implementation libs.slf4jApi testImplementation libs.junitJupiter testImplementation libs.mockitoCore testImplementation testLog4j2Libs testRuntimeOnly runtimeTestLibs } checkstyle { configProperties = checkstyleConfigProperties("import-control-test-common-runtime.xml") } javadoc { enabled = false } } project(':transaction-coordinator') { base { archivesName = "kafka-transaction-coordinator" } configurations { generator } dependencies { implementation libs.jacksonDatabind implementation project(':clients') implementation project(':server-common') implementation project(':coordinator-common') implementation libs.slf4jApi testImplementation testLog4j2Libs testImplementation libs.junitJupiter testImplementation libs.mockitoCore testImplementation project(':clients').sourceSets.test.output testImplementation project(':test-common:test-common-runtime') testImplementation project(':test-common:test-common-internal-api') testRuntimeOnly runtimeTestLibs generator project(':generator') } sourceSets { main { java { srcDirs = ["src/main/java", "${projectDir}/build/generated/main/java"] } } test { java { srcDirs = ["src/test/java"] } } } checkstyle { configProperties = checkstyleConfigProperties("import-control-transaction-coordinator.xml") } task processMessages(type:JavaExec) { mainClass = "org.apache.kafka.message.MessageGenerator" classpath = configurations.generator args = [ "-p", "org.apache.kafka.coordinator.transaction.generated", "-o", "${projectDir}/build/generated/main/java/org/apache/kafka/coordinator/transaction/generated", "-i", "src/main/resources/common/message", "-m", "MessageDataGenerator", "JsonConverterGenerator", "-t", "CoordinatorRecordTypeGenerator", "CoordinatorRecordJsonConvertersGenerator" ] inputs.dir("src/main/resources/common/message") .withPropertyName("messages") .withPathSensitivity(PathSensitivity.RELATIVE) outputs.cacheIf { true } outputs.dir("${projectDir}/build/generated/main/java/org/apache/kafka/coordinator/transaction/generated") } compileJava.dependsOn 'processMessages' srcJar.dependsOn 'processMessages' javadoc { enabled = false } } project(':coordinator-common') { base { archivesName = "kafka-coordinator-common" } dependencies { implementation project(':clients') implementation project(':server-common') implementation project(':metadata') implementation project(':storage') implementation libs.slf4jApi implementation libs.metrics implementation libs.hdrHistogram testImplementation project(':clients').sourceSets.test.output testImplementation project(':server-common').sourceSets.test.output testImplementation libs.junitJupiter testImplementation libs.mockitoCore testImplementation testLog4j2Libs testRuntimeOnly runtimeTestLibs } checkstyle { configProperties = checkstyleConfigProperties("import-control-coordinator-common.xml") } javadoc { enabled = false } } project(':share-coordinator') { base { archivesName = "kafka-share-coordinator" } configurations { generator } dependencies { implementation libs.jacksonDatabind implementation project(':clients') implementation project(':coordinator-common') implementation project(':metadata') implementation project(':server-common') implementation libs.metrics implementation libs.slf4jApi testImplementation project(':clients').sourceSets.test.output testImplementation project(':server-common').sourceSets.test.output testImplementation project(':coordinator-common').sourceSets.test.output testImplementation libs.junitJupiter testImplementation libs.mockitoCore testImplementation testLog4j2Libs testRuntimeOnly runtimeTestLibs generator project(':generator') } sourceSets { main { java { srcDirs = ["src/main/java", "${projectDir}/build/generated/main/java"] } } test { java { srcDirs = ["src/test/java"] } } } checkstyle { configProperties = checkstyleConfigProperties("import-control-share-coordinator.xml") } task processMessages(type:JavaExec) { mainClass = "org.apache.kafka.message.MessageGenerator" classpath = configurations.generator args = [ "-p", "org.apache.kafka.coordinator.share.generated", "-o", "${projectDir}/build/generated/main/java/org/apache/kafka/coordinator/share/generated", "-i", "src/main/resources/common/message", "-m", "MessageDataGenerator", "JsonConverterGenerator", "-t", "CoordinatorRecordTypeGenerator", "CoordinatorRecordJsonConvertersGenerator" ] inputs.dir("src/main/resources/common/message") .withPropertyName("messages") .withPathSensitivity(PathSensitivity.RELATIVE) outputs.cacheIf { true } outputs.dir("${projectDir}/build/generated/main/java/org/apache/kafka/coordinator/share/generated") } compileJava.dependsOn 'processMessages' srcJar.dependsOn 'processMessages' javadoc { enabled = false } } project(':examples') { base { archivesName = "kafka-examples" } dependencies { implementation project(':clients') } javadoc { enabled = false } checkstyle { configProperties = checkstyleConfigProperties("import-control-examples.xml") } } project(':generator') { dependencies { implementation libs.argparse4j implementation libs.jacksonDatabind implementation libs.jacksonJDK8Datatypes implementation libs.jacksonJakartarsJsonProvider implementation 'org.eclipse.jgit:org.eclipse.jgit:7.2.0.202503040940-r' // SSH support for JGit based on Apache MINA sshd implementation 'org.eclipse.jgit:org.eclipse.jgit.ssh.apache:7.2.0.202503040940-r' // GPG support for JGit based on BouncyCastle (commit signing) implementation 'org.eclipse.jgit:org.eclipse.jgit.gpg.bc:7.2.0.202503040940-r' testImplementation libs.junitJupiter testRuntimeOnly runtimeTestLibs } javadoc { enabled = false } } project(':clients') { base { archivesName = "kafka-clients" } configurations { generator shadowed } dependencies { implementation libs.zstd implementation libs.lz4 implementation libs.snappy implementation libs.opentelemetryProto implementation libs.protobuf implementation libs.slf4jApi // libraries which should be added as runtime dependencies in generated pom.xml should be defined here: shadowed libs.zstd shadowed libs.lz4 shadowed libs.snappy shadowed libs.slf4jApi compileOnly libs.jacksonDatabind // for SASL/OAUTHBEARER bearer token parsing compileOnly libs.jacksonJDK8Datatypes compileOnly libs.jose4j // for SASL/OAUTHBEARER JWT validation; only used by broker testImplementation project(':test-common:test-common-util') testImplementation libs.bcpkix testImplementation libs.jacksonJakartarsJsonProvider testImplementation libs.jose4j testImplementation libs.junitJupiter testImplementation libs.jqwik testImplementation libs.spotbugs testImplementation libs.mockitoCore testImplementation libs.mockitoJunitJupiter // supports MockitoExtension testImplementation testLog4j2Libs testCompileOnly libs.bndlib testRuntimeOnly libs.jacksonDatabind testRuntimeOnly libs.jacksonJDK8Datatypes testRuntimeOnly runtimeTestLibs generator project(':generator') } tasks.withType(GenerateModuleMetadata) { enabled = false } task createVersionFile() { def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName") inputs.property "commitId", commitId inputs.property "version", version outputs.file receiptFile outputs.cacheIf { true } doLast { def data = [ commitId: commitId, version: version, ] receiptFile.parentFile.mkdirs() def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n") receiptFile.setText(content, "ISO-8859-1") } } shadowJar { dependsOn createVersionFile // archiveClassifier defines the classifier for the shadow jar, the default is 'all'. // We don't want to use the default classifier because it will cause the shadow jar to // overwrite the original jar. We also don't want to use the 'shadow' classifier because // it will cause the shadow jar to be named kafka-clients-shadow.jar. We want to use the // same name as the original jar, kafka-clients.jar. archiveClassifier = null // KIP-714: move shaded dependencies to a shaded location relocate('io.opentelemetry.proto', 'org.apache.kafka.shaded.io.opentelemetry.proto') relocate('com.google.protobuf', 'org.apache.kafka.shaded.com.google.protobuf') // dependencies excluded from the final jar, since they are declared as runtime dependencies dependencies { project.configurations.shadowed.allDependencies.each { exclude(dependency(it)) } // exclude proto files from the jar exclude "**/opentelemetry/proto/**/*.proto" exclude "**/google/protobuf/*.proto" } from("${layout.buildDirectory.get().asFile.path}") { include "kafka/$buildVersionFileName" } from "$rootDir/LICENSE" from "$rootDir/NOTICE" } jar { enabled false dependsOn 'shadowJar' } clean.doFirst { delete "${layout.buildDirectory.get().asFile.path}/kafka/" } task processMessages(type:JavaExec) { mainClass = "org.apache.kafka.message.MessageGenerator" classpath = configurations.generator args = [ "-p", "org.apache.kafka.common.message", "-o", "${projectDir}/build/generated/main/java/org/apache/kafka/common/message", "-i", "src/main/resources/common/message", "-t", "ApiMessageTypeGenerator", "-m", "MessageDataGenerator", "JsonConverterGenerator" ] inputs.dir("src/main/resources/common/message") .withPropertyName("messages") .withPathSensitivity(PathSensitivity.RELATIVE) outputs.cacheIf { true } outputs.dir("${projectDir}/build/generated/main/java/org/apache/kafka/common/message") } task processTestMessages(type:JavaExec) { mainClass = "org.apache.kafka.message.MessageGenerator" classpath = configurations.generator args = [ "-p", "org.apache.kafka.common.message", "-o", "${projectDir}/build/generated/test/java/org/apache/kafka/common/message", "-i", "src/test/resources/common/message", "-m", "MessageDataGenerator", "JsonConverterGenerator" ] inputs.dir("src/test/resources/common/message") .withPropertyName("testMessages") .withPathSensitivity(PathSensitivity.RELATIVE) outputs.cacheIf { true } outputs.dir("${projectDir}/build/generated/test/java/org/apache/kafka/common/message") } sourceSets { main { java { srcDirs = ["src/main/java", "${projectDir}/build/generated/main/java"] } } test { java { srcDirs = ["src/test/java", "${projectDir}/build/generated/test/java"] } } } compileJava.dependsOn 'processMessages' srcJar.dependsOn 'processMessages' compileTestJava.dependsOn 'processTestMessages' javadoc { include "**/org/apache/kafka/clients/admin/*" include "**/org/apache/kafka/clients/consumer/*" include "**/org/apache/kafka/clients/producer/*" include "**/org/apache/kafka/common/*" include "**/org/apache/kafka/common/acl/*" include "**/org/apache/kafka/common/annotation/*" include "**/org/apache/kafka/common/config/*" include "**/org/apache/kafka/common/config/provider/*" include "**/org/apache/kafka/common/errors/*" include "**/org/apache/kafka/common/header/*" include "**/org/apache/kafka/common/metrics/*" include "**/org/apache/kafka/common/metrics/stats/*" include "**/org/apache/kafka/common/quota/*" include "**/org/apache/kafka/common/resource/*" include "**/org/apache/kafka/common/security/auth/*" include "**/org/apache/kafka/common/security/oauthbearer/*" include "**/org/apache/kafka/common/security/plain/*" include "**/org/apache/kafka/common/security/scram/*" include "**/org/apache/kafka/common/security/token/delegation/*" include "**/org/apache/kafka/common/serialization/*" include "**/org/apache/kafka/server/authorizer/*" include "**/org/apache/kafka/server/policy/*" include "**/org/apache/kafka/server/quota/*" include "**/org/apache/kafka/server/telemetry/*" } } project(':clients:clients-integration-tests') { base { archivesName = "kafka-clients-integration-tests" } dependencies { testImplementation libs.metrics testImplementation libs.slf4jApi testImplementation project(':test-common:test-common-internal-api') testImplementation project(':test-common:test-common-runtime') testImplementation project(':metadata') testImplementation project(':server') testImplementation project(':storage') testImplementation project(':core').sourceSets.test.output testImplementation project(':clients').sourceSets.test.output implementation project(':server-common') testImplementation project(':server-common').sourceSets.test.output testImplementation project(':metadata') implementation project(':group-coordinator') implementation project(':group-coordinator:group-coordinator-api') implementation project(':transaction-coordinator') testImplementation project(':test-common:test-common-util') testImplementation libs.junitJupiter testImplementation libs.junitPlatformSuiteEngine testRuntimeOnly runtimeTestLibs } checkstyle { configProperties = checkstyleConfigProperties("import-control-clients-integration-tests.xml") } } project(':raft') { base { archivesName = "kafka-raft" } configurations { generator } dependencies { implementation project(':server-common') implementation project(':clients') implementation libs.jacksonDatabind implementation libs.slf4jApi testImplementation project(':server-common') testImplementation project(':server-common').sourceSets.test.output testImplementation project(':clients') testImplementation project(':clients').sourceSets.test.output testImplementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testImplementation libs.mockitoCore testImplementation libs.jqwik testImplementation testLog4j2Libs testRuntimeOnly runtimeTestLibs generator project(':generator') } task createVersionFile() { def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName") inputs.property "commitId", commitId inputs.property "version", version outputs.file receiptFile outputs.cacheIf { true } doLast { def data = [ commitId: commitId, version: version, ] receiptFile.parentFile.mkdirs() def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n") receiptFile.setText(content, "ISO-8859-1") } } task processMessages(type:JavaExec) { mainClass = "org.apache.kafka.message.MessageGenerator" classpath = configurations.generator args = [ "-p", "org.apache.kafka.raft.generated", "-o", "${projectDir}/build/generated/main/java/org/apache/kafka/raft/generated", "-i", "src/main/resources/common/message", "-m", "MessageDataGenerator", "JsonConverterGenerator"] inputs.dir("src/main/resources/common/message") .withPropertyName("messages") .withPathSensitivity(PathSensitivity.RELATIVE) outputs.cacheIf { true } outputs.dir("${projectDir}/build/generated/main/java/org/apache/kafka/raft/generated") } sourceSets { main { java { srcDirs = ["src/main/java", "${projectDir}/build/generated/main/java"] } } test { java { srcDirs = ["src/test/java"] } } } compileJava.dependsOn 'processMessages' srcJar.dependsOn 'processMessages' jar { dependsOn createVersionFile from("${layout.buildDirectory.get().asFile.path}") { include "kafka/$buildVersionFileName" } } test { useJUnitPlatform { includeEngines 'jqwik', 'junit-jupiter' } } clean.doFirst { delete "${layout.buildDirectory.get().asFile.path}/kafka/" } javadoc { enabled = false } } project(':server-common') { base { archivesName = "kafka-server-common" } dependencies { api project(':clients') implementation libs.metrics implementation libs.joptSimple implementation libs.jacksonDatabind implementation libs.pcollections implementation libs.slf4jApi testImplementation project(':clients') testImplementation project(':clients').sourceSets.test.output testImplementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testImplementation libs.mockitoCore testImplementation testLog4j2Libs testRuntimeOnly runtimeTestLibs } task createVersionFile() { def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName") inputs.property "commitId", commitId inputs.property "version", version outputs.file receiptFile outputs.cacheIf { true } doLast { def data = [ commitId: commitId, version: version, ] receiptFile.parentFile.mkdirs() def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n") receiptFile.setText(content, "ISO-8859-1") } } jar { dependsOn createVersionFile from("${layout.buildDirectory.get().asFile.path}") { include "kafka/$buildVersionFileName" } } clean.doFirst { delete "${layout.buildDirectory.get().asFile.path}/kafka/" } checkstyle { configProperties = checkstyleConfigProperties("import-control-server-common.xml") } javadoc { enabled = false } } project(':storage:storage-api') { base { archivesName = "kafka-storage-api" } dependencies { implementation project(':clients') implementation project(':server-common') implementation libs.metrics implementation libs.slf4jApi testImplementation project(':clients') testImplementation project(':clients').sourceSets.test.output testImplementation libs.junitJupiter testImplementation libs.mockitoCore testImplementation testLog4j2Libs testRuntimeOnly runtimeTestLibs } task createVersionFile() { def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName") inputs.property "commitId", commitId inputs.property "version", version outputs.file receiptFile outputs.cacheIf { true } doLast { def data = [ commitId: commitId, version: version, ] receiptFile.parentFile.mkdirs() def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n") receiptFile.setText(content, "ISO-8859-1") } } jar { dependsOn createVersionFile from("${layout.buildDirectory.get().asFile.path}") { include "kafka/$buildVersionFileName" } } clean.doFirst { delete "${layout.buildDirectory.get().asFile.path}/kafka/" } javadoc { include "**/org/apache/kafka/server/log/remote/storage/*" } checkstyle { configProperties = checkstyleConfigProperties("import-control-storage.xml") } } project(':storage') { base { archivesName = "kafka-storage" } configurations { generator } dependencies { implementation project(':metadata') implementation project(':storage:storage-api') implementation project(':server-common') implementation project(':clients') implementation(libs.caffeine) { exclude group: 'org.checkerframework', module: 'checker-qual' exclude group: 'com.google.errorprone', module: 'error_prone_annotations' } implementation libs.slf4jApi implementation libs.jacksonDatabind implementation libs.metrics testImplementation project(':clients') testImplementation project(':clients').sourceSets.test.output testImplementation project(':core') testImplementation project(':core').sourceSets.test.output testImplementation project(':storage:storage-api').sourceSets.test.output testImplementation project(':test-common:test-common-internal-api') testImplementation project(':test-common:test-common-runtime') testImplementation project(':test-common:test-common-util') testImplementation project(':server') testImplementation project(':server-common') testImplementation project(':server-common').sourceSets.test.output testImplementation project(':transaction-coordinator') testImplementation libs.hamcrest testImplementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testImplementation libs.mockitoCore testImplementation libs.bcpkix testImplementation testLog4j2Libs testRuntimeOnly runtimeTestLibs generator project(':generator') } task createVersionFile() { def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName") inputs.property "commitId", commitId inputs.property "version", version outputs.file receiptFile outputs.cacheIf { true } doLast { def data = [ commitId: commitId, version: version, ] receiptFile.parentFile.mkdirs() def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n") receiptFile.setText(content, "ISO-8859-1") } } task processMessages(type:JavaExec) { mainClass = "org.apache.kafka.message.MessageGenerator" classpath = configurations.generator args = [ "-p", "org.apache.kafka.server.log.remote.metadata.storage.generated", "-o", "${projectDir}/build/generated/main/java/org/apache/kafka/server/log/remote/metadata/storage/generated", "-i", "src/main/resources/message", "-m", "MessageDataGenerator", "JsonConverterGenerator", "-t", "MetadataRecordTypeGenerator", "MetadataJsonConvertersGenerator" ] inputs.dir("src/main/resources/message") .withPropertyName("messages") .withPathSensitivity(PathSensitivity.RELATIVE) outputs.cacheIf { true } outputs.dir("${projectDir}/build/generated/main/java/org/apache/kafka/server/log/remote/metadata/storage/generated") } task genRemoteLogManagerConfigDoc(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.server.log.remote.storage.RemoteLogManagerConfig' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "remote_log_manager_config.html").newOutputStream() } task genRemoteLogMetadataManagerConfigDoc(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "remote_log_metadata_manager_config.html").newOutputStream() } sourceSets { main { java { srcDirs = ["src/main/java", "${projectDir}/build/generated/main/java"] } } test { java { srcDirs = ["src/test/java"] } } } compileJava.dependsOn 'processMessages' srcJar.dependsOn 'processMessages' jar { dependsOn createVersionFile from("${layout.buildDirectory.get().asFile.path}") { include "kafka/$buildVersionFileName" } } clean.doFirst { delete "${layout.buildDirectory.get().asFile.path}/kafka/" } javadoc { enabled = false } checkstyle { configProperties = checkstyleConfigProperties("import-control-storage.xml") } } project(':tools:tools-api') { base { archivesName = "kafka-tools-api" } dependencies { implementation project(':clients') testImplementation libs.junitJupiter testImplementation testLog4j2Libs testRuntimeOnly runtimeTestLibs } task createVersionFile() { def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName") inputs.property "commitId", commitId inputs.property "version", version outputs.file receiptFile outputs.cacheIf { true } doLast { def data = [ commitId: commitId, version: version, ] receiptFile.parentFile.mkdirs() def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n") receiptFile.setText(content, "ISO-8859-1") } } jar { dependsOn createVersionFile from("${layout.buildDirectory.get().asFile.path}") { include "kafka/$buildVersionFileName" } } clean.doFirst { delete "${layout.buildDirectory.get().asFile.path}/kafka/" } javadoc { include "**/org/apache/kafka/tools/api/*" } } project(':tools') { base { archivesName = "kafka-tools" } configurations { releaseOnly } dependencies { releaseOnly log4jReleaseLibs implementation project(':clients') implementation project(':metadata') implementation project(':storage') implementation project(':server') implementation project(':server-common') implementation project(':connect:runtime') implementation project(':tools:tools-api') implementation project(':transaction-coordinator') implementation project(':group-coordinator') implementation project(':coordinator-common') implementation project(':share-coordinator') implementation project(':raft') implementation libs.argparse4j implementation libs.jacksonDatabind implementation libs.jacksonDataformatCsv implementation libs.jacksonJDK8Datatypes implementation libs.joptSimple implementation libs.slf4jApi implementation libs.re2j implementation libs.jose4j // for SASL/OAUTHBEARER JWT validation implementation libs.jacksonJakartarsJsonProvider compileOnly libs.spotbugs testImplementation project(':clients') testImplementation project(':clients').sourceSets.test.output testImplementation project(':server') testImplementation project(':server').sourceSets.test.output testImplementation project(':core') testImplementation project(':core').sourceSets.test.output testImplementation project(':test-common:test-common-internal-api') testImplementation project(':test-common:test-common-runtime') testImplementation project(':server-common') testImplementation project(':server-common').sourceSets.test.output testImplementation project(':connect:api') testImplementation project(':connect:runtime') testImplementation project(':connect:runtime').sourceSets.test.output testImplementation project(':storage:storage-api').sourceSets.main.output testImplementation project(':storage').sourceSets.test.output testImplementation project(':streams') testImplementation project(':streams').sourceSets.test.output testImplementation project(':streams:integration-tests').sourceSets.test.output testImplementation libs.junitJupiter testImplementation libs.mockitoCore testImplementation libs.mockitoJunitJupiter // supports MockitoExtension testImplementation libs.bcpkix // required by the clients test module, but we have to specify it explicitly as gradle does not include the transitive test dependency automatically testImplementation(libs.jfreechart) { exclude group: 'junit', module: 'junit' } testImplementation libs.apachedsCoreApi testImplementation libs.apachedsInterceptorKerberos testImplementation libs.apachedsProtocolShared testImplementation libs.apachedsProtocolKerberos testImplementation libs.apachedsProtocolLdap testImplementation libs.apachedsLdifPartition testImplementation testLog4j2Libs testRuntimeOnly runtimeTestLibs testRuntimeOnly libs.hamcrest } javadoc { enabled = false } tasks.create(name: "copyDependantLibs", type: Copy) { from (configurations.runtimeClasspath) { exclude('kafka-clients*') } from (configurations.releaseOnly) into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}" duplicatesStrategy 'exclude' } jar { dependsOn 'copyDependantLibs' } } project(':trogdor') { base { archivesName = "trogdor" } dependencies { implementation project(':clients') implementation libs.argparse4j implementation libs.jacksonDatabind implementation libs.jacksonJDK8Datatypes implementation libs.slf4jApi implementation libs.jacksonJakartarsJsonProvider implementation libs.jerseyContainerServlet implementation libs.jerseyHk2 implementation libs.jaxbApi // Jersey dependency that was available in the JDK before Java 9 implementation libs.activation // Jersey dependency that was available in the JDK before Java 9 implementation (libs.jettyServer) { exclude group: 'org.slf4j', module: 'slf4j-api' } implementation (libs.jettyServlet) { exclude group: 'org.slf4j', module: 'slf4j-api' } implementation (libs.jettyServlets) { exclude group: 'org.slf4j', module: 'slf4j-api' } implementation project(':group-coordinator') implementation project(':group-coordinator:group-coordinator-api') testImplementation project(':clients') testImplementation project(':clients').sourceSets.test.output testImplementation project(':group-coordinator') testImplementation libs.junitJupiter testImplementation libs.mockitoCore testImplementation testLog4j2Libs testRuntimeOnly runtimeTestLibs testRuntimeOnly libs.junitPlatformLanucher } javadoc { enabled = false } tasks.create(name: "copyDependantLibs", type: Copy) { from (configurations.runtimeClasspath) { exclude('kafka-clients*') } into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}" duplicatesStrategy 'exclude' } jar { dependsOn 'copyDependantLibs' } } project(':shell') { base { archivesName = "kafka-shell" } dependencies { implementation libs.argparse4j implementation libs.jacksonDatabind implementation libs.jacksonJDK8Datatypes implementation libs.jline implementation libs.slf4jApi implementation project(':server-common') implementation project(':clients') implementation project(':core') implementation project(':metadata') implementation project(':raft') implementation libs.jose4j // for SASL/OAUTHBEARER JWT validation implementation libs.jacksonJakartarsJsonProvider testImplementation project(':clients') testImplementation project(':clients').sourceSets.test.output testImplementation project(':core') testImplementation project(':server-common') testImplementation project(':server-common').sourceSets.test.output testImplementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testImplementation testLog4j2Libs testRuntimeOnly runtimeTestLibs } javadoc { enabled = false } tasks.create(name: "copyDependantLibs", type: Copy) { from (configurations.runtimeClasspath) { include('jline-*jar') } into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}" duplicatesStrategy 'exclude' } jar { dependsOn 'copyDependantLibs' } } project(':streams') { base { archivesName = "kafka-streams" } ext.buildStreamsVersionFileName = "kafka-streams-version.properties" configurations { generator } dependencies { api project(path: ':clients', configuration: 'shadow') // `org.rocksdb.Options` is part of Kafka Streams public api via `RocksDBConfigSetter` api libs.rocksDBJni implementation libs.jacksonAnnotations implementation libs.jacksonDatabind implementation libs.slf4jApi // testCompileOnly prevents streams from exporting a dependency on test-utils, which would cause a dependency cycle testCompileOnly project(':streams:test-utils') testCompileOnly libs.bndlib testImplementation project(':clients').sourceSets.test.output testImplementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testImplementation libs.bcpkix testImplementation libs.hamcrest testImplementation libs.mockitoCore testImplementation libs.mockitoJunitJupiter // supports MockitoExtension testImplementation libs.junitPlatformSuiteEngine // supports suite test testImplementation testLog4j2Libs testRuntimeOnly project(':streams:test-utils') testRuntimeOnly runtimeTestLibs generator project(':generator') } task processMessages(type:JavaExec) { mainClass = "org.apache.kafka.message.MessageGenerator" classpath = configurations.generator args = [ "-p", "org.apache.kafka.streams.internals.generated", "-o", "${projectDir}/build/generated/main/java/org/apache/kafka/streams/internals/generated", "-i", "src/main/resources/common/message", "-m", "MessageDataGenerator" ] inputs.dir("src/main/resources/common/message") .withPropertyName("messages") .withPathSensitivity(PathSensitivity.RELATIVE) outputs.cacheIf { true } outputs.dir("${projectDir}/build/generated/main/java/org/apache/kafka/streams/internals/generated") } sourceSets { main { java { srcDirs = ["src/main/java", "${projectDir}/build/generated/main/java"] } } test { java { srcDirs = ["src/test/java"] } } } compileJava.dependsOn 'processMessages' srcJar.dependsOn 'processMessages' javadoc { include "**/org/apache/kafka/streams/**" exclude "**/org/apache/kafka/streams/internals/**", "**/org/apache/kafka/streams/**/internals/**" } tasks.create(name: "copyDependantLibs", type: Copy) { from (configurations.runtimeClasspath) { exclude('kafka-clients*') } into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}" duplicatesStrategy 'exclude' } task createStreamsVersionFile() { def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildStreamsVersionFileName") inputs.property "commitId", commitId inputs.property "version", version outputs.file receiptFile doLast { def data = [ commitId: commitId, version: version, ] receiptFile.parentFile.mkdirs() def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n") receiptFile.setText(content, "ISO-8859-1") } } jar { dependsOn 'createStreamsVersionFile' from("${layout.buildDirectory.get().asFile.path}") { include "kafka/$buildStreamsVersionFileName" } dependsOn 'copyDependantLibs' } systemTestLibs { dependsOn testJar } task genStreamsConfigDocs(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.streams.StreamsConfig' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "streams_config.html").newOutputStream() } task testAll( dependsOn: [ ':streams:test', ':streams:integration-tests:test', ':streams:test-utils:test', ':streams:streams-scala:test', ':streams:upgrade-system-tests-0110:test', ':streams:upgrade-system-tests-10:test', ':streams:upgrade-system-tests-11:test', ':streams:upgrade-system-tests-20:test', ':streams:upgrade-system-tests-21:test', ':streams:upgrade-system-tests-22:test', ':streams:upgrade-system-tests-23:test', ':streams:upgrade-system-tests-24:test', ':streams:upgrade-system-tests-25:test', ':streams:upgrade-system-tests-26:test', ':streams:upgrade-system-tests-27:test', ':streams:upgrade-system-tests-28:test', ':streams:upgrade-system-tests-30:test', ':streams:upgrade-system-tests-31:test', ':streams:upgrade-system-tests-32:test', ':streams:upgrade-system-tests-33:test', ':streams:upgrade-system-tests-34:test', ':streams:upgrade-system-tests-35:test', ':streams:upgrade-system-tests-36:test', ':streams:upgrade-system-tests-37:test', ':streams:upgrade-system-tests-38:test', ':streams:upgrade-system-tests-39:test', ':streams:upgrade-system-tests-40:test', ':streams:examples:test' ] ) } project(':streams:streams-scala') { apply plugin: 'scala' base { archivesName = "kafka-streams-scala_${versions.baseScala}" } dependencies { api project(':streams') api libs.scalaLibrary testImplementation project(':streams').sourceSets.test.output testImplementation project(':clients').sourceSets.test.output testImplementation project(':streams:test-utils') testImplementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testImplementation libs.mockitoJunitJupiter // supports MockitoExtension testImplementation testLog4j2Libs testRuntimeOnly runtimeTestLibs } javadoc { include "**/org/apache/kafka/streams/scala/**" } scaladoc { scalaDocOptions.additionalParameters = ["-no-link-warnings"] } tasks.create(name: "copyDependantLibs", type: Copy) { from (configurations.runtimeClasspath) { exclude('kafka-streams*') } into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}" duplicatesStrategy 'exclude' } jar { dependsOn 'copyDependantLibs' } apply plugin: 'com.diffplug.spotless' spotless { scala { target '**/*.scala' scalafmt("$versions.scalafmt").configFile('../../checkstyle/.scalafmt.conf').scalaMajorVersion(versions.baseScala) licenseHeaderFile '../../checkstyle/java.header', 'package' } } } project(':streams:integration-tests') { apply plugin: 'scala' base { archivesName = "kafka-streams-integration-tests" } dependencies { implementation libs.slf4jApi implementation libs.scalaLibrary testImplementation project(':clients').sourceSets.test.output testImplementation project(':group-coordinator') testImplementation project(':server') testImplementation project(':server-common') testImplementation project(':server-common').sourceSets.test.output testImplementation project(':storage') testImplementation project(':streams').sourceSets.test.output testImplementation project(':streams:streams-scala') testImplementation project(':test-common:test-common-runtime') testImplementation project(':tools') testImplementation project(':transaction-coordinator') testImplementation libs.bcpkix testImplementation libs.hamcrest testImplementation libs.junitJupiter testImplementation libs.junitPlatformSuiteEngine // supports suite test testImplementation libs.mockitoCore testImplementation testLog4j2Libs testImplementation project(':streams:test-utils') testRuntimeOnly runtimeTestLibs } sourceSets { // Set java/scala source folders in the `scala` block to enable joint compilation main { java { srcDirs = [] } scala { srcDirs = [] } } test { java { srcDirs = [] } scala { srcDirs = ["src/test/java", "src/test/scala"] } } } } project(':streams:test-utils') { base { archivesName = "kafka-streams-test-utils" } dependencies { api project(':streams') api project(':clients') implementation libs.slf4jApi testImplementation project(':clients').sourceSets.test.output testImplementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testImplementation libs.mockitoCore testImplementation libs.hamcrest testImplementation testLog4j2Libs testRuntimeOnly runtimeTestLibs } tasks.create(name: "copyDependantLibs", type: Copy) { from (configurations.runtimeClasspath) { exclude('kafka-streams*') } into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}" duplicatesStrategy 'exclude' } jar { dependsOn 'copyDependantLibs' } } project(':streams:examples') { base { archivesName = "kafka-streams-examples" } dependencies { implementation project(':streams') implementation libs.slf4jApi implementation libs.jacksonDatabind implementation libs.jacksonAnnotations testImplementation project(':streams:test-utils') testImplementation project(':clients').sourceSets.test.output // for org.apache.kafka.test.IntegrationTest testImplementation libs.junitJupiter testImplementation libs.hamcrest testImplementation testLog4j2Libs testRuntimeOnly runtimeTestLibs } javadoc { enabled = false } tasks.create(name: "copyDependantLibs", type: Copy) { from (configurations.runtimeClasspath) { exclude('kafka-streams*') } into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}" duplicatesStrategy 'exclude' } jar { dependsOn 'copyDependantLibs' } } project(':streams:upgrade-system-tests-0110') { base{ archivesName = "kafka-streams-upgrade-system-tests-0110" } dependencies { testImplementation libs.kafkaStreams_0110 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-10') { base { archivesName = "kafka-streams-upgrade-system-tests-10" } dependencies { testImplementation libs.kafkaStreams_10 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-11') { base { archivesName = "kafka-streams-upgrade-system-tests-11" } dependencies { testImplementation libs.kafkaStreams_11 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-20') { base { archivesName = "kafka-streams-upgrade-system-tests-20" } dependencies { testImplementation libs.kafkaStreams_20 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-21') { base { archivesName = "kafka-streams-upgrade-system-tests-21" } dependencies { testImplementation libs.kafkaStreams_21 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-22') { base { archivesName = "kafka-streams-upgrade-system-tests-22" } dependencies { testImplementation libs.kafkaStreams_22 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-23') { base { archivesName = "kafka-streams-upgrade-system-tests-23" } dependencies { testImplementation libs.kafkaStreams_23 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-24') { base { archivesName = "kafka-streams-upgrade-system-tests-24" } dependencies { testImplementation libs.kafkaStreams_24 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-25') { base { archivesName = "kafka-streams-upgrade-system-tests-25" } dependencies { testImplementation libs.kafkaStreams_25 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-26') { base { archivesName = "kafka-streams-upgrade-system-tests-26" } dependencies { testImplementation libs.kafkaStreams_26 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-27') { base { archivesName = "kafka-streams-upgrade-system-tests-27" } dependencies { testImplementation libs.kafkaStreams_27 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-28') { base { archivesName = "kafka-streams-upgrade-system-tests-28" } dependencies { testImplementation libs.kafkaStreams_28 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-30') { base { archivesName = "kafka-streams-upgrade-system-tests-30" } dependencies { testImplementation libs.kafkaStreams_30 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-31') { base { archivesName = "kafka-streams-upgrade-system-tests-31" } dependencies { testImplementation libs.kafkaStreams_31 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-32') { base { archivesName = "kafka-streams-upgrade-system-tests-32" } dependencies { testImplementation libs.kafkaStreams_32 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-33') { base { archivesName = "kafka-streams-upgrade-system-tests-33" } dependencies { testImplementation libs.kafkaStreams_33 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-34') { base { archivesName = "kafka-streams-upgrade-system-tests-34" } dependencies { testImplementation libs.kafkaStreams_34 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-35') { base { archivesName = "kafka-streams-upgrade-system-tests-35" } dependencies { testImplementation libs.kafkaStreams_35 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-36') { base { archivesName = "kafka-streams-upgrade-system-tests-36" } dependencies { testImplementation libs.kafkaStreams_36 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-37') { base { archivesName = "kafka-streams-upgrade-system-tests-37" } dependencies { testImplementation libs.kafkaStreams_37 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-38') { base { archivesName = "kafka-streams-upgrade-system-tests-38" } dependencies { testImplementation libs.kafkaStreams_38 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-39') { base { archivesName = "kafka-streams-upgrade-system-tests-39" } dependencies { testImplementation libs.kafkaStreams_39 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-40') { base { archivesName = "kafka-streams-upgrade-system-tests-40" } dependencies { testImplementation libs.kafkaStreams_40 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':streams:upgrade-system-tests-41') { base { archivesName = "kafka-streams-upgrade-system-tests-41" } dependencies { testImplementation libs.kafkaStreams_41 testRuntimeOnly libs.junitJupiter testRuntimeOnly runtimeTestLibs } systemTestLibs { dependsOn testJar } } project(':jmh-benchmarks') { apply plugin: 'com.gradleup.shadow' shadowJar { archiveBaseName = 'kafka-jmh-benchmarks' } dependencies { implementation(project(':core')) { // jmh requires jopt 4.x while `core` depends on 5.0, they are not binary compatible exclude group: 'net.sf.jopt-simple', module: 'jopt-simple' } implementation project(':server-common') implementation project(':server') implementation project(':raft') implementation project(':clients') implementation project(':coordinator-common') implementation project(':coordinator-common').sourceSets.test.output implementation project(':group-coordinator') implementation project(':group-coordinator:group-coordinator-api') implementation project(':metadata') implementation project(':storage') implementation project(':streams') implementation project(':transaction-coordinator') implementation project(':core') implementation project(':connect:api') implementation project(':connect:transforms') implementation project(':connect:json') implementation project(':clients').sourceSets.test.output implementation project(':core').sourceSets.test.output implementation project(':server-common').sourceSets.test.output implementation project(':metadata').sourceSets.test.output implementation libs.jmhCore annotationProcessor libs.jmhGeneratorAnnProcess implementation libs.jmhCoreBenchmarks implementation libs.jacksonDatabind implementation libs.metrics implementation libs.mockitoCore implementation libs.scalaLibrary implementation libs.slf4jApi } tasks.withType(JavaCompile) { // Suppress warning caused by code generated by jmh: `warning: [cast] redundant cast to long` options.compilerArgs << "-Xlint:-cast" } jar { manifest { attributes "Main-Class": "org.openjdk.jmh.Main" } } checkstyle { configProperties = checkstyleConfigProperties("import-control-jmh-benchmarks.xml") } task jmh(type: JavaExec, dependsOn: [':jmh-benchmarks:clean', ':jmh-benchmarks:shadowJar']) { mainClass = "-jar" doFirst { if (System.getProperty("jmhArgs")) { args System.getProperty("jmhArgs").split(' ') } args = [shadowJar.archiveFile.get().asFile, *args] } } javadoc { enabled = false } } project(':connect:api') { base { archivesName = "connect-api" } dependencies { api project(':clients') implementation libs.jakartaRsApi implementation libs.slf4jApi testImplementation libs.junitJupiter testImplementation project(':clients').sourceSets.test.output testImplementation testLog4j2Libs testRuntimeOnly runtimeTestLibs } javadoc { include "**/org/apache/kafka/connect/**" // needed for the `aggregatedJavadoc` task } tasks.create(name: "copyDependantLibs", type: Copy) { from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } into "${layout.buildDirectory.get().asFile.path}/dependant-libs" duplicatesStrategy 'exclude' } jar { dependsOn copyDependantLibs } } project(':connect:transforms') { base { archivesName = "connect-transforms" } dependencies { api project(':connect:api') implementation libs.slf4jApi testImplementation libs.junitJupiter testImplementation project(':clients').sourceSets.test.output testImplementation testLog4j2Libs testRuntimeOnly runtimeTestLibs } javadoc { enabled = false } tasks.create(name: "copyDependantLibs", type: Copy) { from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } into "${layout.buildDirectory.get().asFile.path}/dependant-libs" duplicatesStrategy 'exclude' } jar { dependsOn copyDependantLibs } } project(':connect:json') { base { archivesName = "connect-json" } dependencies { api project(':connect:api') api libs.jacksonDatabind api libs.jacksonJDK8Datatypes api libs.jacksonBlackbird implementation libs.slf4jApi testImplementation libs.junitJupiter testImplementation testLog4j2Libs testImplementation project(':clients').sourceSets.test.output testRuntimeOnly runtimeTestLibs } javadoc { enabled = false } tasks.create(name: "copyDependantLibs", type: Copy) { from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } into "${layout.buildDirectory.get().asFile.path}/dependant-libs" duplicatesStrategy 'exclude' } jar { dependsOn copyDependantLibs } } project(':connect:runtime') { configurations { swagger } base { archivesName = "connect-runtime" } dependencies { // connect-runtime is used in tests, use `api` for modules below for backwards compatibility even though // applications should generally not depend on `connect-runtime` api project(':connect:api') api project(':clients') api project(':connect:json') api project(':connect:transforms') compileOnly log4j2Libs implementation libs.slf4jApi implementation libs.jose4j // for SASL/OAUTHBEARER JWT validation implementation libs.jacksonAnnotations implementation libs.jacksonJakartarsJsonProvider implementation libs.jerseyContainerServlet implementation libs.jerseyHk2 implementation libs.jaxbApi // Jersey dependency that was available in the JDK before Java 9 implementation libs.activation // Jersey dependency that was available in the JDK before Java 9 implementation (libs.jettyServer) { exclude group: 'org.slf4j', module: 'slf4j-api' } implementation (libs.jettyServlet) { exclude group: 'org.slf4j', module: 'slf4j-api' } implementation (libs.jettyServlets) { exclude group: 'org.slf4j', module: 'slf4j-api' } implementation (libs.jettyClient) { exclude group: 'org.slf4j', module: 'slf4j-api' } implementation libs.classgraph implementation libs.mavenArtifact implementation libs.swaggerAnnotations compileOnly libs.bndlib compileOnly libs.spotbugs // We use this library to generate OpenAPI docs for the REST API, but we don't want or need it at compile // or run time. So, we add it to a separate configuration, which we use later on during docs generation swagger libs.jakartaServletApi swagger libs.jaxrs2Jakarta testImplementation project(':clients').sourceSets.test.output testImplementation project(':core') testImplementation project(':server') testImplementation project(':metadata') testImplementation project(':server-common') testImplementation project(':test-common:test-common-internal-api') testImplementation project(':test-common:test-common-util') testImplementation project(':test-common:test-common-runtime') testImplementation project(':server-common') testImplementation project(':server') testImplementation project(':group-coordinator') testImplementation project(':storage') testImplementation project(':connect:test-plugins') testImplementation project(':server-common').sourceSets.test.output testImplementation libs.jacksonDatabindYaml testImplementation libs.junitJupiter testImplementation libs.mockitoCore testImplementation libs.mockitoJunitJupiter testImplementation libs.httpclient testImplementation testLog4j2Libs testCompileOnly libs.bndlib testRuntimeOnly libs.bcpkix testRuntimeOnly runtimeTestLibs } javadoc { enabled = false } tasks.create(name: "copyDependantLibs", type: Copy) { from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } into "${layout.buildDirectory.get().asFile.path}/dependant-libs" duplicatesStrategy 'exclude' } jar { dependsOn copyDependantLibs } task genConnectConfigDocs(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.connect.runtime.distributed.DistributedConfig' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "connect_config.html").newOutputStream() } task genSinkConnectorConfigDocs(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.connect.runtime.SinkConnectorConfig' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "sink_connector_config.html").newOutputStream() } task genSourceConnectorConfigDocs(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.connect.runtime.SourceConnectorConfig' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "source_connector_config.html").newOutputStream() } task genConnectTransformationDocs(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.connect.tools.TransformationDoc' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "connect_transforms.html").newOutputStream() } task genConnectPredicateDocs(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.connect.tools.PredicateDoc' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "connect_predicates.html").newOutputStream() } task genConnectMetricsDocs(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.connect.runtime.ConnectMetrics' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "connect_metrics.html").newOutputStream() } task setVersionInOpenAPISpec(type: Copy) { from "$rootDir/gradle/openapi.template" into "${layout.buildDirectory.get().asFile.path}/resources/docs" rename ('openapi.template', 'openapi.yaml') expand(kafkaVersion: "$rootProject.version") } task genConnectOpenAPIDocs(type: io.swagger.v3.plugins.gradle.tasks.ResolveTask, dependsOn: setVersionInOpenAPISpec) { classpath = sourceSets.main.runtimeClasspath buildClasspath = classpath + configurations.swagger outputFileName = 'connect_rest' outputFormat = 'YAML' prettyPrint = true sortOutput = true encoding = 'UTF-8' alwaysResolveAppPath = false skipResolveAppPath = false readAllResources = true openApiFile = file("${layout.buildDirectory.get().asFile.path}/resources/docs/openapi.yaml") resourcePackages = ['org.apache.kafka.connect.runtime.rest.resources'] if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } outputDir = file(generatedDocsDir) } } project(':connect:file') { base { archivesName = "connect-file" } dependencies { implementation project(':connect:api') implementation libs.slf4jApi testImplementation testLog4j2Libs testImplementation libs.junitJupiter testImplementation libs.mockitoCore testImplementation project(':clients').sourceSets.test.output testImplementation project(':connect:runtime') testImplementation project(':connect:runtime').sourceSets.test.output testImplementation project(':core') testImplementation project(':test-common:test-common-runtime') testImplementation project(':server-common').sourceSets.test.output testRuntimeOnly runtimeTestLibs } javadoc { enabled = false } tasks.create(name: "copyDependantLibs", type: Copy) { from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } into "${layout.buildDirectory.get().asFile.path}/dependant-libs" duplicatesStrategy 'exclude' } jar { dependsOn copyDependantLibs } } project(':connect:basic-auth-extension') { base { archivesName = "connect-basic-auth-extension" } dependencies { implementation project(':connect:api') implementation libs.slf4jApi implementation libs.jakartaRsApi implementation libs.jaxAnnotationApi testImplementation libs.bcpkix testImplementation libs.mockitoCore testImplementation libs.junitJupiter testImplementation testLog4j2Libs testImplementation project(':clients').sourceSets.test.output testRuntimeOnly libs.jerseyContainerServlet testRuntimeOnly runtimeTestLibs } javadoc { enabled = false } tasks.create(name: "copyDependantLibs", type: Copy) { from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } into "${layout.buildDirectory.get().asFile.path}/dependant-libs" duplicatesStrategy 'exclude' } jar { dependsOn copyDependantLibs } } project(':connect:mirror') { base { archivesName = "connect-mirror" } dependencies { implementation project(':connect:api') implementation project(':connect:runtime') implementation project(':connect:mirror-client') implementation project(':clients') implementation libs.argparse4j implementation libs.slf4jApi implementation libs.jacksonAnnotations implementation libs.jacksonJakartarsJsonProvider implementation libs.jerseyContainerServlet implementation libs.jerseyHk2 implementation libs.jaxbApi // Jersey dependency that was available in the JDK before Java 9 implementation libs.activation // Jersey dependency that was available in the JDK before Java 9 implementation (libs.jettyServer) { exclude group: 'org.slf4j', module: 'slf4j-api' } implementation (libs.jettyServlet) { exclude group: 'org.slf4j', module: 'slf4j-api' } implementation (libs.jettyServlets) { exclude group: 'org.slf4j', module: 'slf4j-api' } implementation (libs.jettyClient) { exclude group: 'org.slf4j', module: 'slf4j-api' } implementation libs.swaggerAnnotations testImplementation testLog4j2Libs testImplementation libs.junitJupiter testImplementation libs.bndlib testImplementation libs.mockitoCore testImplementation project(':clients').sourceSets.test.output testImplementation project(':connect:runtime').sourceSets.test.output testImplementation project(':core') testImplementation project(':test-common:test-common-runtime') testImplementation project(':server') testImplementation project(':server-common') testRuntimeOnly project(':connect:runtime') testRuntimeOnly libs.bcpkix testRuntimeOnly runtimeTestLibs } javadoc { enabled = false } tasks.create(name: "copyDependantLibs", type: Copy) { from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } into "${layout.buildDirectory.get().asFile.path}/dependant-libs" duplicatesStrategy 'exclude' } task genMirrorConnectorConfigDocs(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.connect.mirror.MirrorConnectorConfig' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "mirror_connector_config.html").newOutputStream() } task genMirrorSourceConfigDocs(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.connect.mirror.MirrorSourceConfig' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "mirror_source_config.html").newOutputStream() } task genMirrorCheckpointConfigDocs(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.connect.mirror.MirrorCheckpointConfig' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "mirror_checkpoint_config.html").newOutputStream() } task genMirrorHeartbeatConfigDocs(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'org.apache.kafka.connect.mirror.MirrorHeartbeatConfig' if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } standardOutput = new File(generatedDocsDir, "mirror_heartbeat_config.html").newOutputStream() } jar { dependsOn copyDependantLibs } } project(':connect:mirror-client') { base { archivesName = "connect-mirror-client" } dependencies { implementation project(':clients') implementation libs.slf4jApi testImplementation testLog4j2Libs testImplementation libs.junitJupiter testImplementation project(':clients').sourceSets.test.output testRuntimeOnly runtimeTestLibs } javadoc { enabled = true } tasks.create(name: "copyDependantLibs", type: Copy) { from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } into "${layout.buildDirectory.get().asFile.path}/dependant-libs" duplicatesStrategy 'exclude' } jar { dependsOn copyDependantLibs } } project(':connect:test-plugins') { base { archivesName = "connect-test-plugins" } dependencies { api project(':connect:api') implementation project(':server-common') implementation libs.slf4jApi implementation libs.jacksonDatabind testImplementation testLog4j2Libs } } task aggregatedJavadoc(type: Javadoc, dependsOn: compileJava) { def projectsWithJavadoc = subprojects.findAll { it.javadoc.enabled } source = projectsWithJavadoc.collect { it.sourceSets.main.allJava } classpath = files(projectsWithJavadoc.collect { it.sourceSets.main.compileClasspath }) includes = projectsWithJavadoc.collectMany { it.javadoc.getIncludes() } excludes = projectsWithJavadoc.collectMany { it.javadoc.getExcludes() } }