KAFKA-20032: KIP-1265 Automatic detection of internal API usage

build.gradle

@@ -335,6 +335,8 @@ subprojects {
 
   apply plugin: 'checkstyle'
   apply plugin: "com.github.spotbugs"
+  apply plugin: 'org.apache.kafka.public-api-checker'
+
 
   // We use the shadow plugin for the jmh-benchmarks module and the `-all` jar can get pretty large, so
   // don't publish it
@@ -734,6 +736,31 @@ subprojects {
 
   check.dependsOn('javadoc')
 
+kafkaPublicApiChecker {
+
+    // Jars produced by this project — the surface that's actually validated.
+    projectJarFiles.from(jar.archiveFile)
+
+    // Sibling Kafka modules this project depends on (direct or transitive). Their classes
+    // are merged into the scanned surface so cross-module @InterfaceAudience.Public references
+    // resolve, but they don't contribute to this module's MISSING_JAVADOC or cascade
+    // iteration — each module checks its own surface.
+    referenceJarFiles.from(
+        configurations.compileClasspath.incoming.artifactView {
+            // Resolve project deps to their assembled JAR (not the classes-dir variant Gradle
+            // picks by default for compile-classpath), so the scanner only sees real jars.
+            attributes {
+                attribute(org.gradle.api.attributes.LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE,
+                          objects.named(org.gradle.api.attributes.LibraryElements,
+                                        org.gradle.api.attributes.LibraryElements.JAR))
+            }
+            componentFilter { it instanceof org.gradle.api.artifacts.component.ProjectComponentIdentifier }
+        }.files
+    )
+
+    enabled = true
+    failOnViolation = true
+}
   task systemTestLibs(dependsOn: jar)
 
   if (!sourceSets.test.allSource.isEmpty()) {
@@ -2136,6 +2163,7 @@ project(':clients') {
     include "**/org/apache/kafka/common/annotation/*"
     include "**/org/apache/kafka/common/config/*"
     include "**/org/apache/kafka/common/config/provider/*"
+    include "**/org/apache/kafka/common/config/types/*"
     include "**/org/apache/kafka/common/errors/*"
     include "**/org/apache/kafka/common/header/*"
     include "**/org/apache/kafka/common/metrics/*"

buildSrc/README.md

@@ -0,0 +1,80 @@
+# Kafka API-checker plugins (buildSrc)
+
+This module builds the KIP-1265 API-checker plugins from a single source tree:
+
+| Plugin / Mojo | ID / artifactId | Audience |
+|---|---|---|
+| `KafkaPublicApiCheckerPlugin` | `org.apache.kafka.public-api-checker` (Gradle) | Internal — applied to Kafka's own build to validate that `@InterfaceAudience.Public` types only expose other public types, and that javadoc inclusion matches the audience annotations. |
+| `KafkaInternalApiCheckerPlugin` | `org.apache.kafka.internal-api-checker` (Gradle) | External — published for plugin/connector/Streams-app developers to detect references from their bytecode to non-`@Public` Kafka classes. |
+| `KafkaInternalApiCheckerMojo` | `org.apache.kafka:kafka-internal-api-checker-maven-plugin` | External — Maven equivalent of the Gradle internal-API checker. |
+
+All three share the bytecode scanner and reporting code under `org.apache.kafka.apicheck`. ASM (9.6) does the scanning; nothing in the checker classloads scanned bytecode.
+
+End-user documentation (Gradle/Maven snippets, `@SuppressKafkaInternalApiUsage`, audience-inheritance rules) lives at [`docs/apis/internal-api-checker.md`](../docs/apis/internal-api-checker.md). The notes below cover building, testing, and publishing the plugins themselves.
+
+## Build
+
+```bash
+./gradlew :buildSrc:build
+```
+
+Produces both Gradle plugin jars, the Maven plugin jar (with `META-INF/maven/plugin.xml`), and runs the unit tests.
+
+## Test
+
+```bash
+./gradlew :buildSrc:test
+```
+
+Includes a Gradle TestKit end-to-end test that applies the published `org.apache.kafka.internal-api-checker` plugin to a synthetic consumer project.
+
+## Publish
+
+Both publications inherit the release version from `-PkafkaPluginsVersion` and stage to the URL passed via `-PmavenUrl` (with `-PmavenUsername` / `-PmavenPassword` for credentials). With no overrides the version is `1.0.0-SNAPSHOT` and the publish URL is empty (so `publish` is a no-op suitable for local smoke-testing of the artifact layout).
+
+```bash
+# Stage to ASF Nexus alongside the rest of an AK release
+./gradlew :buildSrc:publish \
+  -PkafkaPluginsVersion=$KAFKA_VERSION \
+  -PmavenUrl=$ASF_NEXUS_STAGING_URL \
+  -PmavenUsername=$NEXUS_USER \
+  -PmavenPassword=$NEXUS_PASS
+
+# Local smoke-test
+./gradlew :buildSrc:publish -PmavenUrl=file:///tmp/local-repo
+```
+
+The publish task produces three artifacts:
+
+- `org.apache.kafka:kafka-internal-api-checker-maven-plugin:$KAFKA_VERSION` — Maven plugin (packaging `maven-plugin`).
+- `org.apache.kafka.internal-api-checker:org.apache.kafka.internal-api-checker.gradle.plugin:$KAFKA_VERSION` — Gradle plugin marker that resolves `kafkaInternalApiChecker` to the implementation jar.
+- `org.apache.kafka:buildSrc:$KAFKA_VERSION` — the implementation jar that backs the Gradle plugin marker. (`pluginMaven` publication; the jar name comes from this module's directory.)
+
+Gradle Plugin Portal publishing is not wired here; the internal-API-checker is distributed through the same Maven coordinates as the rest of Kafka so existing AK consumers don't need a second repository.
+
+## Layout
+
+```
+buildSrc/
+├── build.gradle
+├── src/main/java/org/apache/kafka/
+│   ├── gradle/
+│   │   ├── KafkaInternalApiCheckerPlugin.java       # External Gradle plugin
+│   │   ├── KafkaInternalApiCheckerTask.java
+│   │   ├── KafkaInternalApiCheckerExtension.java
+│   │   ├── KafkaPublicApiCheckerPlugin.java         # Kafka-internal Gradle plugin
+│   │   ├── KafkaPublicApiCheckerTask.java
+│   │   └── KafkaPublicApiCheckerExtension.java
+│   ├── maven/
+│   │   └── KafkaInternalApiCheckerMojo.java         # External Maven mojo
+│   └── apicheck/                                    # Shared scanner / validator / reporter
+│       ├── ApiSurface.java
+│       ├── ApiSurfaceScanner.java
+│       ├── CascadeValidator.java
+│       ├── JavadocConsistencyValidator.java
+│       ├── PluginDeveloperApiUsageScanner.java
+│       ├── PublicApiChecker.java
+│       └── ViolationReporter.java
+└── src/main/resources/META-INF/maven/org.apache.kafka/
+    └── kafka-internal-api-checker-maven-plugin/plugin.xml
+```

buildSrc/build.gradle

@@ -0,0 +1,244 @@
+plugins {
+    id 'java-gradle-plugin'
+    id 'maven-publish'
+    id 'java-library'
+    id 'checkstyle'
+    id 'signing'
+}
+
+group = 'org.apache.kafka'
+// Default the checker plugin version to the root project's version (read directly from the
+// repo-root gradle.properties because buildSrc is a separate Gradle build and does not
+// inherit the root project's `version` property). release.py bumps gradle.properties via
+// the `updateVersion` task, so the plugin artifacts ship under the same dotted version as
+// the rest of Kafka. `-PkafkaPluginsVersion=...` overrides for one-off out-of-band publishes.
+def rootGradleProps = new Properties()
+def rootGradlePropsFile = file('../gradle.properties')
+if (rootGradlePropsFile.exists()) {
+    rootGradlePropsFile.withInputStream { rootGradleProps.load(it) }
+}
+def projectVersion = rootGradleProps.getProperty('version')
+version = project.findProperty('kafkaPluginsVersion') ?: (projectVersion ?: '1.0.0-SNAPSHOT')
+
+repositories {
+    gradlePluginPortal()
+    mavenCentral()
+}
+
+java {
+    sourceCompatibility = JavaVersion.VERSION_11
+    targetCompatibility = JavaVersion.VERSION_11
+    // Maven Central requires sources and javadoc jars alongside the main artifact.
+    withSourcesJar()
+    withJavadocJar()
+}
+
+// Single source of truth for the Maven dependency versions. Both the dependency block below
+// and the hand-maintained plugin.xml descriptor need to agree on these; processResources
+// templates the descriptor at build time so they can never drift.
+def mavenApiVersion = '3.9.4'
+def mavenPluginAnnotationsVersion = '3.9.0'
+
+dependencies {
+    // For bytecode scanning (used by the internal-API-usage checker)
+    implementation 'org.ow2.asm:asm:9.6'
+
+    // For Maven plugin functionality
+    implementation "org.apache.maven:maven-plugin-api:${mavenApiVersion}"
+    implementation "org.apache.maven:maven-core:${mavenApiVersion}"
+    implementation "org.apache.maven:maven-artifact:${mavenApiVersion}"
+    implementation "org.apache.maven.plugin-tools:maven-plugin-annotations:${mavenPluginAnnotationsVersion}"
+
+    // For testing
+    testImplementation 'org.junit.jupiter:junit-jupiter:5.9.2'
+    testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.9.2'
+    testImplementation 'org.junit.platform:junit-platform-launcher:1.9.2'
+    testImplementation 'org.mockito:mockito-core:5.3.1'
+    testImplementation gradleTestKit()
+}
+
+// Template plugin.xml from src/main/resources so its <version> and Maven dep versions are
+// computed from the same source as the published artifact / dependency block above. Using
+// Ant's @token@ delimiter rather than Groovy's `${...}` so Maven's runtime substitutions
+// (e.g. ${project.build.directory}) inside plugin.xml are left untouched.
+processResources {
+    filesMatching('META-INF/maven/plugin.xml') {
+        filter(org.apache.tools.ant.filters.ReplaceTokens, tokens: [
+            version: project.version.toString(),
+            mavenApiVersion: mavenApiVersion,
+            mavenPluginAnnotationsVersion: mavenPluginAnnotationsVersion,
+        ])
+    }
+}
+
+gradlePlugin {
+    plugins {
+        kafkaPublicApiChecker {
+            id = 'org.apache.kafka.public-api-checker'
+            implementationClass = 'org.apache.kafka.gradle.KafkaPublicApiCheckerPlugin'
+            displayName = 'Kafka Public API Checker'
+            description = 'Internal plugin for checking public API consistency in Kafka codebase'
+        }
+
+        kafkaInternalApiChecker {
+            id = 'org.apache.kafka.internal-api-checker'
+            implementationClass = 'org.apache.kafka.gradle.KafkaInternalApiCheckerPlugin'
+            displayName = 'Kafka Internal API Checker'
+            description = 'Plugin for external projects to check they don\'t use internal Kafka APIs'
+        }
+    }
+}
+
+// Configure publishing for both Gradle and Maven plugins
+publishing {
+    // Mirrors the root project's publishing.repositories block so the buildSrc artifacts
+    // (kafka-internal-api-checker Gradle plugin + kafka-internal-api-checker-maven-plugin Maven
+    // plugin) land in the same ASF Nexus / staging repo as the rest of the release. Local
+    // smoke-tests can override with -PmavenUrl=file:///tmp/local-repo .
+    repositories {
+        maven {
+            url = project.hasProperty('mavenUrl') ? project.mavenUrl : ''
+            credentials {
+                username = project.hasProperty('mavenUsername') ? project.mavenUsername : ''
+                password = project.hasProperty('mavenPassword') ? project.mavenPassword : ''
+            }
+        }
+    }
+    publications {
+        // Maven plugin (separate from Gradle plugin publication)
+        mavenPlugin(MavenPublication) {
+            group  = 'org.apache.kafka'
+            artifactId = 'kafka-internal-api-checker-maven-plugin'
+            version = project.version
+
+            from components.java
+
+            pom {
+                name = 'Apache Kafka Internal API Checker Maven Plugin'
+                description = 'Maven plugin to check that external projects don\'t use internal Kafka APIs'
+                url = 'https://kafka.apache.org'
+                packaging = 'maven-plugin'
+
+                licenses {
+                    license {
+                        name = 'The Apache License, Version 2.0'
+                        url = 'https://www.apache.org/licenses/LICENSE-2.0.txt'
+                    }
+                }
+
+                developers {
+                    developer {
+                        id = 'apache-kafka'
+                        name = 'Apache Kafka Team'
+                        email = 'dev@kafka.apache.org'
+                    }
+                }
+
+                scm {
+                    connection = 'scm:git:https://github.com/apache/kafka.git'
+                    developerConnection = 'scm:git:https://github.com/apache/kafka.git'
+                    url = 'https://github.com/apache/kafka'
+                }
+            }
+        }
+    }
+
+    // The `java-gradle-plugin` plugin auto-generates one Maven publication per declared
+    // plugin (the marker poms) plus a `pluginMaven` publication that holds the implementation
+    // jar both markers point to. Rename the implementation jar away from the buildSrc
+    // directory name so it has a sensible Maven coordinate.
+    afterEvaluate {
+        publications.matching { it.name == 'pluginMaven' }.configureEach {
+            artifactId = 'kafka-internal-api-checker-gradle-plugin'
+        }
+        publications.matching { it.name.endsWith('PluginMarkerMaven') }.configureEach {
+            pom {
+                name = 'Apache Kafka Gradle Plugins'
+                description = 'Gradle plugins for Apache Kafka public API validation'
+                url = 'https://kafka.apache.org'
+
+                licenses {
+                    license {
+                        name = 'The Apache License, Version 2.0'
+                        url = 'https://www.apache.org/licenses/LICENSE-2.0.txt'
+                    }
+                }
+
+                developers {
+                    developer {
+                        id = 'apache-kafka'
+                        name = 'Apache Kafka Team'
+                        email = 'dev@kafka.apache.org'
+                    }
+                }
+
+                scm {
+                    connection = 'scm:git:https://github.com/apache/kafka.git'
+                    developerConnection = 'scm:git:https://github.com/apache/kafka.git'
+                    url = 'https://github.com/apache/kafka'
+                }
+            }
+        }
+
+        // Also configure the main plugin publication
+        publications.matching { it.name == 'pluginMaven' }.configureEach {
+            pom {
+                name = 'Apache Kafka Gradle Plugins'
+                description = 'Gradle plugins for Apache Kafka public API validation'
+                url = 'https://kafka.apache.org'
+
+                licenses {
+                    license {
+                        name = 'The Apache License, Version 2.0'
+                        url = 'https://www.apache.org/licenses/LICENSE-2.0.txt'
+                    }
+                }
+
+                developers {
+                    developer {
+                        id = 'apache-kafka'
+                        name = 'Apache Kafka Team'
+                        email = 'dev@kafka.apache.org'
+                    }
+                }
+
+                scm {
+                    connection = 'scm:git:https://github.com/apache/kafka.git'
+                    developerConnection = 'scm:git:https://github.com/apache/kafka.git'
+                    url = 'https://github.com/apache/kafka'
+                }
+            }
+        }
+    }
+}
+
+// GPG-sign the published artifacts on real releases (i.e. when the version is not a SNAPSHOT
+// and -PskipSigning is not set), matching the policy in the root build.gradle. ASF Nexus
+// rejects unsigned artifacts on the staging repo, so this is required for `release.py stage`.
+def skipSigning = project.hasProperty('skipSigning') && skipSigning.toBoolean()
+def shouldSign = !skipSigning && !version.toString().endsWith('SNAPSHOT')
+if (shouldSign) {
+    signing {
+        sign publishing.publications
+    }
+}
+
+test {
+    useJUnitPlatform()
+}
+
+checkstyle {
+    toolVersion = '12.3.1'
+    configDirectory = file("$rootDir/../checkstyle")
+    configProperties = [importControlFile: 'import-control-buildsrc.xml']
+}
+
+tasks.named('checkstyleMain').configure {
+    group = 'Verification'
+    description = 'Run checkstyle on all main Java sources'
+}
+
+tasks.named('checkstyleTest').configure {
+    group = 'Verification'
+    description = 'Run checkstyle on all test Java sources'
+}