diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 931eade..006e0fe 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -249,6 +249,18 @@ jobs: -Dchdb.cache.dir="$PWD/target/ci-cache" \ org.chdb.examples.QuickStart + - name: Consume the driver as a repository artifact + # Issue #10: nothing had ever depended on this as a published artifact, and a reactor + # build cannot find the things a consumer hits -- the POMs come from target/, the BOM is + # never imported, and the native libraries are read from a directory rather than + # unpacked from a JAR in a local repository. + # + # The script deploys to a throwaway file:// repository and resolves from it in a project + # outside this checkout with its own empty local repository. Run here, on every platform, + # so that the answer does not go stale between releases; the one thing it cannot cover is + # a real network transfer, which only Central can. + run: scripts/verify-consumer.sh ${{ matrix.platform }} + - name: Archive debug symbols # Kept out of the runtime JAR (they are several times its size) but needed to read a # crash report, so they are archived per build. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..96cc6b6 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,527 @@ +name: release + +# Issue #15: assembling a release needs all four platform packages staged in one working +# directory, and no single machine can produce them -- build-native.sh refuses to cross-build, +# and macos-x86_64 needs an Intel Mac. `build` already builds all four on their own runners +# and throws the result away. This workflow keeps it: each platform stages on its own runner +# and uploads target/native/, and one deploy job reassembles the four into a single reactor and +# runs `mvn -Prelease deploy`. +# +# Three decisions the issue asked to settle, with the reasoning kept here rather than in a +# commit message nobody will read again: +# +# 1. WHERE THE VERSION BUMP LIVES -- in git, not here. +# This workflow never runs `versions:set`. A release is cut by committing the release +# version to the POMs and tagging that commit. Rationale: `versions:set` in CI means the +# bytes on Central correspond to no commit in this repository, so `git show ` cannot +# tell you what was published and a re-run of the same tag can produce different POMs. A +# `-SNAPSHOT` in the POM after the tag is the ordinary "back to development" commit. +# +# Enforced on both paths, not just the one that has a tag in its ref, and enforced twice. +# scripts/check-release-tag.sh refuses a release unless a tag named v exists +# and points at the commit being built; `preflight` runs it so a mistake costs nothing, and +# `deploy` runs it again as the last step before Maven, because between the two there is +# ten-plus minutes of staging and a manual approval gate during which a tag can be moved or +# deleted. Neither of those is a substitute for protecting the release tags against being +# moved -- they shrink the window, they do not remove it -- and docs/release-readiness.md +# step 8 records the protection as the other half of the same guarantee. The rule is worth +# having only if it has no bypass: a manual run used to be one, and a long approval gate +# was another. +# +# 2. SNAPSHOTS FROM THE SAME WORKFLOW -- yes, and that is the default for workflow_dispatch. +# Issue #10 is blocked on nothing ever having consumed this as a published artifact, and a +# snapshot is the only way to move it. Running the same jobs, the same staging, the same +# signing and the same plugin for a snapshot is what makes the release path already-tested +# when a release finally depends on it. The only difference is the version: the publishing +# plugin routes a `-SNAPSHOT` deploy straight to Central's snapshot repository instead of +# building a portal bundle, so autoPublish and portal review do not apply to it. Snapshot +# publishing is opt-in per namespace -- "Enable SNAPSHOTs" in the portal -- and the upload +# is rejected until that is done. See docs/publishing.md section 6. +# +# 3. REPRODUCIBILITY -- the engine is reproducible, the shim is not bit-for-bit, and we +# measured exactly how far off it is rather than leaving it open. +# Two consecutive clean builds of the macos-aarch64 shim on the same machine, same source, +# same toolchain, produced files of identical length differing in 116 bytes: +# - 16 bytes of LC_UUID, which ld64 derives from a content hash that includes the symbol +# table; +# - 4 bytes, one per translation unit, holding the N_OSO debug-map timestamp -- the mtime +# of each .o file (1788931405 vs 1788931408, three seconds apart); +# - ~96 bytes of ad-hoc code signature, which covers everything above. +# __text, __cstring, __const and __data were byte-identical. So the shim's *content* is +# reproducible and only build-time metadata varies. That is not worth chasing here: it +# would take a deterministic-UUID link plus zeroed object mtimes, and it would buy nothing +# a consumer can act on, because the shim is ~130 KB of a ~326 MB package. What does matter +# is the other 99.96%: the engine tarball is pinned by SHA-256 in scripts/engine.properties +# and fetch-libchdb.sh verifies it, so the engine bytes in a release are the engine bytes +# anyone else can download and check. This workflow therefore does not claim bit-identical +# builds; it records what it shipped -- every staged library's SHA-256 goes into the job +# summary and into manifest.properties inside the JAR. + +on: + push: + # A release is a tag on a commit whose POMs already carry the release version. `build` does + # not run on tags (it filters on branches), which is why preflight checks that `build` + # succeeded for this exact commit rather than assuming a tag implies a tested tree. + tags: ["v*"] + workflow_dispatch: + inputs: + channel: + description: "snapshot (Central snapshot repository) or release (portal bundle)" + type: choice + options: [snapshot, release] + default: snapshot + dry_run: + description: "Stage, sign and package, but upload nothing" + type: boolean + default: true + +# A second concurrent release would race on the portal deployment and, worse, make it unclear +# which run produced the bundle a human is about to confirm. Never cancel one in flight: a +# half-finished deploy is the one state this repository cannot recover from. +concurrency: + group: release + cancel-in-progress: false + +permissions: + contents: read + # preflight reads the `build` workflow's conclusion for this commit. + actions: read + +env: + MAVEN_ARGS: "--batch-mode --no-transfer-progress" + +jobs: + # Everything that can say "no" before 1.3 GB of native libraries move across the network. + preflight: + name: preflight + runs-on: ubuntu-latest + outputs: + version: ${{ steps.resolve.outputs.version }} + channel: ${{ steps.resolve.outputs.channel }} + dry_run: ${{ steps.resolve.outputs.dry_run }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "11" + cache: maven + + - name: Resolve the version and the channel + id: resolve + env: + # A tag push is always a release; workflow_dispatch says which it wants. + INPUT_CHANNEL: ${{ github.event_name == 'push' && 'release' || inputs.channel }} + # A tag push is never a dry run: the tag is the decision. + INPUT_DRY_RUN: ${{ github.event_name == 'push' && 'false' || inputs.dry_run }} + REF_NAME: ${{ github.ref_name }} + run: | + set -euo pipefail + + # The POMs are the single source of truth for the version. Read from Maven rather + # than with sed, so an inherited or property-substituted version reads correctly. + VERSION=$(mvn $MAVEN_ARGS -q -DforceStdout help:evaluate -Dexpression=project.version) + echo "POM version: $VERSION" + echo "channel: $INPUT_CHANNEL, dry run: $INPUT_DRY_RUN" + + case "$INPUT_CHANNEL" in + release) + case "$VERSION" in + *-SNAPSHOT) + echo "::error::the POMs are at $VERSION. A release needs a non-SNAPSHOT version committed and tagged; see the header of this workflow." + exit 1 + ;; + esac + if [ "${{ github.event_name }}" = "push" ]; then + # Decision 1 enforced: the tag names the bytes, so it has to agree with them. + TAG_VERSION="${REF_NAME#v}" + if [ "$TAG_VERSION" != "$VERSION" ]; then + echo "::error::tag $REF_NAME implies version $TAG_VERSION but the POMs say $VERSION. Commit the release version, then tag that commit." + exit 1 + fi + fi + ;; + snapshot) + case "$VERSION" in + *-SNAPSHOT) ;; + *) + echo "::error::the POMs are at $VERSION, which is not a snapshot. Publishing it to the snapshot repository would put release coordinates somewhere they are never promoted from." + exit 1 + ;; + esac + ;; + esac + + { + echo "version=$VERSION" + echo "channel=$INPUT_CHANNEL" + echo "dry_run=$INPUT_DRY_RUN" + } >> "$GITHUB_OUTPUT" + + - name: A release must be a commit that a matching tag points at + # The step that makes decision 1 true rather than merely documented. + # + # The version check above compares the tag name to the POMs, but only on the push + # path -- `github.ref_name` is a branch on a workflow_dispatch. So a manual run with + # channel=release and dry_run=false used to deploy any non-SNAPSHOT POM with no tag + # anywhere, which is exactly the "bytes that correspond to no commit" case the design + # claims to prevent. A documented guarantee with a manual bypass is not a guarantee. + # + # This is the cheap call: it fails before anything is built. The deploy job runs the + # same script again immediately before publishing, because that is the one that + # protects the upload -- see the comment there. + if: steps.resolve.outputs.channel == 'release' + env: + GH_TOKEN: ${{ github.token }} + run: | + # pipefail is load-bearing: without it the pipe into tee would return tee's status + # and the gate could never fail. + set -euo pipefail + scripts/check-release-tag.sh \ + "${{ github.repository }}" \ + "${{ steps.resolve.outputs.version }}" \ + "${{ github.sha }}" \ + | tee -a "$GITHUB_STEP_SUMMARY" + + - name: The engine version this release carries + # Read, never written: scripts/engine.properties is the pin, and hard-coding a version + # here would give it a second source of truth that drifts on the next baseline bump. + run: | + set -euo pipefail + ENGINE=$(sed -n 's/^engine.version=//p' scripts/engine.properties | head -1) + echo "engine $ENGINE, pinned by SHA-256 in scripts/engine.properties" >> "$GITHUB_STEP_SUMMARY" + + - name: The `build` workflow must have passed for this commit + # `build` filters on branches, so a tag push does not run it. Without this check a + # release could be cut from a commit whose test matrix was never green -- and the + # thing being published cannot be unpublished. + # + # Advisory for a snapshot: the point of a snapshot is to publish early, sometimes from + # a commit whose matrix is still running. + env: + GH_TOKEN: ${{ github.token }} + CHANNEL: ${{ steps.resolve.outputs.channel }} + run: | + set -euo pipefail + CONCLUSION=$(gh api \ + "repos/${{ github.repository }}/actions/runs?head_sha=${{ github.sha }}&per_page=100" \ + --jq '[.workflow_runs[] | select(.name == "build")] | first | .conclusion // "none"') + echo "build workflow for ${{ github.sha }}: $CONCLUSION" + if [ "$CONCLUSION" != "success" ]; then + if [ "$CHANNEL" = "release" ]; then + echo "::error::the build workflow for this commit concluded '$CONCLUSION'. Push the commit to a branch, let the matrix go green, then tag it." + exit 1 + fi + echo "::warning::the build workflow for this commit concluded '$CONCLUSION'; continuing because this is a snapshot." + fi + + # One job per platform, on its own runner, because build-native.sh refuses to cross-build: + # a shim linked for another architecture fails at System.load() in a user's JVM rather than + # here. This is the half of issue #15 that CI could already do and was discarding. + stage: + name: stage ${{ matrix.platform }} + needs: preflight + runs-on: ${{ matrix.runner }} + strategy: + # No fail-fast: knowing that three platforms staged and one did not is more useful than + # a cancelled matrix, and a release cannot proceed on three anyway. + fail-fast: false + matrix: + include: + - platform: linux-x86_64-gnu + runner: ubuntu-22.04 + - platform: linux-aarch64-gnu + runner: ubuntu-22.04-arm + - platform: macos-aarch64 + runner: macos-15 + # macos-15-intel, not macos-15: the plain label is Apple Silicon. This runner is the + # whole reason a release could not be assembled from one machine. + - platform: macos-x86_64 + runner: macos-15-intel + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-java@v4 + with: + # The floor, deliberately: building the shim against Java 11's jni.h and compiling + # the driver with Java 11's javac is what makes maven.compiler.release=11 a fact + # rather than a claim. `build` covers 17, 21 and 25 on every push; re-running the + # whole JDK matrix here would double a release's wall time to re-prove what + # preflight already confirmed was green for this commit. + distribution: temurin + java-version: "11" + cache: maven + + - name: Cache the pinned engine download + uses: actions/cache@v4 + with: + # Same key shape as `build`, so a release reuses the download that workflow already + # made for this commit instead of pulling ~326 MB per platform again. + path: target/engine/download + key: chdb-engine-${{ matrix.platform }}-${{ hashFiles('scripts/engine.properties') }} + + - name: Compile the Java side + # build-native.sh refuses to run without javac -h output, so a signature drift between + # Java and native is a compile error rather than a user's UnsatisfiedLinkError. + run: mvn $MAVEN_ARGS -pl chdb-jdbc -am compile + + - name: Fetch the engine, build the shim, stage the platform package + # Verifies the download's SHA-256 against scripts/engine.properties, checks the shim's + # $ORIGIN/@loader_path linkage, refuses an absolute build-machine path in its load + # commands, and holds the platform's version floor to its ceiling. Linux builds in + # manylinux_2_28 so the package keeps working on the RHEL 8 family. + run: | + case "${{ matrix.platform }}" in + linux-*) scripts/build-native-in-container.sh ${{ matrix.platform }} ;; + *) scripts/build-native.sh ${{ matrix.platform }} ;; + esac + + - name: Record what was staged + # The reproducibility answer in practice: not "these bytes are inevitable" but "these + # are the bytes, and here is their hash". Also lands in the JAR as + # META-INF/chdb/native///manifest.properties. + run: | + set -euo pipefail + case "${{ matrix.platform }}" in + macos-*) OS=macos ;; + *) OS=linux ;; + esac + case "${{ matrix.platform }}" in + *aarch64*) ARCH=aarch64 ;; + *) ARCH=x86_64 ;; + esac + DIR="chdb-native-${{ matrix.platform }}/target/native/META-INF/chdb/native/$OS/$ARCH" + { + echo "### ${{ matrix.platform }}" + echo '```' + cat "$DIR/sha256sums.txt" + grep -E '^(engine\.version|engine\.source|platform\.min\.|build\.commit)' \ + "$DIR/manifest.properties" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Integration tests against the staged package + # A release must not be assembled from a package that cannot run a query, even though + # preflight already checked `build` was green: the staged bytes here are new bytes. + run: | + set -euo pipefail + case "${{ matrix.platform }}" in + macos-*) OS=macos ;; + *) OS=linux ;; + esac + case "${{ matrix.platform }}" in + *aarch64*) ARCH=aarch64 ;; + *) ARCH=x86_64 ;; + esac + LIBS="$PWD/chdb-native-${{ matrix.platform }}/target/native/META-INF/chdb/native/$OS/$ARCH" + mvn $MAVEN_ARGS -pl chdb-integration-tests -am verify \ + -Dchdb.it.platform=${{ matrix.platform }} \ + -Dchdb.it.library.path="$LIBS" + + - name: Upload the staged platform package + uses: actions/upload-artifact@v4 + with: + name: native-${{ matrix.platform }} + path: chdb-native-${{ matrix.platform }}/target/native + # Everything the loader reads has to survive the round trip, so an empty upload must + # fail here rather than produce a JAR with no engine in it. + if-no-files-found: error + # Long enough to re-run a failed deploy against the same bytes, short enough not to + # leave ~600 MB per platform in the repository's artifact storage for a fortnight. + retention-days: 5 + + - name: Archive debug symbols + # Several times the size of the runtime library, so kept out of the JAR -- but a crash + # report from a released build is unreadable without them, and a release is exactly + # when they are worth keeping longer than a CI run. + if: always() + uses: actions/upload-artifact@v4 + with: + name: debug-symbols-${{ matrix.platform }} + path: target/debug-symbols/ + if-no-files-found: warn + retention-days: 90 + + # The job issue #15 exists for: four platforms in one working directory, on one machine. + deploy: + name: deploy (${{ needs.preflight.outputs.channel }}${{ needs.preflight.outputs.dry_run == 'true' && ', dry run' || '' }}) + needs: [preflight, stage] + runs-on: ubuntu-latest + # The first of two human gates. This environment is where CENTRAL_* and GPG_* live, so + # configuring required reviewers on it means nothing reaches Sonatype without someone + # saying so. The second gate is autoPublish=false: the bundle then waits in the portal. + environment: maven-central + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "11" + cache: maven + # Writes the `central` server entry the release profile's publishingServerId expects, + # and imports the signing key. Doing it here rather than by hand keeps the token and + # the key out of the workflow's own shell history. + server-id: central + server-username: CENTRAL_USERNAME + server-password: CENTRAL_PASSWORD + gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }} + # The *name* of the variable holding the passphrase, which is what maven-gpg-plugin + # reads. The release profile passes --pinentry-mode loopback so it never wants a tty. + gpg-passphrase: MAVEN_GPG_PASSPHRASE + + - name: Download every staged platform package + uses: actions/download-artifact@v4 + with: + pattern: native-* + path: /tmp/staged + + - name: Put each platform back where its module expects it + # Two things the artifact round trip does not preserve and the build depends on: + # + # - the executable bit. dlopen() does not require it, but a 0644 shared library in a + # released JAR is the kind of difference that shows up as one platform behaving + # oddly six months later, so it is restored rather than reasoned about. + # - nothing else: the layout under target/native is flat files, no symlinks. + run: | + set -euo pipefail + for platform in linux-x86_64-gnu linux-aarch64-gnu macos-aarch64 macos-x86_64; do + src="/tmp/staged/native-$platform" + dest="chdb-native-$platform/target/native" + [ -d "$src" ] || { echo "::error::no artifact for $platform"; exit 1; } + mkdir -p "$(dirname "$dest")" + rm -rf "$dest" + mv "$src" "$dest" + find "$dest" -type f \( -name '*.so' -o -name '*.dylib' \) -exec chmod 0755 {} + + done + + - name: Every platform must be complete, and none may be a local engine + # The release profile fails on both of these too (see the antrun execution in each + # native module). Repeated here because this job knows all four at once: a message + # naming which platform is short is worth more than the first module to fail, and it + # arrives before signing spends four minutes. + run: | + set -euo pipefail + total=0 + for spec in linux-x86_64-gnu:linux:x86_64:so \ + linux-aarch64-gnu:linux:aarch64:so \ + macos-aarch64:macos:aarch64:dylib \ + macos-x86_64:macos:x86_64:dylib; do + IFS=: read -r platform os arch jniext <<< "$spec" + dir="chdb-native-$platform/target/native/META-INF/chdb/native/$os/$arch" + for f in libchdb.so "libchdb_java_jni.$jniext" manifest.properties sha256sums.txt; do + [ -s "$dir/$f" ] || { echo "::error::$platform is missing $f"; exit 1; } + done + if grep -q '^engine.source=local' "$dir/manifest.properties"; then + echo "::error::$platform was staged from a --local-engine build, which skips the pinned checksum and ships no licence inventory" + exit 1 + fi + # The checksums the loader will verify at runtime, verified now, against the bytes + # that came back from the artifact store. This is the one thing on issue #10's list + # that only a real upload and download can exercise -- so exercise it. + ( cd "$dir" && sha256sum -c sha256sums.txt ) + size=$(du -sm "$dir" | cut -f1) + total=$((total + size)) + echo "$platform: ${size} MB" + done + echo "staged total: ${total} MB" + # ~1.3 GB staged, the same bytes again inside the JARs, plus sources, javadoc and + # signatures. Printed rather than guessed at, because running out of disk halfway + # through a deploy is the failure this job least wants. + df -h . + + - name: The tag still names this commit + # Deliberately the last thing before Maven runs, and deliberately the same check + # `preflight` already made. + # + # Between the two there is four-platform staging -- ten minutes and more -- and the + # `maven-central` environment's approval gate, which can hold a run for as long as it + # takes somebody to look at it. A tag can be force-moved or deleted inside that + # window, and nothing here would have noticed: the run would publish + # `github.sha` under a version whose tag now names something else, permanently, + # because a Central release cannot be unpublished. Checking once at the start + # describes the start, not the upload. + # + # This does not make the tag immutable, and is not a substitute for protecting the + # release tags so that it cannot move at all. It shrinks the window from "the whole + # run" to "the seconds between this step and the deploy". docs/release-readiness.md + # step 8 records the tag protection as the organisational half of the same guarantee. + if: needs.preflight.outputs.channel == 'release' + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + scripts/check-release-tag.sh \ + "${{ github.repository }}" \ + "${{ needs.preflight.outputs.version }}" \ + "${{ github.sha }}" + + - name: Package, sign and publish + env: + CENTRAL_USERNAME: ${{ secrets.CENTRAL_TOKEN_USERNAME }} + CENTRAL_PASSWORD: ${{ secrets.CENTRAL_TOKEN_PASSWORD }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + DRY_RUN: ${{ needs.preflight.outputs.dry_run }} + CHANNEL: ${{ needs.preflight.outputs.channel }} + VERSION: ${{ needs.preflight.outputs.version }} + # Modules named explicitly rather than building the whole reactor: chdb-examples and + # chdb-integration-tests are not published, and the integration suite has already run + # on each platform's own runner, where it could actually exercise that platform. + # + # `verify`, not `package`, for the dry run: signing is bound to verify, so `package` + # would silently produce no .asc files and prove nothing. + run: | + set -euo pipefail + GOAL=deploy + if [ "$DRY_RUN" = "true" ]; then GOAL=verify; fi + echo "mvn -Prelease $GOAL for $VERSION ($CHANNEL)" + mvn $MAVEN_ARGS -Prelease "$GOAL" \ + -pl .,chdb-bom,chdb-jdbc,chdb-native-macos-aarch64,chdb-native-macos-x86_64,chdb-native-linux-x86_64-gnu,chdb-native-linux-aarch64-gnu + + - name: What was produced + if: always() + env: + DRY_RUN: ${{ needs.preflight.outputs.dry_run }} + CHANNEL: ${{ needs.preflight.outputs.channel }} + VERSION: ${{ needs.preflight.outputs.version }} + run: | + set -euo pipefail + # ${DRY_RUN:+...} tests for non-empty, not for true, so it also expanded on the + # string "false" -- a real release printed "dry run: false" in its own heading. The + # person this summary is written for is about to decide whether to confirm an + # irreversible upload, so the heading has to say which of the two things happened + # and nothing else. + if [ "$DRY_RUN" = "true" ]; then + HEADING="### $VERSION ($CHANNEL, dry run -- nothing was uploaded)" + else + HEADING="### $VERSION ($CHANNEL)" + fi + { + echo "$HEADING" + echo '```' + find . -path ./target -prune -o -name '*.jar' -print -o -name '*.asc' -print \ + | sort | while read -r f; do printf '%9s %s\n' "$(du -h "$f" | cut -f1)" "$f"; done + echo '```' + if [ "$DRY_RUN" != "true" ] && [ "$CHANNEL" = "release" ]; then + echo "The bundle is uploaded but **not published**: autoPublish=false." + echo "Confirm it at -- a" + echo "release cannot be unpublished, so read the file list before pressing it." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Archive the signatures and POMs + # Not the JARs: they are 500 MB and already on their way to Central. The signatures and + # POMs are what you need to answer "what exactly did that release contain" later, and + # they are a few hundred kilobytes. + if: always() + uses: actions/upload-artifact@v4 + with: + name: release-metadata + path: | + **/target/*.asc + **/target/*.pom + **/target/native/META-INF/chdb/native/*/*/manifest.properties + **/target/native/META-INF/chdb/native/*/*/sha256sums.txt + if-no-files-found: warn + retention-days: 90 diff --git a/README.md b/README.md index 08be157..4d5b578 100644 --- a/README.md +++ b/README.md @@ -308,6 +308,8 @@ chdb-examples/ runnable examples scripts/engine.properties the pinned engine version and its checksums scripts/fetch-libchdb.sh downloads and verifies the pinned engine scripts/build-native.sh builds the shim and stages a platform package +scripts/verify-consumer.sh resolves the driver from a repository, outside this checkout +scripts/check-release-tag.sh refuses a release whose tag does not name the commit ``` ## Reporting a problem diff --git a/chdb-jdbc/src/main/java/org/chdb/internal/NativeLibraryLoader.java b/chdb-jdbc/src/main/java/org/chdb/internal/NativeLibraryLoader.java index 0539144..d48051b 100644 --- a/chdb-jdbc/src/main/java/org/chdb/internal/NativeLibraryLoader.java +++ b/chdb-jdbc/src/main/java/org/chdb/internal/NativeLibraryLoader.java @@ -14,8 +14,10 @@ import java.nio.file.attribute.PosixFilePermission; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; import java.util.EnumSet; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; @@ -278,6 +280,13 @@ private static LoadedRuntime loadFromJavaLibraryPath(Platform platform) { System.loadLibrary("chdb"); System.loadLibrary("chdb_java_jni"); } catch (UnsatisfiedLinkError e) { + // Naming the packages that *are* on the classpath, when they are for some other + // machine. Without it this message reads the same whether a consumer declared no + // platform package at all or declared the wrong one, and the second is the more + // likely mistake and the more confusing to read: they can see the dependency in + // their POM and are being told to add it. Measured by scripts/verify-consumer.sh, + // which reproduces both cases from a Maven repository. + String foreign = describeForeignPackages(platform); throw new ChdbNativeException( "No chDB native runtime for " + platform @@ -286,6 +295,7 @@ private static LoadedRuntime loadFromJavaLibraryPath(Platform platform) { + " 1. -D" + PROP_LIBRARY_PATH + " (not set)\n" + " 2. a chdb-native-" + platform.id() + " JAR on the classpath (not found)\n" + " 3. java.library.path (" + System.getProperty("java.library.path") + ")\n" + + foreign + "\nAdd the platform package for this machine:\n" + " \n" + " org.chdb\n" @@ -299,6 +309,33 @@ private static LoadedRuntime loadFromJavaLibraryPath(Platform platform) { return verify(platform, null, null, "java.library.path", new LinkedHashMap()); } + /** + * A sentence naming the platform packages on the classpath that are for another machine, or + * the empty string if there are none. + * + *

Never throws: this runs while building an error message, and a diagnostic that can + * fail turns a clear failure into an obscure one. + */ + private static String describeForeignPackages(Platform platform) { + try { + List foreign = new ArrayList<>(); + for (String id : NativePackage.foreignPackages(platform)) { + foreign.add("chdb-native-" + id); + } + if (foreign.isEmpty()) { + return ""; + } + return "\nThe classpath does carry " + + String.join(" and ", foreign) + + ", which " + + (foreign.size() == 1 ? "is" : "are") + + " for a different machine. A platform package is not interchangeable: it" + + " holds a shim linked for one architecture.\n"; + } catch (RuntimeException | LinkageError ignored) { + return ""; + } + } + private static LoadedRuntime loadPair( Platform platform, Path engine, Path jni, String source, Map manifest) { // Engine first: the shim's dependency on @rpath/libchdb.so is satisfied from the diff --git a/chdb-jdbc/src/main/java/org/chdb/internal/NativePackage.java b/chdb-jdbc/src/main/java/org/chdb/internal/NativePackage.java index e4a40ea..ca7c487 100644 --- a/chdb-jdbc/src/main/java/org/chdb/internal/NativePackage.java +++ b/chdb-jdbc/src/main/java/org/chdb/internal/NativePackage.java @@ -6,8 +6,10 @@ import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Properties; @@ -67,6 +69,28 @@ static NativePackage find(Platform platform) { return new NativePackage(platform, manifest, checksums); } + /** + * Platform package ids on the classpath that are for a machine other than this one. + * + *

{@link #find(Platform)} looks only for the current platform's resources, so a JAR for + * another architecture is indistinguishable from no JAR at all -- which makes the "no + * native runtime" message read identically whether a consumer declared nothing or declared + * the wrong package. This is how the two are told apart. Diagnostics only: nothing here + * makes a foreign package loadable. + */ + static List foreignPackages(Platform platform) { + List found = new ArrayList<>(); + for (String id : Platform.ALL_IDS) { + if (id.equals(platform.id())) { + continue; + } + if (loader().getResource(Platform.resourcePrefixFor(id) + "/manifest.properties") != null) { + found.add(id); + } + } + return found; + } + /** * Parses {@code sha256sums.txt}, which is in {@code sha256sum(1)} format: a hex digest, * whitespace, then the file name. diff --git a/chdb-jdbc/src/main/java/org/chdb/internal/Platform.java b/chdb-jdbc/src/main/java/org/chdb/internal/Platform.java index f7438cb..6470348 100644 --- a/chdb-jdbc/src/main/java/org/chdb/internal/Platform.java +++ b/chdb-jdbc/src/main/java/org/chdb/internal/Platform.java @@ -25,6 +25,13 @@ public final class Platform { public static final String LINUX_X86_64_GNU = "linux-x86_64-gnu"; public static final String LINUX_AARCH64_GNU = "linux-aarch64-gnu"; + /** + * Every platform id V1 ships. Only used for diagnostics -- {@link #current()} answers the + * question that matters -- but a message that can say "the package you declared is for + * another machine" needs to know what the other machines are called. + */ + static final String[] ALL_IDS = {MACOS_AARCH64, MACOS_X86_64, LINUX_X86_64_GNU, LINUX_AARCH64_GNU}; + private final String id; private final String os; private final String arch; @@ -85,6 +92,31 @@ public String resourcePrefix() { return "META-INF/chdb/native/" + os + "/" + arch; } + /** + * The same prefix, for a platform this JVM is not running on. + * + *

There is no {@code Platform} instance for another machine, on purpose: one exists only + * for the detected triple, so nothing can accidentally load a package for the wrong + * architecture. Diagnostics still need to look for those packages on the classpath, which + * is what this is for. + * + * @throws IllegalArgumentException if the id is not one of {@link #ALL_IDS} + */ + static String resourcePrefixFor(String platformId) { + switch (platformId) { + case MACOS_AARCH64: + return "META-INF/chdb/native/macos/aarch64"; + case MACOS_X86_64: + return "META-INF/chdb/native/macos/x86_64"; + case LINUX_X86_64_GNU: + return "META-INF/chdb/native/linux/x86_64"; + case LINUX_AARCH64_GNU: + return "META-INF/chdb/native/linux/aarch64"; + default: + throw new IllegalArgumentException("not a V1 platform id: " + platformId); + } + } + @Override public String toString() { return id + " (os=" + os + " arch=" + arch + " libc=" + libc + ")"; diff --git a/chdb-jdbc/src/test/java/org/chdb/internal/PlatformTest.java b/chdb-jdbc/src/test/java/org/chdb/internal/PlatformTest.java index e2b3416..8d8266d 100644 --- a/chdb-jdbc/src/test/java/org/chdb/internal/PlatformTest.java +++ b/chdb-jdbc/src/test/java/org/chdb/internal/PlatformTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.DisplayName; @@ -74,4 +75,26 @@ void currentPlatform() { // Detection is cached, so repeated calls must agree. assertEquals(platform.id(), Platform.current().id()); } + + @Test + @DisplayName("resourcePrefixFor covers every V1 id and agrees with the detected platform's own") + void resourcePrefixForEveryId() { + // Two ways of computing the same path -- one from the detected triple, one from an id + // for a machine this JVM is not -- and the diagnostic that tells a consumer they + // declared the wrong platform package depends on them matching. A new platform added + // to ALL_IDS without a case in the switch fails here rather than in a message. + assertEquals(4, Platform.ALL_IDS.length); + for (String id : Platform.ALL_IDS) { + String prefix = Platform.resourcePrefixFor(id); + assertTrue(prefix.startsWith("META-INF/chdb/native/"), prefix); + } + Platform platform = Platform.current(); + assertEquals(platform.resourcePrefix(), Platform.resourcePrefixFor(platform.id())); + } + + @Test + @DisplayName("resourcePrefixFor rejects an id that is not a V1 platform") + void resourcePrefixForRejectsUnknown() { + assertThrows(IllegalArgumentException.class, () -> Platform.resourcePrefixFor("windows-x86_64")); + } } diff --git a/docs/native-loading.md b/docs/native-loading.md index 0eca73b..def1cc9 100644 --- a/docs/native-loading.md +++ b/docs/native-loading.md @@ -120,6 +120,18 @@ Every load failure names what was tried and what to do. The ones you are most li **No platform package.** Lists all three locations that were searched and gives the Maven coordinate for your platform. +**The wrong platform package.** The same message, plus a line naming the `chdb-native-*` JARs +that *are* on the classpath. Without that line the two cases read identically, which is worst +for the more common one: someone looking at a `chdb-native-linux-x86_64-gnu` dependency in +their POM being told to add a platform package. + +Both were found by `scripts/verify-consumer.sh`, and both are still reproduced by it on every +push: it resolves the driver from a repository rather than from the reactor and then runs it +twice, once with `chdb-jdbc` alone and once with a platform package for another architecture. +It asserts on the text of each message — the coordinate the machine needs in both cases, and +the "classpath does carry" line in the second — so neither message can quietly stop saying +what this section says it says. `build` runs it on all four platforms. + **Unsupported platform.** Names the detected `os.name`/`os.arch`/libc and the supported set. musl Linux is detected specifically — via `/proc/self/maps`, falling back to `ldd --version` — and reported as unsupported rather than left to fail as a link error. diff --git a/docs/publishing.md b/docs/publishing.md index e179f4c..961edcd 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -16,7 +16,8 @@ needs one DNS record, and one licence question needs somebody with authority to | Publishing plugin | **done** — `central-publishing-maven-plugin`, `autoPublish=false` | | Third-party licence inventory | **done** — generated from the engine, shipped in the package, drift-tested | | `org.chdb` namespace | **not started** — needs a DNS TXT record, and there is no fallback | -| A way to stage all four platforms for one release | **not started** — issue #15 | +| A way to stage all four platforms for one release | **done** — `.github/workflows/release.yml`, issue #15 | +| Proof that the artifacts work when consumed from a repository | **done, locally** — `scripts/verify-consumer.sh`, issue #10 | | GPG key for the project | **not started** — needs a decision about whose key | | Position on the LGPL components | **not started** — needs chdb-io | @@ -42,6 +43,12 @@ mvn versions:set -DnewVersion=26.7.0.1 # a release, not the -SNAPSHOT in the mvn -Prelease deploy ``` +**In practice, do not do this by hand.** `.github/workflows/release.yml` does all of it, one +platform per runner, and is the only way to get macOS x86_64 without an Intel Mac. The manual +sequence above is kept because it is what the workflow runs, and because reading it is how you +understand what the workflow is for. Section 6 is the running order; `docs/release-readiness.md` +is the checklist. + **Staging is not optional and Maven does not do it.** The two shared libraries are put into `target/native/` by `scripts/build-native.sh`, and the native modules pick them up as a resource directory. Run `mvn -Prelease deploy` on a clean checkout and each native module packages an @@ -63,10 +70,38 @@ release profile rejects. purpose: a shim linked for another architecture fails at `System.load()` in a user's JVM rather than at build time. Linux is covered from either host by the container helper — given a Linux JDK to point `JAVA_HOME` at, which a macOS machine does not have lying around — but macOS -x86_64 needs an Intel Mac. CI already builds all four on their own runners and does not currently -upload the packaged jars, so assembling a release means either four machines or a release -workflow. That gap is issue #15 and should be closed before the first release rather than -worked around by hand. +x86_64 needs an Intel Mac. So assembling a release means either four machines or a release +workflow, and it is a workflow: `.github/workflows/release.yml` stages each platform on its own +runner, uploads `target/native/` as an artifact, and downloads all four into one deploy job +that runs `mvn -Prelease deploy`. That was issue #15. + +Three things about it that are decisions rather than mechanics, recorded at length in the +workflow's own header and summarised here: + +- **The version lives in git, not in the workflow.** A release is a commit that carries the + release version in its POMs plus a tag on that commit. `versions:set` in CI would publish + bytes that correspond to no commit, so `git show ` could not tell you what went out. + The workflow enforces this on **both** trigger paths: before anything is staged it looks up + `v` and refuses unless that tag exists and points at the commit being built. So + `workflow_dispatch` with `channel=release` needs the tag as much as a tag push does — an + earlier version of this workflow checked only the tag name in `github.ref_name`, which is a + branch on a manual run, and a manual release could therefore deploy an untagged commit. A + guarantee with a manual bypass is not one. +- **The same workflow publishes snapshots**, and that is the default for a manual run. It is + the only way to move issue #10, and running the release path for every snapshot is what makes + it already-tested when a release depends on it. The publishing plugin routes a `-SNAPSHOT` + deploy to Central's snapshot repository rather than building a portal bundle, so + `autoPublish` does not apply there. +- **The builds are not bit-identical, and the gap was measured rather than left open.** Two + clean builds of the macos-aarch64 shim from the same source on the same machine differ in + 116 bytes of the same-length file: 16 bytes of `LC_UUID`, four bytes holding the per-object + `N_OSO` debug-map timestamps, and ~96 bytes of ad-hoc code signature covering those. `__text`, + `__cstring`, `__const` and `__data` are byte-identical, so the shim's content reproduces and + only build metadata does not. Since the shim is ~130 KB of a ~326 MB package and the engine + half *is* reproducible — pinned by SHA-256 in `scripts/engine.properties` and verified by + `fetch-libchdb.sh` — chasing the remaining bytes would buy nothing a consumer can act on. + What the workflow does instead is record what it shipped: every staged library's SHA-256 goes + into the job summary and into `manifest.properties` inside the JAR. The version matters too: `deploy` on the `26.7.0.1-SNAPSHOT` currently in the POM publishes a snapshot, which goes to a different place and is never validated or promoted. A Central release @@ -315,12 +350,25 @@ Each step can invalidate the next, so: ``` - With both in place this is the cheap step that finds everything the reactor build cannot: - whether the POMs resolve from a repository, whether the BOM behaves, whether declaring one - platform package yields a working driver, whether a 166 MB artifact uploads. Tracked as + This used to be described as the step that finds everything the reactor build cannot. Most + of it can now be found locally, without publishing anything: + + ```bash + scripts/verify-consumer.sh + ``` + + That deploys to a throwaway `file://` repository and resolves from it in a project outside + this checkout, with its own empty local repository — so the POMs, the BOM import, the + transitive `chdb-jdbc` dependency, the unpack-from-JAR path and both "wrong platform + package" error messages are all exercised against a repository rather than a reactor. What + it cannot exercise is a network transfer: a `file://` deploy is a copy. **So the only thing + left for a real snapshot to prove is that a 98–166 MB artifact uploads and downloads + intact** — which is worth knowing, and is the cheapest possible way to know it. Tracked as issue #10. 4. Then the first real release, with `autoPublish=false` so the bundle is reviewed in the portal - before it becomes permanent. + before it becomes permanent. `.github/workflows/release.yml` does the assembly; the tag has + to be on a commit whose POMs already carry the release version, and whose `build` run was + green. ## 7. Who can do what diff --git a/docs/release-readiness.md b/docs/release-readiness.md index 2a93119..5582cc6 100644 --- a/docs/release-readiness.md +++ b/docs/release-readiness.md @@ -7,6 +7,135 @@ Written as three separate tracks because they proceed independently and have dif blockers: **signing and publishing** is mechanical and self-service, **the namespace** needs DNS access, and **licences** need a person with authority to say yes. +`docs/publishing.md` is the reference — the release profile, the namespace claim, the +publishing limits, the licence inventory — and is the only place any of those numbers live. +This file is the **running order and the state**: what is left, in what sequence, and how to +know each step worked. Where the two could disagree, this one points there rather than +restating. + +--- + +## Running order + +The tracks below say *what* and *who*. This says *when*, because each step can invalidate the +next. Eight steps. Four of them need somebody outside this repository — 1 needs DNS access to +`chdb.org`, 2 may need Sonatype support, 5 needs chdb-io, and 6 needs two projects willing to +depend on us — so those are the ones to start asking about early. Steps 3, 4, 7 and 8 are +self-service. + +### 1. Claim the `org.chdb` namespace — needs DNS access to `chdb.org` + +Register at , start a namespace claim for `org.chdb`, and hand +the TXT record Sonatype names to whoever administers the `chdb.org` zone. Verified +automatically, no human review, no queue. §1.3 has the detail, including why there is no +fallback. + +**Done when:** `org.chdb` shows as verified in the portal. + +### 2. Read the Usage Center, and ask for an exception if the numbers are tight + +Enforcement of the organisation-level publishing limits starts 1 October 2026. +`docs/publishing.md` §4 has our measured numbers and the text of the case to make. + +**Done when:** has been read, and **either** the +numbers there are comfortably inside the thresholds **or** Sonatype has granted an exception +that covers the numbers we actually publish. + +Not "an email has been sent", and not "Sonatype replied" — a refusal is a reply, and so is a +request for more detail. Either of those leaves the release liable to be rejected at upload +time, which is the thing this step exists to find out about in advance. If the answer is no, +that is a real finding and the next question is what to do about ~510 MB in a release month, +not whether to carry on. + +### 3. Decide whose GPG key signs, generate it, and load the CI secrets + +§1.2 for the decision, which is not technical. Then four secrets, in a GitHub environment named +`maven-central` so `.github/workflows/release.yml`'s deploy job can require a reviewer: + +| Secret | What | +|---|---| +| `GPG_PRIVATE_KEY` | the ASCII-armoured private key | +| `GPG_PASSPHRASE` | its passphrase | +| `CENTRAL_TOKEN_USERNAME` | portal token username | +| `CENTRAL_TOKEN_PASSWORD` | portal token password | + +**Done when:** `release.yml` run with `channel=snapshot` and `dry_run=true` produces a `.asc` +for every artifact. That exercises staging on all four platforms, signing and packaging, and +uploads nothing. + +### 4. Enable SNAPSHOTs for the namespace and publish a snapshot + +Snapshot publishing is opt-in per namespace — **Enable SNAPSHOTs** in the portal, or the upload +is rejected. Then `release.yml` with `channel=snapshot` and `dry_run=false`. + +Worth doing before any release polish. §1.5 says what it answers that nothing local can. + +**Done when:** the snapshot resolves from +`https://central.sonatype.com/repository/maven-snapshots/` in a project that is not this one. + +### 5. Get the licence position on the engine — needs chdb-io + +Track 2. Inherit the position taken for the existing PyPI and npm distributions of the same +`libchdb.so` rather than deriving one here. + +**Done when:** there is a written answer from someone who can give it, and whatever notice it +requires ships in `META-INF/licenses/`. + +### 6. Have two external projects consume the snapshot + +The V1 gate, and the reason step 4 comes before the release polish. Candidates worth asking: a +chDB cookbook example, the ADBC driver work, any internal tool that wants an embedded +analytical engine on the JVM. + +**Done when:** two projects outside this repository depend on a published coordinate, and what +broke is written down. The value of this gate is that list. + +### 7. Run the concurrent soak + +Work plan §5.11: one to six hours of *concurrent* querying. Blocking — see §3.1, which says why +a single-threaded run does not discharge it. + +**Done when:** the run is clean and the RSS/PSS slope over its length is written down. + +### 8. Cut the release + +```bash +# The version lives in git, not in CI: the tag has to name bytes a commit describes. +mvn versions:set -DnewVersion= -DgenerateBackupPoms=false +git commit -am "release " +git tag -a v -m "..." +git push origin main # let build.yml go green on the commit first +git push origin v +``` + +`release.yml` then stages each platform on its own runner, reassembles the four in one deploy +job, signs, and uploads a bundle that **waits in the portal**. Confirm it by hand at +: a release cannot be unpublished. + +Before it stages anything it refuses to continue unless the POMs are at a non-`SNAPSHOT` +version, a tag named `v` exists and points at the commit being built, and +`build` concluded successfully for that same commit. The tag check applies to a manual +`channel=release` run too, so there is no path that publishes an untagged commit, and it runs +a second time as the last step before Maven publishes. + +**Protect the release tags, once, before the first release.** The second check exists because +staging takes ten minutes and more and the `maven-central` approval gate can hold a run for +much longer, and a tag can be force-moved or deleted inside that window — so a run could +otherwise publish one commit under a version whose tag now names another, permanently. Checking +twice shrinks that window to the seconds between the last check and the upload; it does not +close it. Closing it is an organisational setting rather than a workflow one: + +- a tag protection rule on `v*` in the repository's rulesets, forbidding update and deletion, + which makes a release tag immutable once pushed; +- and, if the plan is releases from `main` only, restricting who can create those tags. + +Do this as part of step 3, when the `maven-central` environment is being set up, because it is +the same conversation with the same person. The workflow does not verify that the rule exists — +it cannot tell a protected tag from an unprotected one — so it is on this checklist rather than +in CI. + +Afterwards, the ordinary "back to development" commit returns the POMs to a `-SNAPSHOT`. + --- ## Track 1 — Signing and publishing mechanics @@ -14,22 +143,31 @@ DNS access, and **licences** need a person with authority to say yes. **Nobody's approval needed. All of it can be done by whoever holds the repository and one DNS record.** -### 1.1 What Maven Central requires that we do not have +### 1.1 What Maven Central requires, and where we stand -Checked against the current `main`: +Checked against the current `main`. An earlier version of this table listed all of the first +seven rows as absent; they were built in #13 and this is the state after it. | Requirement | Status | |---|---| -| `sources` JAR | declared in `pluginManagement` only, so **not produced** | -| `javadoc` JAR | same, **not produced** | -| GPG signature (`.asc`) per file | **absent** | -| `` in the POM | **absent**, and Central rejects without it | -| `` in the POM | **absent**, likewise | -| `` | present | -| ``, ``, `` | present | -| Publishing plugin and credentials | **absent** | - -None of this is hard; it is simply not started. It is roughly a day of work and one CI secret. +| `sources` JAR | **done** — a real one for `chdb-jdbc`, empty by construction for the native packages | +| `javadoc` JAR | **done**, same split | +| GPG signature (`.asc`) per file | **done**, verified end to end with a disposable key | +| `` in the POM | **done** | +| `` in the POM | **done** | +| `` | **done** | +| ``, ``, `` | **done** | +| Publishing plugin | **done** — `central-publishing-maven-plugin`, `autoPublish=false` | +| Third-party licence inventory | **done**, generated from the engine and drift-tested | +| Refusal to package an unstaged or `--local-engine` native module | **done** | +| A way to stage all four platforms for one release | **done** — §1.6 | +| Proof the artifacts work when consumed from a repository | **done locally** — §1.5 | +| Portal token in a CI secret | **not started** — running order step 3 | +| A GPG key that is the project's rather than a laptop's | **not started** — §1.2 | +| `org.chdb` namespace | **not started**, and the one item with no workaround — §1.3 | + +So the mechanics are finished and the remaining items in this track are two credentials and +one DNS record. `docs/publishing.md` §1 is the runbook for what the profile does. ### 1.2 The GPG key — self-service, but decide whose key it is @@ -50,9 +188,12 @@ means releases stop when that person is unavailable and the key follows them if project key shared through the CI secret store is the usual answer for an organisation, at the cost of the key being only as protected as the secret store. -Whichever is chosen, CI needs two secrets — the ASCII-armoured private key and its passphrase — -and the release job needs `maven-gpg-plugin` bound to `verify`. Signing locally on a laptop for -the first release is fine and is the faster path to finding out whether the rest works. +Whichever is chosen, CI needs two secrets — the ASCII-armoured private key and its passphrase. +`maven-gpg-plugin` is already bound to `verify` in the release profile, and +`.github/workflows/release.yml` imports the key through `setup-java`, so the only outstanding +part is the key itself and the four secrets in running-order step 3. Signing locally on a +laptop for the first release is fine and is the faster path to finding out whether the rest +works. ### 1.3 The namespace — needs DNS access, approved automatically @@ -64,70 +205,114 @@ Verification is by DNS. `chdb.org` resolves (13.248.169.48), so whoever administ adds a TXT record Sonatype names during registration, Sonatype's checker sees it, and the namespace is granted. No human review, no waiting on a queue. -If DNS access is awkward, `io.github.chdb-io` is verified by creating a repository whose name -is the verification code under the `chdb-io` GitHub org — no DNS involved. It is the fallback, -not the preference: `org.chdb` is the coordinate already in every POM, README and document, and -changing it later is a breaking change for consumers. +**There is no fallback, contrary to what an earlier version of this section said.** It offered +`io.github.chdb-io`, verified by creating a repository under the `chdb-io` GitHub org. That is +not available; Sonatype's wording, quoted in `docs/publishing.md` §3, is that only the GitHub +username used to sign up is supported, so `io.github.` cannot be registered +automatically. GitHub verification would grant `io.github.` — a personal +coordinate, and the wrong thing to publish an organisation's driver under. + +**So DNS access to `chdb.org` is a hard prerequisite**, not a convenience, and it is worth +confirming who has it before anything else in this document is started. `org.chdb` is also the +coordinate already in every POM, README and document; changing it later is a breaking change +for consumers. **Action:** register at , start the `org.chdb` namespace claim, and hand the TXT record to whoever runs `chdb.org` DNS. ### 1.4 Publishing limits — check the org's standing before the first release -Maven Central introduced organisation-level monthly limits on **file count**, **release size** -and **release count**, informational since 16 June 2026 and **enforced from 1 October 2026**. -The comparison is a rolling three-month average. Published percentile table: +Maven Central introduced organisation-level limits on **file count**, **release size** and +**release count**, informational since 16 June 2026 and **enforced from 1 October 2026**. -| Percentile | File count | Release size (MB) | Release count | -|---|---|---|---| -| 90th | 1,167 | 78 | 7 | -| 95th | 2,805 | 247 | 13 | -| 97th | 5,188 | 522 | 19 | -| 99th | 17,724 | 1,808 | 50 | +**The numbers, the percentile table and the case to make are in `docs/publishing.md` §4 and +only there.** An earlier version of this section reproduced them and got the measurement window +wrong — it described a rolling three-month average and then reasoned that a 2–3 month cadence +divides the release size by two or three. Sonatype is explicit that each metric resets at the +start of the month and is compared directly against that month's threshold, not averaged, so +a slower cadence does not make a release month cheaper; it makes the other months empty. Two +copies of a number is how that kind of error survives, which is why this file now keeps none of +them. -Sonatype has not said which percentile is the line; the Usage Center shows an organisation -where it stands. - -Our numbers, measured: - -- **Release size: 537 MB of native packages** (112 + 128 + 130 + 167) plus about 10 MB of - everything else, so **~547 MB per release**. -- **File count: roughly 100.** Six artifacts × (jar, sources, javadoc, pom) × (file + `.asc` + - `.sha1` + `.md5`). Nowhere near even the 90th percentile of 1,167. -- **Release count: whatever the cadence is.** - -Release size is the only metric in play, and cadence decides it because the comparison is a -three-month average: - -| Cadence | 3-month average | Roughly | -|---|---|---| -| Monthly | 547 MB | above the 97th percentile | -| **Every 2 months** | **274 MB** | **just above the 95th** | -| **Every 3 months** | **182 MB** | **between the 93rd and 94th** | - -At the 2–3 month cadence the other chDB bindings use, we land around the 93rd–95th percentile -on one of three metrics and near the median on the other two. That is a very different position -from the one a monthly cadence would put us in, and probably fine — but "probably" is doing work -there, because the actual threshold is not public. +The shape of the answer, from that section: release size is the only metric in play, we are +around the 97th percentile on it in a release month and near the floor on the other two, and +what enforcement actually looks for is sustained rather than occasional usage — which an +event-driven release every two to three months is not. **Action, and it is cheap:** log into the Usage Center at once the namespace exists and read the real numbers. If they are tight, email `central-support@sonatype.com` before the first release -rather than after a rejection, describing the pattern — four platform packages of an embedded -database engine, published every two to three months. Sonatype's own documentation says -exceptions are granted as a permanent adjustment once the pattern is understood. +rather than after a rejection, with what the exception form asks for; `docs/publishing.md` §4 +has the wording. Sonatype's own documentation says exceptions are granted as a permanent +adjustment once the pattern is understood. **One thing to confirm rather than assume:** from 1 October 2026, artifacts of a *commercial nature* require Maven Central Publisher Pro regardless of volume. Whether chDB counts is a question for whoever owns the relationship, not a technical one. -### 1.5 What to do first +### 1.5 What to do first, and what it is still for -Publish a **snapshot**, before any of the release polish. It costs an afternoon and is the only -thing that tests the parts nothing else can: whether the POMs resolve, whether the BOM does what -it should, whether a consumer declaring one platform package really gets a working driver, and -whether a 167 MB artifact uploads at all. Everything else in this track is easier to fix after -that has been seen to work. +Publish a **snapshot**, before any of the release polish. That has not changed. What has changed +is why: most of what a snapshot was going to find can now be found locally, in a few minutes, +without publishing anything. + +```bash +scripts/verify-consumer.sh +``` + +It deploys to a throwaway `file://` repository, builds a consumer project outside this checkout +with its own empty local repository, imports the BOM, declares exactly one platform package with +no version, and runs a query. It asserts that the POMs resolve from a repository, that the BOM +supplies the versions, that `chdb-jdbc` arrives transitively, that the libraries are unpacked +from the JAR rather than read from `target/`, that their checksums match what +`scripts/build-native.sh` staged, and that the "no platform package" and "wrong platform +package" errors each name the coordinate the machine actually needs. `build` runs it on all +four platforms, so the answer cannot go stale between releases. + +**What the gate found on its first run**, which is the part issue #10 asks for: + +- The BOM import and the transitive `chdb-jdbc` dependency were already right, and are now + asserted rather than assumed. +- **A snapshot deploy does not write the filenames a reactor build uses.** It writes + `chdb-jdbc---1.jar` plus a `maven-metadata.xml` that maps `-SNAPSHOT` onto + that timestamp; without the metadata a consumer gets a 404 for a version that is demonstrably + there. Nothing exercised that indirection before. The consumer's *local* repository then ends + up holding both names at once — the timestamped file it downloaded and a `-SNAPSHOT.jar` + beside it — so anything that looks for the driver by file name is depending on a resolver + detail. `scripts/verify-consumer.sh` locates it by coordinate, `org/chdb/chdb-jdbc/`, for + that reason. +- **"Declared the wrong platform package" was indistinguishable from "declared none".** Both + produced the same message — three locations searched, none found, here is the coordinate to + add. The advice is right either way, but somebody looking at + `chdb-native-linux-x86_64-gnu` in their own POM while being told to add a platform package + has no way to work out what they did wrong. The loader now names the `chdb-native-*` packages + that are on the classpath when they are for another machine. This was the one real defect the + gate found. +- **`central-publishing-maven-plugin` was configured with a parameter it does not have.** + `waitUntilValidated` is not a parameter of version 0.11.0; the mojo declares `waitUntil`, + `waitMaxTime`, `waitPollingInterval` and `waitForPublishCompletion`. Maven drops configuration + elements a mojo does not declare rather than failing on them, so it had been doing nothing. + Now `VALIDATED`, which is also the default — so the behaviour was + right by accident, and a release run reporting "validated" now means it. + +**What is left for a real snapshot** is one thing, and it is the reason to still do it early: +whether a 98–166 MB artifact survives an upload and a download over the network. A `file://` +deploy is a copy, so it cannot fail the way a transfer can. + +### 1.6 Assembling all four platforms — done, in CI + +A release needs all four platform packages staged in one working directory and no machine can +produce them: `scripts/build-native.sh` refuses to cross-build on purpose, and `macos-x86_64` +needs an Intel Mac. `.github/workflows/release.yml` stages each platform on its own runner, +uploads `target/native/` as an artifact, and downloads all four into one deploy job that runs +`mvn -Prelease deploy`. It publishes snapshots from the same jobs, so the release path is +exercised long before a release depends on it. + +The workflow's header records the three decisions behind it, including the measured answer on +build reproducibility: the engine half is reproducible because it is pinned by SHA-256, and the +shim's `__text`, `__cstring`, `__const` and `__data` reproduce byte for byte while `LC_UUID`, +four debug-map timestamps and the ad-hoc code signature do not. Nothing in the release gate +depends on that, and the workflow records the SHA-256 of everything it shipped instead. --- @@ -142,29 +327,24 @@ ours to give.** ### 2.2 The redistributed engine — inherited, not new -Each platform package ships `libchdb.so` verbatim from a chdb-core release. That library -statically links the ClickHouse tree, which carries **156 contrib submodules**. A scan of their -licence files: - -| Licence | Submodules | -|---|---| -| Apache-2.0 | 58 | -| MIT | 19 | -| LGPL | 7 | -| GPL | 8 | -| Boost, BSD, CC0, zlib, Unlicense | ~10 | - -**The GPL count needs reading carefully rather than reacting to.** Most are dual-licensed and -the permissive half is the one in force — `zstd` is BSD-3 *or* GPL-2, `rocksdb` is Apache-2.0 -*or* GPL-2, `liburing` is MIT *or* LGPL — and in `xz` the GPL covers the command-line tools -while `liblzma` itself is public domain. Several of the rest are not linked into a chDB build -at all. - -The ones that do warrant a look are the LGPL components with no permissive alternative: -`libssh`, `mariadb-connector-c`, `libgsasl`, `lemmagen-c`, and `libnuma` from `numactl`. LGPL -carries obligations under **static** linking that it does not under dynamic, and `libchdb.so` -links everything statically — we measured it: its only shared dependencies are `libc`, `libm`, -`libdl`, `librt`, `libpthread` and the loader. +Each platform package ships `libchdb.so` verbatim from a chdb-core release, and that library +statically links the ClickHouse tree. + +**The inventory of what is inside it, and the licence of each component, is in +`docs/publishing.md` §5 and only there.** An earlier version of this section counted 156 +contrib submodules by scanning their licence files and found seven LGPL and eight GPL. That +method has since been replaced by querying the binary we actually ship — +`SELECT DISTINCT library_name, license_type FROM system.licenses`, which ClickHouse generates +at build time — and the answer is 968 components, of which twelve carry a copyleft licence +with no permissive alternative, in two families rather than one. A filename scan calls every +dual-licensed component GPL and describes a source checkout rather than an artifact, so the +older numbers should not be reasoned from; the result is committed at +`licenses/engine-third-party-.tsv`, shipped in the package, and guarded by +`LicenseInventoryIT`, which fails if it stops matching the engine. + +What is unchanged, and is the point of this track: LGPL carries obligations under **static** +linking that it does not under dynamic, and `libchdb.so` links everything statically — its only +shared dependencies are `libc`, `libm`, `libdl`, `librt`, `libpthread` and the loader. **The decisive point: this is not a new question.** The identical `libchdb.so` is already redistributed on PyPI and npm by chdb-io. Whatever notice and offer-of-source obligations @@ -174,25 +354,17 @@ case the Java package is not where that gets discovered. So the task is **inherit and verify**, not analyse from scratch: 1. Ask chdb-core maintainers for the notice bundle their PyPI and npm releases ship, and what - position was taken on the LGPL components. If there is one, mirror it. + position was taken on the copyleft components. If there is one, mirror it. 2. If there is not, that is a finding about all three distributions and should be raised as such rather than solved inside this repository. -### 2.3 What we can produce mechanically - -ClickHouse ships `utils/list-licenses/list-licenses.sh`, which walks `contrib` and emits the -inventory. Run it against the exact engine tag and ship the output in the package: - -```bash -# in a chdb-core checkout at the pinned tag -utils/list-licenses/list-licenses.sh > /tmp/engine-licenses.txt -``` +### 2.3 What we already produce mechanically -`scripts/build-native.sh` already creates `META-INF/licenses/` with our licence and a README -pointing at the engine's release, and `META-INF/sbom/bom.json` in CycloneDX form listing both -libraries with their checksums. The gap is that the SBOM lists the engine as one component -rather than enumerating what is inside it. Folding in the generated inventory closes that, and -is half a day. +`scripts/build-native.sh` creates `META-INF/licenses/` with our licence, a README pointing at +the engine's release and the generated third-party inventory, plus `META-INF/sbom/bom.json` in +CycloneDX form listing both libraries with their checksums. This section previously described +the inventory as a gap and estimated half a day for it; it is done, and +`docs/publishing.md` §5 says how to regenerate it after an engine bump. ### 2.4 Who decides what @@ -201,21 +373,37 @@ is half a day. | Apache-2.0 on our own code | already done | no | | Generating and publishing a GPG key | any maintainer | no | | `org.chdb` namespace claim | whoever runs `chdb.org` DNS | automatic once the TXT record is up | -| Shipping the engine's licence inventory | any maintainer | no | -| **Whether redistributing `libchdb.so` under Apache-2.0 discharges the LGPL obligations** | **chdb-io / ClickHouse legal** | **yes, and it is the only real gate in this track** | +| Shipping the engine's licence inventory | already done | no | +| **Whether redistributing `libchdb.so` under Apache-2.0 discharges the obligations of the twelve copyleft components** | **chdb-io / ClickHouse legal** | **yes, and it is the only real gate in this track** | | Whether chDB counts as "commercial nature" for Central | whoever owns the Sonatype relationship | yes | --- ## Track 3 — What is still untested -Closing these does not need anyone's permission, only time. Ordered by what a first release -would most regret missing. +Two of these **block a release** and the rest do not, so they are in separate tables. An +earlier version of this section put the soak test in one table with the optional work and then +described the whole table as parallel and non-blocking, which let an operator read past a +required gate on the way to the sentence saying the gate was required. + +Everything in §3.2 needs only time and nobody's permission. Of the two in §3.1, the soak needs +only time; external consumption needs other people, which is why it is also running-order +step 6. + +### 3.1 Blocking — a release does not go out without these + +| Gate | What it requires | Effort | +|---|---|---| +| **Concurrent soak, 1–6 hours** | Work plan §5.11 asks for a *concurrent* soak, not a single-threaded loop: several connections on several threads for the whole window, with the RSS/PSS slope recorded across it. A sequential run measures allocator behaviour and nothing else. The milestone it is there to discharge is that cancel, timeout, early close and cascading `Connection` close "leak nothing and never deadlock" — and a deadlock cannot occur, so cannot be ruled out, in a single thread. Existing coverage is two proxies, each missing one half: the 1000-query RSS plateau is long but sequential, and `HikariPoolIT` is concurrent but minutes rather than hours. | a day, mostly waiting | +| **External consumption of a published artifact** | The gate says two projects outside this repository. `scripts/verify-consumer.sh` closes the mechanical half locally (§1.5); this is the half that needs other people, and running-order step 6 is where it sits. | depends on others | + +### 3.2 Not blocking — worth doing, in this order + +Ordered by what a first release would most regret missing. | Gap | Why it matters | Effort | |---|---|---| -| **Soak test, 1–6 hours** | An explicit gate. The 1000-query plateau is a proxy, not the thing. | a day, mostly waiting | -| **External consumption of a snapshot** | The gate says two external projects. Nothing has ever consumed this as a dependency. | depends on others | +| **Upload and download of a 98–166 MB artifact** | The one question on the consumption list that a `file://` repository cannot reach, because a copy cannot fail the way a transfer can. Comes free with running-order step 4. | free with the snapshot | | **OpenJ9 / Semeru smoke test** | The signal-handler guard is written against HotSpot's behaviour and has never met another JVM. | half a day | | **`noexec` /tmp** | The loader has a specific message for it that has never been executed. | an hour, in a container | | **Corrupted library, architecture mismatch** | Named in the plan; the checksum path is tested, these two are not. | half a day | @@ -223,31 +411,43 @@ would most regret missing. | **ShardingSphere, Spring `JdbcTemplate`** | HikariCP is done and found the useful thing. These two are lower yield. | half a day | | **The §3.3 batch-access benchmark** | The plan requires three approaches measured before the data path is fixed. One was chosen by reasoning. | a day | | **Full-process ASan and LSan** | Blocked on a sanitizer build of chdb-core. Not ours to close. | upstream | +| **Bit-identical shim builds** | Measured, and the answer is "no, by 116 bytes of build metadata" — `.github/workflows/release.yml`'s header has the breakdown. Nothing in the release gate depends on it. | not planned | ### What is already covered -253 tests across sixteen platform-and-JDK combinations, a 199-check native sanitizer harness, -UBSan over the whole suite, the full suite again on AlmaLinux 8 to demonstrate the glibc floor, -and a container test for the memory-limit gate. The type matrix, parameter binding, streaming -memory, cancellation, handle lifetime, the storage-path rule, five loader failure paths, -connection pooling, ClassLoader isolation and process exit behaviour all have tests that were -written to fail if the behaviour regressed — and several of them found real defects when first -run. +266 tests — 156 unit and 110 integration, counted from a run rather than estimated — across +sixteen platform-and-JDK combinations, a 199-check native sanitizer harness, UBSan over the +whole suite, the full suite again on AlmaLinux 8 to demonstrate the glibc floor, and a consumer +that resolves the driver from a repository rather than from the reactor. + +**The memory-limit gate is covered, and it is worth naming where** because it is not a JUnit +test and reading the test list alone would suggest otherwise. `scripts/run-memory-limit-test.sh` +runs a probe in an AlmaLinux 8 container under `docker --memory=2g`, with `-Xmx256m` and a +`groupArray` over 200 million values — so all three limits the work plan asks to be combined +(`-Xmx`, the cgroup limit and chDB's `max_memory_usage`) are in play at once. It runs both with +an explicit `max_memory_usage=200MB` and without one, and requires a catchable error code 241 +in both; a completed query, a different error, or a process that dies without reporting are all +failures. `.github/workflows/build.yml` invokes it in the `oldest-supported-linux` job on both +`linux-x86_64-gnu` and `linux-aarch64-gnu`, unconditionally, on every push. (`docs/v1-progress.md` +phase 10 still lists a "cgroup + `max_memory_usage` matrix" as outstanding; that line is stale, +not a second opinion.) + +The type matrix, parameter binding, streaming memory, cancellation, handle lifetime, the +storage-path rule, five loader failure paths, connection pooling — `HikariPoolIT`, a real +HikariCP 5.1.0 pool over the shared in-memory database — ClassLoader isolation and process exit +behaviour all have tests that were written to fail if the behaviour regressed, and several of +them found real defects when first run. --- ## The short answer -**Not ready, and the gating item is not code.** - -Three things have to happen in this order, because each can invalidate the next: - -1. **Confirm Maven Central will take it** — namespace claimed, Usage Center read, and an - exception arranged if the numbers are tight. Enforcement starts 1 October 2026. -2. **Get the licence position on the engine** from whoever owns it, inherited from the existing - PyPI and npm distributions rather than derived here. -3. **Publish a snapshot and have somebody consume it.** Everything else in Track 1 is easier to - finish once that has worked once. +**Not ready, and the gating items are not code.** Everything mechanical is finished: the release +profile, the signing, the licence inventory, the four-platform assembly, and a local proof that +the artifacts work when consumed from a repository rather than from the reactor. -Track 3 can proceed in parallel and does not block; the soak test is the only item there that a -first release would be embarrassed to skip. +What is left is a DNS record on `chdb.org`, a decision about whose key signs, a licence answer +from whoever owns the engine, two projects willing to depend on a snapshot, and a concurrent +soak. The **Running order** at the top of this file is the sequence, because each of those can +invalidate the next; §3.1 is the part of Track 3 that blocks, and §3.2 is the part that does +not. diff --git a/pom.xml b/pom.xml index bd85d3e..316526c 100644 --- a/pom.xml +++ b/pom.xml @@ -221,7 +221,19 @@ extra step, at least until several releases have gone out uneventfully. --> false - true + + VALIDATED diff --git a/scripts/check-release-tag.sh b/scripts/check-release-tag.sh new file mode 100755 index 0000000..aae717c --- /dev/null +++ b/scripts/check-release-tag.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# +# Answers one question: does the tag `v` exist, and does it point at ? +# +# A release is a commit that carries the release version in its POMs plus a tag on that commit, +# so that `git show ` can say what was published. This is what makes that true rather than +# documented, and .github/workflows/release.yml runs it twice on purpose: +# +# - in `preflight`, so a mistake costs nothing -- it fails before 1.3 GB of native libraries +# are built and moved across the network; +# - in `deploy`, immediately before `mvn -Prelease deploy`, because between the two there is +# four-platform staging (ten minutes and more) and a manual approval gate, and a tag can be +# force-moved or deleted inside that window. Publishing is irreversible; a check that ran a +# quarter of an hour before the upload does not describe the upload. +# +# Neither call replaces protecting the release tags so they cannot be moved at all. This closes +# the window down to the seconds between the check and the upload; only an immutable tag closes +# it entirely. See docs/release-readiness.md, running order step 8. +# +# Usage: +# scripts/check-release-tag.sh +# +# Needs `gh` authenticated for a read of the git ref namespace, and `jq`. Read-only: it makes +# GET requests and nothing else. + +set -euo pipefail + +fail() { + # Annotated when a workflow is reading, plain when a person is. + if [ -n "${GITHUB_ACTIONS:-}" ]; then + printf '::error::%s\n' "$*" + else + printf 'check-release-tag: %s\n' "$*" >&2 + fi + exit 1 +} + +REPO="${1:-}" +VERSION="${2:-}" +COMMIT="${3:-}" +if [ -z "$REPO" ] || [ -z "$VERSION" ] || [ -z "$COMMIT" ]; then + printf 'usage: %s \n' "$0" >&2 + exit 2 +fi + +TAG="v${VERSION}" + +# Through the API rather than git: actions/checkout fetches the ref that triggered the run and +# not the tag namespace, so `git rev-parse` in a workflow would report a tag that is simply not +# in the local clone as missing -- and would be wrong in the one direction that matters. +if ! REF=$(gh api "repos/${REPO}/git/ref/tags/${TAG}" 2>/dev/null); then + fail "no tag ${TAG} in ${REPO}. A release is a commit carrying the release version in its POMs plus a tag on that commit; create the tag first." +fi + +TYPE=$(printf '%s' "$REF" | jq -r '.object.type') +SHA=$(printf '%s' "$REF" | jq -r '.object.sha') + +# An annotated tag -- which is what `git tag -a` creates, and what the runbook in +# docs/release-readiness.md tells you to create -- has a ref pointing at a *tag object*, not at +# a commit. Without this hop the check would reject every release cut the recommended way. +if [ "$TYPE" = "tag" ]; then + SHA=$(gh api "repos/${REPO}/git/tags/${SHA}" --jq '.object.sha') +fi + +printf 'check-release-tag: %s -> %s (%s ref)\n' "$TAG" "$SHA" "$TYPE" +printf 'check-release-tag: this run -> %s\n' "$COMMIT" + +if [ "$SHA" != "$COMMIT" ]; then + fail "${TAG} points at ${SHA}, but this run is on ${COMMIT}. Release ${VERSION} from the commit its tag names, or move the tag and start again." +fi + +printf 'check-release-tag: %s names this commit\n' "$TAG" diff --git a/scripts/verify-consumer.sh b/scripts/verify-consumer.sh new file mode 100755 index 0000000..562b986 --- /dev/null +++ b/scripts/verify-consumer.sh @@ -0,0 +1,505 @@ +#!/usr/bin/env bash +# +# Consumes the driver the way a user does -- from a Maven repository, in a project that is not +# part of this build -- and asserts the things that only fail when consumed. +# +# Issue #10 is the release gate that says nothing has ever depended on this as a published +# artifact: every test to date builds it in the same reactor, where the POMs are read from +# target/ and the native libraries are found next to them. That hides five questions. This +# script answers four and a half of them, locally, in a few minutes, without publishing +# anything: +# +# 1. do the POMs resolve from a repository rather than from target/? -- yes, checked below +# 2. does the BOM do what a BOM should when imported? -- yes, checked below +# 3. does declaring exactly one platform package yield a working +# driver, and what is the error when someone declares none or +# the wrong one? -- checked below +# 4. does a 98-166 MB artifact upload and download intact? -- NOT covered here. +# This is the half that needs a real repository. `mvn deploy` to a file:// URL is a copy, +# not a transfer, so it cannot fail the way an upload can. The release workflow's deploy +# job verifies sha256sums.txt after the artifact round trip, which covers the CI half of +# it; only Central itself can cover the rest. +# 5. does the native extraction path behave when the JAR comes from a +# local repository rather than a build directory? -- yes, checked below +# +# How it works, and why it is arranged this way: +# +# `mvn deploy -DaltDeploymentRepository=...file://` writes a real repository layout -- +# POMs, maven-metadata.xml, checksums -- into a temporary directory. The consumer project +# then resolves from that directory with its own empty local repository, so nothing it gets +# can have come from this reactor or from the developer's ~/.m2. `-Dmaven.install.skip=true` +# keeps the ~100 MB platform JAR out of the real ~/.m2 as well. +# +# The consumer lives in a temporary directory outside this checkout on purpose. Put it in +# the reactor and Maven resolves the modules from the reactor, which is the exact thing this +# script exists to stop doing. +# +# Usage: +# scripts/verify-consumer.sh [platform-id] +# +# Defaults to this machine's platform, which must already be staged: +# mvn -pl chdb-jdbc compile && scripts/build-native.sh +# +# Needs network on first run: the consumer's empty local repository has to fetch Maven's own +# plugins. Everything under org.chdb comes from the file repository, and the script asserts it. +# +# What is isolated, and what is not: +# +# org.chdb artifacts isolated. The consumer's project repositories are the throwaway file +# repository and a disabled `central`, and every downloaded file's +# recorded provenance is checked afterwards. Nothing else may serve +# them, because a good copy on Central masking a bad one we just +# deployed is the single failure this script exists to detect. +# Maven's own plugins not isolated, on purpose. They come from the real Central through the +# inherited pluginRepositories. Vendoring them would make this script +# unrunnable for no gain: org.chdb is never a plugin dependency. +# the local repository overridden, not shared: -Dmaven.repo.local points at an empty +# directory, so nothing can be satisfied from a previous build. +# settings.xml not suppressed. A caller's mirrors and profiles still apply, and a +# * would intercept even the file repository -- so +# the provenance check fails loudly on such a machine rather than +# reporting a pass it cannot justify. Its message says to re-run with +# -s. Suppressing settings.xml outright would break the environments +# where that mirror is the only route to Central for the plugins above. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +die() { printf 'verify-consumer: %s\n' "$*" >&2; exit 1; } +step() { printf '\n\033[1m== %s\033[0m\n' "$*"; } +ok() { printf ' ok: %s\n' "$*"; } + +MVN_FLAGS=(--batch-mode --no-transfer-progress) + +# ---------------------------------------------------------------- platform + +case "$(uname -s)" in + Darwin) HOST_OS=macos ;; + Linux) HOST_OS=linux ;; + *) die "unsupported host OS $(uname -s)" ;; +esac +case "$(uname -m)" in + arm64|aarch64) HOST_ARCH=aarch64 ;; + x86_64|amd64) HOST_ARCH=x86_64 ;; + *) die "unsupported host CPU $(uname -m)" ;; +esac +DEFAULT_PLATFORM="${HOST_OS}-${HOST_ARCH}" +[ "$HOST_OS" = linux ] && DEFAULT_PLATFORM="${DEFAULT_PLATFORM}-gnu" + +PLATFORM="${1:-$DEFAULT_PLATFORM}" +case "$PLATFORM" in + macos-aarch64) OS=macos; ARCH=aarch64; JNIEXT=dylib ;; + macos-x86_64) OS=macos; ARCH=x86_64; JNIEXT=dylib ;; + linux-x86_64-gnu) OS=linux; ARCH=x86_64; JNIEXT=so ;; + linux-aarch64-gnu) OS=linux; ARCH=aarch64; JNIEXT=so ;; + *) die "unknown platform '${PLATFORM}'" ;; +esac +[ "$OS" = "$HOST_OS" ] && [ "$ARCH" = "$HOST_ARCH" ] \ + || die "cannot run ${PLATFORM} on ${HOST_OS}-${HOST_ARCH}: the point of this script is to run a query." + +MODULE="chdb-native-${PLATFORM}" +STAGED="${ROOT}/${MODULE}/target/native/META-INF/chdb/native/${OS}/${ARCH}" +[ -f "${STAGED}/libchdb.so" ] || die "${PLATFORM} is not staged (no ${STAGED}/libchdb.so). +Run: mvn -pl chdb-jdbc compile && scripts/build-native.sh ${PLATFORM}" + +# The other platform's coordinate, for the "declared the wrong one" case below. +case "$ARCH" in + aarch64) OTHER_ARCH=x86_64 ;; + *) OTHER_ARCH=aarch64 ;; +esac +OTHER_PLATFORM="${OS}-${OTHER_ARCH}" +[ "$OS" = linux ] && OTHER_PLATFORM="${OTHER_PLATFORM}-gnu" + +ENGINE_VERSION="$(sed -n 's/^engine.version=//p' "${SCRIPT_DIR}/engine.properties" | head -n1)" + +# Read from Maven rather than with sed, and read it first: it is the cheapest thing here that +# can fail, and an unparseable POM should not be discovered forty seconds into a deploy. +VERSION="$(cd "$ROOT" && mvn "${MVN_FLAGS[@]}" -q -DforceStdout help:evaluate -Dexpression=project.version)" \ + || die "could not read the project version; the POMs do not parse" + +# cd/pwd rather than mktemp's own answer: TMPDIR usually ends in a slash, so mktemp hands back +# a path with a doubled separator, and Maven reports the normalised form. The check below that +# every org.chdb artifact came from the consumer's own local repository compares paths as +# strings, and would fail on that difference alone. +WORK="$(cd "$(mktemp -d "${TMPDIR:-/tmp}/chdb-verify-consumer.XXXXXX")" && pwd)" + +# Kept on failure, removed on success. The interesting failures here are a Maven deploy or a +# dependency resolution that did not work, and both write their whole output to a log under +# $WORK -- which an unconditional `rm -rf` deleted a moment before the message telling you to +# go and read it. On success there is nothing in there worth 200 MB of disk. +cleanup() { + status=$? + if [ "$status" -eq 0 ]; then + rm -rf "$WORK" + else + printf '\nverify-consumer: left %s in place for diagnosis:\n' "$WORK" >&2 + for log in deploy.log resolve.log; do + [ -f "${WORK}/${log}" ] && printf ' %s\n' "${WORK}/${log}" >&2 + done + printf ' remove it when you are done.\n' >&2 + fi +} +trap cleanup EXIT +REPO="${WORK}/repository" +CONSUMER="${WORK}/consumer" +CONSUMER_M2="${WORK}/consumer-m2" +CACHE="${WORK}/native-cache" +mkdir -p "$REPO" "$CONSUMER/src" "$CONSUMER_M2" "$CACHE" + +printf 'platform: %s\n' "$PLATFORM" +printf 'engine: %s\n' "$ENGINE_VERSION" +printf 'version: %s\n' "$VERSION" +printf 'work: %s\n' "$WORK" + +# ---------------------------------------------------------------- 1. publish to a file repo + +step "Deploying to a throwaway file:// repository" + +# `deploy`, not `install`: it is the phase that writes a repository layout with +# maven-metadata.xml, which is what a consumer resolving a -SNAPSHOT actually reads. Install +# is skipped so a ~100 MB platform JAR does not land in the developer's real ~/.m2. +# +# Only the published modules: chdb-integration-tests and chdb-examples set +# maven.deploy.skip, and building them here would run the reactor tests this script is not +# about. +( + cd "$ROOT" + mvn "${MVN_FLAGS[@]}" deploy \ + -pl ".,chdb-bom,chdb-jdbc,${MODULE}" \ + -DskipTests \ + -Dmaven.install.skip=true \ + -DaltDeploymentRepository="chdb-verify::file://${REPO}" +) > "${WORK}/deploy.log" 2>&1 || { tail -40 "${WORK}/deploy.log"; die "deploy to the file repository failed; see ${WORK}/deploy.log"; } + +# Every artifact a consumer needs has to be in there, POM included -- a jar with no POM +# resolves for a direct dependency and then fails to bring in chdb-jdbc. +for coord in \ + "org/chdb/chdb-java-parent/${VERSION}" \ + "org/chdb/chdb-bom/${VERSION}" \ + "org/chdb/chdb-jdbc/${VERSION}" \ + "org/chdb/${MODULE}/${VERSION}"; do + [ -d "${REPO}/${coord}" ] || die "nothing published at ${coord}" + # A JAR with no POM resolves for a direct dependency and then quietly brings in nothing. + compgen -G "${REPO}/${coord}/*.pom" > /dev/null || die "no POM published at ${coord}" +done +ok "parent, BOM, driver and ${MODULE} are in the repository, each with a POM" + +case "$VERSION" in + *-SNAPSHOT) + # A snapshot deploy does not write chdb-jdbc--SNAPSHOT.jar. It writes + # chdb-jdbc-26.7.0.1-20260909.053707-1.jar and a maven-metadata.xml that maps -SNAPSHOT + # onto that timestamp, and a consumer cannot resolve the version without it. Nothing in + # the reactor exercises that indirection, which is the sort of thing issue #10 is about. + [ -f "${REPO}/org/chdb/chdb-jdbc/${VERSION}/maven-metadata.xml" ] \ + || die "no snapshot metadata for chdb-jdbc; a consumer could not resolve ${VERSION}" + ok "snapshot metadata maps ${VERSION} onto the timestamped filenames" + ;; +esac + +NATIVE_JAR="" +for candidate in "${REPO}/org/chdb/${MODULE}/${VERSION}/"*.jar; do + case "$candidate" in *-sources.jar|*-javadoc.jar) continue ;; esac + NATIVE_JAR="$candidate" + break +done +[ -n "$NATIVE_JAR" ] || die "no platform JAR published for ${MODULE}" +printf 'native jar: %s MB\n' "$(( $(wc -c < "$NATIVE_JAR") / 1024 / 1024 ))" + +# ---------------------------------------------------------------- 2. the consumer project + +step "Building a consumer project outside this checkout" + +# The shape the README tells a user to write: import the BOM, declare one platform package, +# name no versions. If the BOM does not manage the versions this will not resolve. +cat > "${CONSUMER}/pom.xml" < + + 4.0.0 + org.example + chdb-consumer + 1 + jar + + + + chdb-verify + file://${REPO} + true + truealways + + + + central + file://${WORK}/there-is-no-such-repository + false + false + + + + + + + + + org.chdb + chdb-bom + ${VERSION} + pom + import + + + + + + + + org.chdb + ${MODULE} + + + +EOF + +# A fresh, empty local repository, so nothing can be resolved from a previous build of this +# checkout. Maven's own plugins come from Central; everything under org.chdb must come from +# the file repository, which is asserted below. +( + cd "$CONSUMER" + mvn "${MVN_FLAGS[@]}" -Dmaven.repo.local="$CONSUMER_M2" \ + dependency:build-classpath -Dmdep.outputFile="${WORK}/cp.txt" +) > "${WORK}/resolve.log" 2>&1 || { tail -40 "${WORK}/resolve.log"; die "the consumer could not resolve its dependencies; see ${WORK}/resolve.log"; } + +CP="$(cat "${WORK}/cp.txt")" +ok "resolved with no version declared, so the imported BOM supplied it" + +# The point of the whole exercise: these bytes came out of a repository. +# Split on the classpath separator rather than piping into a loop: a `die` inside a pipeline +# runs in a subshell, so the failure would be reported and then ignored. +IFS=':' read -r -a CP_ENTRIES <<< "$CP" +for entry in "${CP_ENTRIES[@]}"; do + case "$entry" in + *org/chdb/*) + case "$entry" in + "${CONSUMER_M2}"/*) ;; + *) die "an org.chdb artifact resolved from outside the consumer's local repository: ${entry}" ;; + esac + ;; + esac +done + +# ...and out of *our* repository, which the check above cannot tell. Every path it accepts is +# under the consumer's local repository, and an artifact served by Central or by a corporate +# mirror lands there too. So the configuration is not trusted to be right; the result is read +# back. +# +# Maven writes _remote.repositories beside each downloaded file, recording the id of the +# repository that served it. Anything other than chdb-verify means this run did not test what +# it claims to: most likely the `central` block above stopped being disabled, or a +# * in the caller's settings.xml intercepted the file repository. Both are +# reasons to stop rather than to report a pass. +PROVENANCE_LINES=0 +while IFS= read -r record; do + while IFS= read -r line; do + case "$line" in + \#*|"") continue ;; + esac + PROVENANCE_LINES=$((PROVENANCE_LINES + 1)) + case "$line" in + *">chdb-verify="*) ;; + *) + die "an org.chdb artifact was served by a repository other than the throwaway one: + ${record} + ${line} +Only 'chdb-verify' proves the artifact under test is the one this run deployed. If a mirror in +your settings.xml claims '*', run this with -s pointing at a settings file that does not." + ;; + esac + done < "$record" +done < <(find "${CONSUMER_M2}/org/chdb" -name '_remote.repositories' -type f) +[ "$PROVENANCE_LINES" -gt 0 ] \ + || die "no _remote.repositories entries under ${CONSUMER_M2}/org/chdb: nothing was actually +downloaded, so this run proved nothing about resolving from a repository." +ok "all ${PROVENANCE_LINES} downloaded org.chdb files came from the throwaway repository, not from Central" +# Located by its repository path rather than by a file name, and this matters more than it +# looks. A snapshot exists under two names at once: the remote repository holds +# chdb-jdbc-26.7.2-rc.2.1-20260909.081309-1.jar, and Maven's local repository ends up with +# *both* that file and a chdb-jdbc-26.7.2-rc.2.1-SNAPSHOT.jar beside it -- measured, not +# assumed. Which of the two `dependency:build-classpath` emits is a resolver detail that this +# check has no business depending on, and matching "chdb-jdbc-${VERSION}.jar" quietly did: +# it happens to be the base-version name today, and a version scheme or a Maven upgrade that +# changed that would not fail this assertion, it would skip the two error paths below and +# still print "all consumer checks passed". +# +# org/chdb/chdb-jdbc/ is the coordinate. It cannot drift. +JDBC_ONLY="" +for entry in "${CP_ENTRIES[@]}"; do + case "$entry" in + */org/chdb/chdb-jdbc/*.jar) + [ -z "$JDBC_ONLY" ] || die "two chdb-jdbc jars on the consumer's classpath: + ${JDBC_ONLY} + ${entry}" + JDBC_ONLY="$entry" + ;; + esac +done +[ -n "$JDBC_ONLY" ] \ + || die "chdb-jdbc is not on the consumer's classpath; the platform package did not pull it in. +Classpath was: +$(printf '%s' "$CP" | tr ':' '\n' | sed 's/^/ /')" +[ -s "$JDBC_ONLY" ] || die "the resolved chdb-jdbc jar is missing or empty: ${JDBC_ONLY}" +ok "chdb-jdbc arrived transitively from ${MODULE}, from the local repository" +printf ' as: %s\n' "$(basename "$JDBC_ONLY")" + +# ---------------------------------------------------------------- 3. run a query + +cat > "${CONSUMER}/src/Consumer.java" <<'EOF' +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; + +/** + * The smallest thing a consumer does: get a connection, ask the engine its version, run a + * query. No Class.forName -- the driver has to be found through + * META-INF/services/java.sql.Driver in a JAR that came from a repository. + */ +public final class Consumer { + public static void main(String[] args) throws Exception { + try (Connection c = DriverManager.getConnection("jdbc:chdb::memory:")) { + System.out.println("engine=" + c.getMetaData().getDatabaseProductVersion()); + try (Statement s = c.createStatement(); + ResultSet rs = s.executeQuery( + "SELECT count(), sum(number) FROM numbers(1000)")) { + rs.next(); + System.out.println("rows=" + rs.getLong(1) + " sum=" + rs.getLong(2)); + } + } + } +} +EOF + +JAVAC="${JAVA_HOME:+$JAVA_HOME/bin/}javac" +JAVA="${JAVA_HOME:+$JAVA_HOME/bin/}java" +"$JAVAC" -cp "$CP" -d "${CONSUMER}/classes" "${CONSUMER}/src/Consumer.java" + +step "Running a query against the JAR from the local repository" + +# No -Dchdb.library.path: the loader must find the platform package on the classpath, unpack +# it into a content-addressed cache and verify the checksums. That is the production path and +# the one the reactor tests never take, because they point at target/native instead. +set +e +OUT="$("$JAVA" -cp "${CP}:${CONSUMER}/classes" \ + -Dchdb.cache.dir="$CACHE" \ + Consumer 2>&1)" +STATUS=$? +set -e +printf '%s\n' "$OUT" | sed 's/^/ | /' +[ $STATUS -eq 0 ] || die "the consumer could not run a query" + +printf '%s\n' "$OUT" | grep -q "engine=${ENGINE_VERSION}" \ + || die "the engine reported a version other than the pinned ${ENGINE_VERSION}" +printf '%s\n' "$OUT" | grep -q 'rows=1000 sum=499500' \ + || die "the query returned the wrong answer" +ok "query ran, engine reports the pinned ${ENGINE_VERSION}" + +# The extraction half of question 5: the libraries a consumer actually loaded are the ones the +# cache holds, and they are byte-identical to what build-native.sh staged. If the JAR round +# trip or the unpack had mangled anything, the loader's own checksum gate would have failed +# first -- this asserts the gate compared against the right bytes rather than trivially. +UNPACKED_ENGINE="$(find "$CACHE" -name libchdb.so -type f | head -n1)" +UNPACKED_JNI="$(find "$CACHE" -name "libchdb_java_jni.${JNIEXT}" -type f | head -n1)" +[ -n "$UNPACKED_ENGINE" ] || die "nothing was unpacked into ${CACHE}: the libraries did not come from the JAR" +sha_of() { + if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | cut -d' ' -f1 + else shasum -a 256 "$1" | cut -d' ' -f1; fi +} +for pair in "${UNPACKED_ENGINE}:libchdb.so" "${UNPACKED_JNI}:libchdb_java_jni.${JNIEXT}"; do + unpacked="${pair%%:*}"; name="${pair##*:}" + expected="$(awk -v n="$name" '$2 == n {print $1}' "${STAGED}/sha256sums.txt")" + actual="$(sha_of "$unpacked")" + [ "$expected" = "$actual" ] || die "${name} unpacked from the JAR differs from the staged file + staged: ${expected} + unpacked: ${actual}" +done +ok "both libraries were unpacked from the JAR into $(dirname "$UNPACKED_ENGINE" | sed "s|${WORK}|\$WORK|") and match the staged checksums" + +# ---------------------------------------------------------------- 4. the two error cases + +step "Declaring no platform package" + +# chdb-jdbc alone -- $JDBC_ONLY, found by coordinate above. This is the mistake a consumer +# makes when they copy a one-line dependency snippet, so the message has to name the coordinate +# for the machine they are on. +set +e +NONE_OUT="$("$JAVA" -cp "${JDBC_ONLY}:${CONSUMER}/classes" -Dchdb.cache.dir="$CACHE" Consumer 2>&1)" +NONE_STATUS=$? +set -e +[ $NONE_STATUS -ne 0 ] || die "a driver with no platform package ran a query, which it must not" +printf '%s\n' "$NONE_OUT" | grep -q "chdb-native-${PLATFORM}" \ + || die "the error does not name chdb-native-${PLATFORM}: +${NONE_OUT}" +printf '%s\n' "$NONE_OUT" | grep -q '' \ + || die "the error does not show the dependency to add" +ok "names chdb-native-${PLATFORM} and prints the dependency to add" +printf '%s\n' "$NONE_OUT" | grep -A12 'No chDB native runtime' | sed 's/^/ | /' + +step "Declaring the wrong platform package" + +# The platform package for the *other* architecture, simulated by overriding os.arch rather +# than by staging a second platform -- Platform.detect() reads that property, so the loader +# takes exactly the branch it would on a real ${OTHER_PLATFORM} machine with a +# chdb-native-${PLATFORM} JAR on its classpath. +set +e +WRONG_OUT="$("$JAVA" -cp "${CP}:${CONSUMER}/classes" \ + -Dos.arch="${OTHER_ARCH}" -Dchdb.cache.dir="$CACHE" Consumer 2>&1)" +WRONG_STATUS=$? +set -e +[ $WRONG_STATUS -ne 0 ] || die "the driver loaded a package for the wrong architecture" +printf '%s\n' "$WRONG_OUT" | grep -q "chdb-native-${OTHER_PLATFORM}" \ + || die "the error does not name the package this machine needs (chdb-native-${OTHER_PLATFORM}): +${WRONG_OUT}" +# The distinguishing half. Without it this message is identical to the "declared nothing" +# one, and a consumer looking at chdb-native-${PLATFORM} in their POM is told to add a +# dependency they can see is already there. +printf '%s\n' "$WRONG_OUT" | grep -q "The classpath does carry chdb-native-${PLATFORM}" \ + || die "the error does not mention that chdb-native-${PLATFORM} is on the classpath, so it +reads exactly like the no-package-at-all case: +${WRONG_OUT}" +ok "names chdb-native-${OTHER_PLATFORM}, and says which package is on the classpath instead" +printf '%s\n' "$WRONG_OUT" | grep -A6 'No chDB native runtime' | sed 's/^/ | /' + +printf '\n\033[1mAll consumer checks passed for %s at %s.\033[0m\n' "$PLATFORM" "$VERSION" +printf 'Not covered here: whether a %s MB artifact survives a real upload and download.\n' \ + "$(( $(wc -c < "$NATIVE_JAR") / 1024 / 1024 ))" +printf 'Only Maven Central can answer that; see issue #10 and docs/release-readiness.md.\n'