diff --git a/.github/workflows/dist-smoke.yml b/.github/workflows/dist-smoke.yml new file mode 100644 index 00000000..1b8bd6a5 --- /dev/null +++ b/.github/workflows/dist-smoke.yml @@ -0,0 +1,142 @@ +name: Dist Smoke Test + +on: + # Manual trigger only — this is a release-engineering tool, not a per-PR + # gate. A maintainer runs it against a release-candidate branch to verify + # the wheel that will be published actually serves a working login page + # through the real klangkd → caddy → UDS → uvicorn path before tagging + # (#1611). Trigger via the Actions UI or `gh workflow run dist-smoke.yml + # --ref `. + workflow_dispatch: + +concurrency: + group: dist-smoke + cancel-in-progress: false + +jobs: + # Build the wheel (#1656). No publish here — this is a release-candidate + # validation workflow, not a release; the wheel is uploaded as a workflow + # artifact for dist-smoke-test to install + exercise. release.yml owns + # the actual PyPI publish. + # + # The distribution name is ``klangk`` (one unified wheel ships the klangkd + # server + klangk CLI, #1606); hatch-vcs derives the version from git + # history (hence the fetch-depth: 0 below). + build-wheel: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 # hatch-vcs needs full history to derive the version + + - uses: ./.github/actions/devenv-setup + with: + # Build the frontend with the default plugin set (checked-in + # plugins.yaml, #1660) so the wheel's klangk/frontend/ carries + # the compiled UI + features.json. The hatch build hook + # (hatch_build_frontend.py) force-includes it and *requires* it + # for non-editable wheel builds (#1600). + build-tasks: klangk:flutter-build + + - name: Build wheel + run: | + . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh + # scripts/build_wheel.sh installs python-build transiently (uv-sync + # at shell entry would wipe it; the script installs + builds in one + # process). Requires the frontend artifact from klangk:flutter-build + # above — the hatch build hook force-includes it and requires it for + # non-editable wheel builds (#1600). + devenv shell -- bash scripts/build_wheel.sh + + - name: Upload wheel artifact + uses: actions/upload-artifact@v7 + with: + name: klangk-wheel + path: src/klangk/dist/*.whl + retention-days: 7 + + # Smoke test the installed wheel OUTSIDE devenv (#1611). The whole point + # is to prove a bare `pip install klangk && klangkd` works on a stock + # Ubuntu runner with no devenv/nix in the loop — that's the audience + # the #1607 / #1645 first-run story targets. Runs apt-get install for + # caddy (klangkd forks it as a child via shutil.which) + the Playwright + # Chromium browser deps; pip-installs the wheel into an isolated venv; + # starts real klangkd (caddy engine, UDS upstream, Flutter Web frontend + # served from the wheel's bundled klangk/frontend/); runs the one-file + # dist-smoke Playwright spec against it. + # + # What this catches: + # - Frontend not included in wheel → 404 / blank page + # - _DEFAULT_FRONTEND_DIR resolves wrong post-install + # - Flutter build broken / incomplete → flutter-view never attaches + # - main.dart.js missing or corrupt → engine doesn't boot + # - klangkd entrypoint registration broken → server never starts + # - caddy binary not found → engine refuses to start + # - Caddyfile render broken → caddy rejects the config + # - UDS upstream misconfigured → caddy 502s + # - Missing transitive dep in pyproject.toml → import fails fast + # + # NOTE: currently expected to FAIL until #1709 (klangkd's caddy child + # can't coexist with another caddy on the host / bootstrap binds 2019 + # instead of the UDS) is fixed. The dist-smoke gate is intentionally + # decoupled from release.yml — release.yml publishes without waiting + # on this workflow so the release pipeline isn't held hostage to the + # smoke being green. A maintainer runs this against a release-candidate + # branch before tagging; if it fails on a non-#1709 cause, that's a + # real release-blocker worth investigating before tagging. + dist-smoke-test: + runs-on: ubuntu-latest + timeout-minutes: 15 + needs: build-wheel + steps: + - uses: actions/checkout@v7 + + - name: Install caddy + Python via apt + # Stock Ubuntu packages — no devenv, no nix. Caddy is klangkd's + # default proxy engine child (KLANGK_PROXY_BIN overrides if needed, + # but the default shutil.which("caddy") → /usr/bin/caddy path is + # exactly what apt installs). The Playwright browser runtime libs + # (libnss3, libgbm1, libasound2t64, …) are NOT installed here — + # scripts/dist-smoke-test.sh runs `npx playwright install --with-deps + # chromium`, which resolves the correct package names per-OS. Hand- + # maintaining the lib list broke on the Ubuntu 24.04 t64 transition + # (libasound2 → libasound2t64, libatk1.0-0 → libatk1.0-0t64, …). + # + # MASK caddy.service: the Debian/Ubuntu caddy package ships a systemd + # unit that autostarts on install and binds the default admin port + # 127.0.0.1:2019. klangkd owns its own caddy child process + # (CADDY_ADMIN=unix//), but caddy's initial config-load still + # tries to bind 2019 before applying the new admin address — so the + # system service sitting on 2019 makes every klangkd-managed caddy + # fail to start ("bind: address already in use"). Mask the unit so + # the apt-installed caddy is just a binary on PATH, which is what + # klangkd's contract expects. (This is the #1709 mode-1 workaround; + # mode-2 still fails and is the open bug.) + run: | + sudo apt-get update + sudo apt-get install -y caddy curl python3 python3-venv + sudo systemctl stop caddy.service 2>/dev/null || true + sudo systemctl disable caddy.service 2>/dev/null || true + sudo systemctl mask caddy.service + + - name: Download wheel artifact + uses: actions/download-artifact@v7 + with: + name: klangk-wheel + path: wheel-cache + + - name: Run dist-smoke test + run: bash scripts/dist-smoke-test.sh "$(echo wheel-cache/*.whl)" + + - name: Upload smoke logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: dist-smoke-logs + retention-days: 7 + path: | + /tmp/klangk-smoke-state/ + /tmp/klangk-smoke-data/ + src/frontend/e2e-tests/test-results/ + src/frontend/e2e-tests/logs/ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c09f8507..86847d36 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,197 +4,28 @@ on: push: tags: - "v[0-9]*" - # Manual trigger for running the dist-smoke test against a wheel built - # from the current branch WITHOUT publishing anything (#1611). Dispatch - # runs build-wheel → dist-smoke-test; publish-and-release is gated on - # `github.event_name == 'push'` so a manual run stops at the smoke gate. - # Use this to validate a release candidate before tagging — `gh workflow - # run release.yml` from the branch, or the Actions UI. - workflow_dispatch: concurrency: group: release cancel-in-progress: false jobs: - # Build the klangk wheel (#1656). No publish here — the wheel is uploaded - # as a workflow artifact for dist-smoke-test to install + exercise, and - # PyPI publish is gated on the smoke test passing (publish-and-release - # below). Pre-#1705 the wheel built + published in one job in parallel - # with the image build; the smoke gate serializes that and holds PyPI - # until the smoke passes so a broken wheel never reaches the index — - # no yank ever needed. - # - # The distribution name is ``klangk`` (one unified wheel ships the klangkd - # server + klangk CLI, #1606); the tag is ``vX.Y.Z``; hatch-vcs derives - # the version from it. - build-wheel: - runs-on: ubuntu-latest - timeout-minutes: 30 - permissions: - id-token: write # unused here; kept for symmetry with publish-and-release - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 # hatch-vcs needs full history to derive the version - - - uses: ./.github/actions/devenv-setup - with: - # Build the frontend with the default plugin set (checked-in - # plugins.yaml, #1660) so the wheel's klangk/frontend/ carries - # the compiled UI + features.json. The hatch build hook - # (hatch_build_frontend.py) force-includes it and *requires* it - # for non-editable wheel builds (#1600). - build-tasks: klangk:flutter-build - - - name: Build wheel - run: | - . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh - # scripts/build_wheel.sh installs python-build transiently (uv-sync - # at shell entry would wipe it; the script installs + builds in one - # process). Requires the frontend artifact from klangk:flutter-build - # above — the hatch build hook force-includes it and requires it for - # non-editable wheel builds (#1600). - devenv shell -- bash scripts/build_wheel.sh - - - name: Upload wheel artifact - uses: actions/upload-artifact@v7 - with: - name: klangk-wheel - path: src/klangk/dist/*.whl - retention-days: 30 - - # Smoke test the installed wheel OUTSIDE devenv (#1611). The whole point - # is to prove a bare `pip install klangk && klangkd` works on a stock - # Ubuntu runner with no devenv/nix in the loop — that's the audience - # the #1607 / #1645 first-run story targets. Runs apt-get install for - # caddy (klangkd forks it as a child via shutil.which) and the Playwright - # Chromium browser; pip-installs the wheel into an isolated venv; starts - # real klangkd (caddy engine, UDS upstream, Flutter Web frontend served - # from the wheel's bundled klangk/frontend/); runs the one-file dist-smoke - # Playwright spec against it. - # - # What this catches before anything publishes: - # - Frontend not included in wheel → 404 / blank page - # - _DEFAULT_FRONTEND_DIR resolves wrong post-install - # - Flutter build broken / incomplete → flutter-view never attaches - # - main.dart.js missing or corrupt → engine doesn't boot - # - klangkd entrypoint registration broken → server never starts - # - caddy binary not found → engine refuses to start - # - Caddyfile render broken → caddy rejects the config - # - UDS upstream misconfigured → caddy 502s - # - Missing transitive dep in pyproject.toml → import fails fast - dist-smoke-test: - runs-on: ubuntu-latest - timeout-minutes: 15 - needs: build-wheel - steps: - - uses: actions/checkout@v7 - - - name: Install caddy + Python via apt - # Stock Ubuntu packages — no devenv, no nix. Caddy is klangkd's - # default proxy engine child (KLANGK_PROXY_BIN overrides if needed, - # but the default shutil.which("caddy") → /usr/bin/caddy path is - # exactly what apt installs). The Playwright browser runtime libs - # (libnss3, libgbm1, libasound2t64, …) are NOT installed here — - # scripts/dist-smoke-test.sh runs `npx playwright install --with-deps - # chromium`, which resolves the correct package names per-OS. Hand- - # maintaining the lib list broke on the Ubuntu 24.04 t64 transition - # (libasound2 → libasound2t64, libatk1.0-0 → libatk1.0-0t64, …). - # - # MASK caddy.service: the Debian/Ubuntu caddy package ships a systemd - # unit that autostarts on install and binds the default admin port - # 127.0.0.1:2019. klangkd owns its own caddy child process - # (CADDY_ADMIN=unix//), but caddy's initial config-load still - # tries to bind 2019 before applying the new admin address — so the - # system service sitting on 2019 makes every klangkd-managed caddy - # fail to start ("bind: address already in use"). Mask the unit so - # the apt-installed caddy is just a binary on PATH, which is what - # klangkd's contract expects. - run: | - sudo apt-get update - sudo apt-get install -y caddy curl python3 python3-venv - sudo systemctl stop caddy.service 2>/dev/null || true - sudo systemctl disable caddy.service 2>/dev/null || true - sudo systemctl mask caddy.service - - - name: Download wheel artifact - uses: actions/download-artifact@v7 - with: - name: klangk-wheel - path: wheel-cache - - - name: Run dist-smoke test - run: bash scripts/dist-smoke-test.sh "$(echo wheel-cache/*.whl)" - - - name: Upload smoke logs on failure - if: failure() - uses: actions/upload-artifact@v7 - with: - name: dist-smoke-logs - retention-days: 7 - path: | - /tmp/klangk-smoke-state/ - /tmp/klangk-smoke-data/ - src/frontend/e2e-tests/test-results/ - src/frontend/e2e-tests/logs/ - - # Publish + release. Runs only after dist-smoke-test passes: PyPI publish, - # GHCR image push, and the GitHub Release all happen here. Gating PyPI on - # the smoke (vs. publishing in parallel and yanking on failure) means a - # broken wheel never reaches the index — there is no yank path and no need - # for one. The host image build (the slow step, ~25-30 min) runs in this - # same job, before the pushes; it can't easily be split into a parallel - # job because GitHub Actions jobs run on separate runners and the built - # images can't transfer without an explicit save/load step that costs - # roughly the time the parallelism would save (the host image embeds the - # workspace tarball, multi-GB). Tag-push-only frequency makes the serial - # SLO acceptable. - publish-and-release: + build-and-release: runs-on: ubuntu-latest timeout-minutes: 45 - needs: dist-smoke-test - # Skip on workflow_dispatch — a manual run is for validating the smoke - # gate, not for actually publishing. Only a v* tag push runs the full - # release. Lets a maintainer run `gh workflow run release.yml` from a - # release-candidate branch to verify build-wheel + dist-smoke pass - # before tagging. - if: github.event_name == 'push' - environment: pypi # PyPI trusted publisher binds to this environment name permissions: packages: write contents: write - id-token: write # OIDC attestation for trusted PyPI publishing steps: - uses: actions/checkout@v7 - with: - fetch-depth: 0 # hatch-vcs / changelog extraction both need history - uses: ./.github/actions/devenv-setup - - name: Download wheel artifact - uses: actions/download-artifact@v7 - with: - name: klangk-wheel - path: wheel-dist - - name: Build host image run: | . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh devenv shell -- build-host-image - - name: Publish wheel to PyPI - # Trusted publishing (OIDC) — no API token. pypa/gh-action-pypi-publish - # negotiates an OIDC token from GitHub Actions and presents it to - # PyPI, which validates it against the trusted-publisher config on - # the `klangk` project (environment: pypi, workflow: release.yml). - # Runs after the smoke test has passed, so a broken wheel never - # reaches PyPI — no yank path exists. - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages-dir: wheel-dist - skip-existing: true # idempotent on re-runs of the same tag - - name: Push images to GHCR run: | echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin @@ -245,3 +76,68 @@ jobs: fi env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Build + publish the klangk wheel to PyPI via trusted publishing (OIDC, + # no API token — same shape as the deleted cli-publish.yml pre-#1606). + # Runs in parallel with build-and-release — the wheel doesn't need + # podman/docker, and pip/uv users (the #1607 / #1645 first-run audience) + # get an installable artifact via `pip install klangk==`. The + # distribution name is ``klangk`` (one unified wheel ships the klangkd + # server + klangk CLI, #1606); the tag is ``vX.Y.Z``; hatch-vcs derives + # the version from it. + # + # Pre-#1705 the wheel build + publish lived in this same job; #1705/#1707/ + # #1708 split it into a serial chain through a dist-smoke-test gate. The + # smoke gate is now its own workflow (.github/workflows/dist-smoke.yml) + # that does NOT block release — the smoke is a release-engineering tool + # the release manager runs against a release-candidate branch before + # tagging, not a CI gate. Releases publish without waiting on the smoke + # so the pipeline isn't held hostage to the smoke being green (it's + # currently red on #1709, a klangkd caddy bug, not a wheel-packaging + # bug). Run dist-smoke.yml manually before tagging to validate a + # release candidate. + # + # Trusted-publishing prereq (PyPI side, one-time): the `klangk` PyPI + # project must have a trusted publisher configured for this repo + + # workflow + environment (mcdonc/klangk, .github/workflows/release.yml, + # environment name: pypi). With that in place PyPI validates the OIDC + # token from GitHub Actions and publishes with no secret on our side. + build-wheel: + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: pypi # PyPI trusted publisher binds to this environment name + permissions: + id-token: write # OIDC attestation — the only permission trusted publishing needs + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 # hatch-vcs needs full history to derive the version + + - uses: ./.github/actions/devenv-setup + with: + # Build the frontend with the default plugin set (checked-in + # plugins.yaml, #1660) so the wheel's klangk/frontend/ carries + # the compiled UI + features.json. The hatch build hook + # (hatch_build_frontend.py) force-includes it and *requires* it + # for non-editable wheel builds (#1600). + build-tasks: klangk:flutter-build + + - name: Build wheel + run: | + . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh + # scripts/build_wheel.sh installs python-build transiently (uv-sync + # at shell entry would wipe it; the script installs + builds in one + # process). Requires the frontend artifact from klangk:flutter-build + # above — the hatch build hook force-includes it and requires it for + # non-editable wheel builds (#1600). + devenv shell -- bash scripts/build_wheel.sh + + - name: Publish to PyPI + # Trusted publishing (OIDC) — no API token. pypa/gh-action-pypi-publish + # negotiates an OIDC token from GitHub Actions and presents it to + # PyPI, which validates it against the trusted-publisher config on + # the `klangk` project (environment: pypi, workflow: release.yml). + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: src/klangk/dist + skip-existing: true # idempotent on re-runs of the same tag diff --git a/scripts/dist-smoke-test.sh b/scripts/dist-smoke-test.sh index 58b546f0..a685139f 100755 --- a/scripts/dist-smoke-test.sh +++ b/scripts/dist-smoke-test.sh @@ -50,9 +50,10 @@ # devenv shell -- bash scripts/flutterbuildweb.sh # produce the frontend # devenv shell -- bash scripts/build_wheel.sh # produce the wheel # bash scripts/dist-smoke-test.sh src/klangk/dist/klangk-*.whl -# Or via CI without tagging: `gh workflow run release.yml` from your branch -# (release.yml's workflow_dispatch runs build-wheel + dist-smoke-test only; -# publish-and-release is gated on `github.event_name == 'push'`). +# Or via CI without tagging: `gh workflow run dist-smoke.yml --ref ` +# (.github/workflows/dist-smoke.yml is the standalone smoke workflow — +# build-wheel + dist-smoke-test, no publish. release.yml no longer runs +# the smoke; it publishes without waiting on it.) set -euo pipefail PORT="${KLANGK_PORT:-18997}"