diff --git a/.ci/scripts/abi_baseline_key.sh b/.ci/scripts/abi_baseline_key.sh new file mode 100755 index 00000000000..5a081a1fc25 --- /dev/null +++ b/.ci/scripts/abi_baseline_key.sh @@ -0,0 +1,130 @@ +#!/bin/bash +#=============================================================================== +# Copyright contributors to the oneDAL project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#=============================================================================== + +# Chooses the cached main build that the ABI check compares against and writes +# its cache key to GITHUB_OUTPUT. +# +# Usage: abi_baseline_key.sh [window] +# +# Taking the newest entry in main's cache scope is wrong for three reasons. +# +# Scope. `gh cache list` shows every scope in the repository, but +# `cache/restore` can only read main's. An entry from another pull request's +# scope would miss, and the comparison would then run against an empty +# directory. +# +# Provenance. Candidates come from commits, not from the cache listing, so the +# newest key in the listing does not decide anything. Only trusted triggers +# (push, schedule, workflow_dispatch) can write to main's cache scope; runs from +# workflow_run, pull_request_target and issue_comment get read-only access. Even +# so, a key for a real main commit whose entry has expired could be re-created +# by anything that can write there, and the walk below would pick it if nothing +# newer is cached. The restored tree is only ever read by abidiff. +# +# Ancestry. abidiff compares two trees; it does not report what a pull request +# changed. So start from the base commit the build contains: when that commit is +# cached, the report holds the pull request's own changes and nothing else. When +# it is not (its main CI run has not finished saving the cache yet), the newest +# cached ancestor is used and the main commits in between show up in the report +# as if the pull request had made them. A change that undoes one of them -- such +# as removing a symbol main just added -- then cancels out and is not reported. +# Every inexact comparison says so in a warning. + +set -euo pipefail + +BASE_SHA=${1:-} +WINDOW=${2:-20} +KEY_PREFIX=__release_lnx + +# An empty base is always a wiring bug in the caller (for example reading the +# parents of a shallow clone's HEAD, which git reports as having none), never a +# cache gap, so do not let it fall through to the inexact baseline. +if [ -z "${BASE_SHA}" ]; then + echo "::error::No base commit was passed. The caller must pass the first parent of the merge commit under test." + exit 1 +fi + +# Runs "$@" up to three times, backing off in between, and echoes its output, so +# one API hiccup does not get to decide which baseline the check uses. +retry() { + local delay out + for delay in 0 5 10; do + sleep "${delay}" + if out=$("$@"); then + echo "${out}" + return 0 + fi + done + return 1 +} + +available=$(retry gh cache list --repo "${GITHUB_REPOSITORY}" --ref refs/heads/main --key "${KEY_PREFIX}" --limit 100 --json key --jq '.[].key') || { + echo "::error::Could not list the caches on refs/heads/main. Rerun this job." + exit 1 +} + +# A failure here is not safe to carry on from. It yields no candidates, which is +# indistinguishable from "no cached ancestor" and would quietly pick the inexact +# baseline, so both callers stop the job. +ancestors_of() { + retry gh api "repos/${GITHUB_REPOSITORY}/commits?sha=$1&per_page=${WINDOW}" --jq '.[].sha' +} + +# Echoes " " for the newest commit in $1 that has an +# entry, or returns 1 if none of them does. +newest_cached_in() { + local behind=0 sha + for sha in $1; do + if grep -qxF "${KEY_PREFIX}-${sha}" <<< "${available}"; then + echo "${KEY_PREFIX}-${sha} ${behind}" + return 0 + fi + behind=$((behind + 1)) + done + return 1 +} + +base_commits=$(ancestors_of "${BASE_SHA}") || { + echo "::error::Could not list the commits reachable from ${BASE_SHA}. Rerun this job." + exit 1 +} +exact=yes +result=$(newest_cached_in "${base_commits}") || result="" +if [ -z "${result}" ]; then + exact=no + main_commits=$(ancestors_of main) || { + echo "::error::Could not list the commits on refs/heads/main. Rerun this job." + exit 1 + } + result=$(newest_cached_in "${main_commits}") || { + echo "::error::No ${KEY_PREFIX} cache on refs/heads/main for any of its last ${WINDOW} commits. Rerun the 'CI' workflow on main via workflow dispatch to regenerate it." + exit 1 + } +fi +read -r cache_key behind <<< "${result}" + +if [ "${exact}" = no ]; then + echo "::warning::No cached main build for this pull request's base or its last ${WINDOW} ancestors, so the baseline is main's tip. Changes merged into main since the base will show up in this check, and a change this pull request shares with one of them will not show up at all. Rebase onto main for an exact comparison." +elif [ "${behind}" -gt 0 ]; then + exact=no + echo "::warning::The base commit's main build is not cached yet, so the baseline is ${behind} commit(s) behind it. Those commits' ABI changes show up in this check, and a change in this pull request that undoes one of them is not reported. Rerun this job once the 'CI' run on main for the base commit has finished." +fi + +echo "${cache_key} (exact: ${exact}, ${behind} commits back)" +if [ -n "${GITHUB_OUTPUT:-}" ]; then + echo "key=${cache_key}" >> "${GITHUB_OUTPUT}" +fi diff --git a/.github/workflows/ci-win.yml b/.github/workflows/ci-win.yml index f08e49a9ce2..051a6ad289f 100644 --- a/.github/workflows/ci-win.yml +++ b/.github/workflows/ci-win.yml @@ -54,6 +54,9 @@ on: - main - 'feature/**' +permissions: + contents: read + env: DEP_DIR: ${{ github.workspace }}/__deps TBB_VERSION: v2023.1.0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99a5dfca53b..115a0b2f6c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -172,19 +172,12 @@ jobs: - name: Check Cache id: get-cache-key run: | - # `gh cache list` reports every scope in the repository, not only the - # ones this run can restore from. Without `--ref` the newest matching - # key can belong to another pull request's scope, which `cache/restore` - # cannot read: the restore then misses silently and the check below - # compares against an empty directory. Only main-scoped entries are - # both restorable here and produced by a build of main. - CACHE_KEY=$(gh cache list --ref refs/heads/main --sort created_at --limit 1 --key __release_lnx --json key --jq '.[0].key') - if [ -z "${CACHE_KEY}" ] || [ "${CACHE_KEY}" = "null" ]; then - echo "::error::No __release_lnx cache found on refs/heads/main. Rerun the 'CI' workflow on main via workflow dispatch to regenerate it." - exit 1 - fi - echo "key=${CACHE_KEY}" >> "$GITHUB_OUTPUT" - echo $CACHE_KEY + # `checkout` leaves the pull request's merge commit at HEAD; its first + # parent is the main commit this build contains. That commit is the + # baseline, not main's tip -- the script explains why. The clone is + # shallow, so `git log --format=%P` and `HEAD^1` see no parents; the + # commit object still records them. + .ci/scripts/abi_baseline_key.sh "$(git cat-file -p HEAD | awk '/^parent /{print $2; exit}')" env: GH_TOKEN: ${{ github.token }} - name: Download oneDAL main build diff --git a/.github/workflows/nightly-test.yml b/.github/workflows/nightly-test.yml index ff460e41092..9b84ef45018 100644 --- a/.github/workflows/nightly-test.yml +++ b/.github/workflows/nightly-test.yml @@ -35,24 +35,23 @@ name: Nightly-test # only build a fork's commit when it carries none of the write scopes, secrets, # caches or on-disk credentials that make such a job a target. # -# Cache access: a `workflow_run` job runs with GITHUB_REF pointing at the -# default branch, so the `ACTIONS_RUNTIME_TOKEN` the runner injects is scoped to -# main's cache. Read-only `permissions`, `persist-credentials: false` and the -# absence of any cache action do not take that token away, and it is all the -# checked-out code needs to write an entry that main-scoped caches are read from -# on every branch — `ci.yml`'s ABI check restores such an entry into pull -# request jobs. Nothing in these jobs uses the cache or uploads an artifact, so -# every step that runs checked-out code blanks those variables. The artifact -# downloads keep them because they run pinned actions, not repository code. +# Cache access: these jobs run with GITHUB_REF pointing at the default branch, +# and entries in main's cache scope are restored on every branch — `ci.yml`'s +# ABI check restores one into pull request jobs. `workflow_run` runs only get +# read access to that scope, but `workflow_dispatch` runs get write access, and +# a dispatch can point `source_run_id` at a fork's build. `cache-mode: none` +# below takes cache access away from the job token entirely. The cache service +# enforces it: a write with such a token is refused with "cache write denied: +# token has no writable scopes". Nothing in these jobs needs the cache; the CVE +# scan keeps the read access it had, for the action's pip cache. # # CodeQL flags both checkouts under `actions/untrusted-checkout` (alerts 9 and -# 10), and the same steps under `actions/cache-poisoning/poisonable-step`. Both -# findings are accurate about the shape and neither changes with any of the -# mitigations above, because building a fork's commit is the point of these -# jobs; the cache-poisoning query models no mitigation at all. Nothing beyond -# the mitigations should be added to appease the queries; if the risk is judged -# unacceptable instead, the fix is to stop testing fork pull requests here, not -# to hide the checkout from the scanner. +# 10), and the same steps under `actions/cache-poisoning/poisonable-step` +# (alerts 15 and 16, raised for the `workflow_dispatch` trigger). The query does +# not model `cache-mode`, so those two alerts stay open and are dismissed with a +# reference to this note. Building a fork's commit is the point of these jobs; +# if the risk is judged unacceptable, the fix is to stop testing fork pull +# requests here, not to hide the checkout from the scanner. on: workflow_run: @@ -69,6 +68,9 @@ on: permissions: contents: read +# See the `Cache access` note at the top of this file. +cache-mode: none + concurrency: group: ${{ github.workflow }}-${{ github.event.workflow_run.id || github.run_id }} cancel-in-progress: false @@ -86,6 +88,9 @@ jobs: && github.event.workflow_run.event != 'pull_request' }} permissions: security-events: write + # Runs only on workflow_run and executes no repository code. `read` is what + # workflow_run gets by default, and lets the action restore its pip cache. + cache-mode: read env: BUILD_FOLDER: "builds" steps: @@ -190,10 +195,9 @@ jobs: # scopes, secrets and default-branch cache — the "pwn request" shape. # None of that is present here: the permissions block above grants # read only, the job references no secrets, `persist-credentials: - # false` above keeps the token off disk, and the steps that run this - # checkout blank the cache-service credentials, so building a fork's - # commit is no more privileged than the pull_request run that already - # builds it. + # false` above keeps the token off disk, and `cache-mode: none` leaves + # the job no cache access, so building a fork's commit is no more + # privileged than the pull_request run that already builds it. allow-unsafe-pr-checkout: true - name: Download oneDAL Linux build artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -202,41 +206,17 @@ jobs: github-token: ${{ github.token }} run-id: ${{ github.event.workflow_run.id || inputs.source_run_id }} path: ./__release_lnx - # Every step below runs code from the checked-out commit. Each one blanks - # the cache-service credentials the runner injects, for the reason given - # in the `Cache access` note at the top of this file. Any step added after - # this point must carry the same block. - name: Install DPC++ - env: - ACTIONS_RUNTIME_TOKEN: "" - ACTIONS_RESULTS_URL: "" - ACTIONS_CACHE_URL: "" run: .ci/env/apt.sh dpcpp - name: Install MKL - env: - ACTIONS_RUNTIME_TOKEN: "" - ACTIONS_RESULTS_URL: "" - ACTIONS_CACHE_URL: "" run: .ci/env/apt.sh mkl - name: Install Bazelisk - env: - ACTIONS_RUNTIME_TOKEN: "" - ACTIONS_RESULTS_URL: "" - ACTIONS_CACHE_URL: "" run: .ci/env/bazelisk.sh - name: Bazel release - env: - ACTIONS_RUNTIME_TOKEN: "" - ACTIONS_RESULTS_URL: "" - ACTIONS_CACHE_URL: "" run: | source /opt/intel/oneapi/setvars.sh bazel build //:release --config=release-dpc --cpu=avx2 - name: Compare make and Bazel releases - env: - ACTIONS_RUNTIME_TOKEN: "" - ACTIONS_RESULTS_URL: "" - ACTIONS_CACHE_URL: "" run: | source /opt/intel/oneapi/setvars.sh python dev/release_tests/compare_release_trees.py \ @@ -271,7 +251,7 @@ jobs: persist-credentials: false # See `bazel_lnx` for why opting out of checkout's fork guard is safe # in a job that holds no write scope, no secrets, no persisted - # credentials and no cache access in the steps that run this checkout. + # credentials and no cache access. allow-unsafe-pr-checkout: true - name: Download oneDAL Windows build artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -286,32 +266,18 @@ jobs: name: intel_oneapi_basekit github-token: ${{ github.token }} run-id: ${{ github.event.workflow_run.id || inputs.source_run_id }} - # As in `bazel_lnx`, every step below this point runs checked-out code or - # unpacks downloaded content, so each blanks the cache-service - # credentials. Any step added after this point must carry the same block. - name: Decompress Intel BaseKit shell: cmd - env: - ACTIONS_RUNTIME_TOKEN: "" - ACTIONS_RESULTS_URL: "" - ACTIONS_CACHE_URL: "" run: | tar -xvzf oneapi.tar.gz echo "Unzip complete" - name: Install Bazelisk shell: pwsh - env: - ACTIONS_RUNTIME_TOKEN: "" - ACTIONS_RESULTS_URL: "" - ACTIONS_CACHE_URL: "" run: .\.ci\env\bazelisk.ps1 - name: Bazel release shell: cmd env: BAZEL_SH: 'C:\Program Files\Git\bin\bash.exe' - ACTIONS_RUNTIME_TOKEN: "" - ACTIONS_RESULTS_URL: "" - ACTIONS_CACHE_URL: "" run: | if not exist "%BAZEL_SH%" ( echo ERROR: Required Bazel shell not found: %BAZEL_SH% @@ -323,10 +289,6 @@ jobs: bazel build //:release --config=release-dpc --cpu=avx2 - name: Compare make and Bazel releases shell: cmd - env: - ACTIONS_RUNTIME_TOKEN: "" - ACTIONS_RESULTS_URL: "" - ACTIONS_CACHE_URL: "" run: | call .\oneapi\setvars.bat call "%ONEAPI_ROOT%\setvars-vcvarsall.bat" %VS_VER% @@ -336,10 +298,6 @@ jobs: -CheckLevel 4 - name: Build and run CMake example from Bazel release shell: cmd - env: - ACTIONS_RUNTIME_TOKEN: "" - ACTIONS_RESULTS_URL: "" - ACTIONS_CACHE_URL: "" run: | call .\oneapi\setvars.bat call "%ONEAPI_ROOT%\setvars-vcvarsall.bat" %VS_VER%