Skip to content

Commit d92cbc8

Browse files
backspacebackspace
authored
Pin, verify and cache the grafanactl download (#5984)
1 parent a8afeb3 commit d92cbc8

8 files changed

Lines changed: 231 additions & 139 deletions

File tree

‎.github/actions/fetch-cached-release-asset/action.yml‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
name: Fetch a cached release asset
2+
description: |
3+
Fetch one pinned release asset, verify it against its expected SHA-256, and
4+
cache it between jobs so the release CDN is reached only on a cache miss.
5+
Outputs the path to a verified local copy; what to do with it — install it,
6+
unpack it — is the caller's business.
7+
8+
inputs:
9+
name:
10+
description: >-
11+
Short identifier for the asset, used for the cache directory, the cache
12+
key, and log messages (e.g. "mkcert").
13+
required: true
14+
url:
15+
description: >-
16+
URL of the release asset. Its last path segment becomes the local
17+
filename, so this must be a plain asset URL with no query string.
18+
required: true
19+
sha256:
20+
description: >-
21+
Expected SHA-256 of the asset, pinned in this repository rather than
22+
taken from a checksums file served beside the asset. A checksum fetched
23+
from the same host it vouches for only proves the two files agree; a
24+
pinned one is what makes both the download and the cache entry
25+
trustworthy.
26+
required: true
27+
28+
outputs:
29+
path:
30+
description: Path to the verified local copy of the asset.
31+
value: ${{ steps.plan.outputs.path }}
32+
33+
runs:
34+
using: composite
35+
steps:
36+
# The path and the cache key are derived in one step so the cache, the
37+
# download and the verification cannot disagree about them.
38+
#
39+
# The checksum is the whole identity in the key: any change to the pin is a
40+
# fresh entry. Keying on a version instead would be actively harmful, since
41+
# entries are immutable and `actions/cache` skips its save after an exact
42+
# key hit — a corrected checksum would be shadowed by the entry it
43+
# invalidates, every job restoring it, rejecting it, re-downloading, and
44+
# unable to save the replacement, quietly putting the CDN back on the
45+
# critical path this action exists to keep it off.
46+
- name: Plan the fetch
47+
id: plan
48+
shell: bash
49+
env:
50+
ASSET_NAME: ${{ inputs.name }}
51+
ASSET_URL: ${{ inputs.url }}
52+
ASSET_SHA256: ${{ inputs.sha256 }}
53+
run: |
54+
set -euo pipefail
55+
dir="${HOME}/.cache/boxel/release-assets/${ASSET_NAME}"
56+
{
57+
echo "dir=${dir}"
58+
echo "path=${dir}/${ASSET_URL##*/}"
59+
echo "cache-key=${RUNNER_OS}-${RUNNER_ARCH}-${ASSET_NAME}-${ASSET_SHA256}"
60+
} >> "$GITHUB_OUTPUT"
61+
62+
# Restore and save are the split form of `actions/cache` rather than the
63+
# combined one, because the combined form saves from a post step, and a
64+
# post step this far inside nested composite actions is not given its
65+
# inputs: it warns "Input required and not supplied: path" and saves
66+
# nothing, so the entry never appears and every job goes to the CDN
67+
# forever. An explicit save is an ordinary step and runs with these values
68+
# resolved.
69+
#
70+
# No `restore-keys`: a near-miss would restore some other asset, which the
71+
# checksum below would then reject and re-download anyway.
72+
- name: Restore the asset from the cache
73+
uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
74+
with:
75+
path: ${{ steps.plan.outputs.dir }}
76+
key: ${{ steps.plan.outputs.cache-key }}
77+
78+
- name: Fetch and verify
79+
id: fetch
80+
shell: bash
81+
env:
82+
ASSET_NAME: ${{ inputs.name }}
83+
ASSET_URL: ${{ inputs.url }}
84+
ASSET_SHA256: ${{ inputs.sha256 }}
85+
ASSET_PATH: ${{ steps.plan.outputs.path }}
86+
ASSET_DIR: ${{ steps.plan.outputs.dir }}
87+
CACHE_KEY: ${{ steps.plan.outputs.cache-key }}
88+
run: |
89+
set -euo pipefail
90+
91+
# Verified on the cache-hit path too: a restored entry is as much a
92+
# supply-chain input as a fresh download, and the cache is writable
93+
# from any branch of this repository.
94+
if [ -f "$ASSET_PATH" ] &&
95+
printf '%s %s\n' "$ASSET_SHA256" "$ASSET_PATH" | sha256sum -c --status -; then
96+
echo "${ASSET_NAME} restored from the cache"
97+
echo "downloaded=false" >> "$GITHUB_OUTPUT"
98+
exit 0
99+
fi
100+
101+
# A present-but-wrong file is not an alarm to stop on — the fresh
102+
# copy's checksum below is fatal, so nothing unverified reaches the
103+
# caller either way. It is worth an annotation rather than a log line,
104+
# though, because the state is silent and does not clear itself: the
105+
# entry was saved under the same key this job just hit, and
106+
# `actions/cache` skips its save after an exact hit, so the download
107+
# below cannot replace it. Every job then restores it, rejects it, and
108+
# goes to the CDN, indefinitely, while still passing. Only evicting the
109+
# entry ends that, so the annotation names it.
110+
if [ -e "$ASSET_PATH" ]; then
111+
echo "::warning title=Cached ${ASSET_NAME} failed its checksum::Evict the '${CACHE_KEY}' cache entry: it cannot be overwritten, so every job is downloading ${ASSET_NAME} from the release CDN until it is gone."
112+
rm -f "$ASSET_PATH"
113+
fi
114+
115+
# Downloaded outside the cache directory so a half-written file cannot
116+
# be what gets saved. The retry window is four minutes, long enough to
117+
# ride out a CDN outage that a default one-second backoff exhausts in
118+
# eighteen seconds, inside a six-minute ceiling that bounds the step
119+
# whatever curl does.
120+
#
121+
# `--max-time` resets on each retry, so it bounds one attempt rather
122+
# than the whole loop — without it a single stalled connection would
123+
# eat the entire window and never reach a second attempt. The stall it
124+
# cuts short exits 28, which curl counts as transient, so the retry
125+
# loop picks it up unaided.
126+
#
127+
# `--retry-all-errors` adds the connection-level failures — reset,
128+
# refused, DNS, TLS — that curl does not count as transient; a 504 or a
129+
# stalled transfer it would retry regardless. It retries permanent
130+
# failures too, so a pin to a tag that does not exist spends the whole
131+
# window below before failing, in every job that fetches it.
132+
download="$(mktemp -d)/${ASSET_URL##*/}"
133+
timeout 6m curl -fsSL \
134+
--connect-timeout 20 --max-time 90 \
135+
--retry 12 --retry-delay 20 --retry-max-time 240 --retry-all-errors \
136+
-o "$download" "$ASSET_URL"
137+
printf '%s %s\n' "$ASSET_SHA256" "$download" | sha256sum -c -
138+
139+
mkdir -p "$ASSET_DIR"
140+
mv "$download" "$ASSET_PATH"
141+
echo "downloaded=true" >> "$GITHUB_OUTPUT"
142+
echo "${ASSET_NAME} downloaded and verified"
143+
144+
# Saving only what was just downloaded keeps this a no-op on a cache hit.
145+
# It warns rather than fails if the key is taken — by a concurrent job, or
146+
# by the unoverwritable entry the annotation above describes.
147+
- name: Save the asset to the cache
148+
if: steps.fetch.outputs.downloaded == 'true'
149+
uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
150+
with:
151+
path: ${{ steps.plan.outputs.dir }}
152+
key: ${{ steps.plan.outputs.cache-key }}

‎.github/actions/init/action.yml‎

Lines changed: 25 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -92,113 +92,42 @@ runs:
9292
# because a release asset is a supply-chain input in a way a signed
9393
# distribution package is not.
9494
#
95-
# Every job in the repository runs this action, so a bad window on the
96-
# release CDN takes out many jobs at once, and a dead shard costs more than
97-
# its own result: the Percy build it would have contributed to is then
98-
# incomplete and rejected as a baseline.
99-
#
100-
# So the CDN is reached only on a cache miss, and a miss rides out an
101-
# outage of several minutes. The retry window below is four minutes, each
102-
# attempt separately bounded so a stalled connection is retried rather than
103-
# consuming the window, inside a six-minute ceiling that bounds the step
104-
# whatever curl does.
105-
#
106-
# The cache key carries the pinned version and its checksum, so any change
107-
# to the pin is a fresh entry. The checksum belongs in the key and not only
108-
# in the verification below: entries are immutable and `actions/cache`
109-
# skips its save step after an exact key hit, so a key on the version alone
110-
# would let a corrected checksum be shadowed by the entry it invalidates —
111-
# every job restoring it, rejecting it, re-downloading, and unable to save
112-
# the replacement, quietly putting the CDN back on the critical path this
113-
# step exists to keep it off.
114-
#
115-
# The version, checksum and key are built in one step so the cache and the
116-
# download cannot disagree about them.
117-
- name: Pin mkcert
118-
id: mkcert
95+
# That every job runs this action also sets what a bad window on the
96+
# release CDN costs: many jobs at once, and more than their own results,
97+
# since a dead shard leaves the Percy build it would have contributed to
98+
# incomplete and so rejected as a baseline. The fetch is cached and bounded
99+
# accordingly — see .github/actions/fetch-cached-release-asset, which owns
100+
# the caching, the retry window and the checksum verification.
101+
- name: Look for a preinstalled mkcert
102+
id: mkcert-preinstalled
119103
shell: bash
120104
run: |
121-
version=1.4.4
122-
sha256=6d31c65b03972c6dc4a14ab429f2928300518b26503f58723e532d1b0a3bbb52
123-
{
124-
echo "version=$version"
125-
echo "sha256=$sha256"
126-
echo "dir=$HOME/.cache/boxel/mkcert"
127-
echo "cache-key=${RUNNER_OS}-${RUNNER_ARCH}-mkcert-${version}-${sha256}"
128-
} >> "$GITHUB_OUTPUT"
105+
if command -v mkcert >/dev/null 2>&1; then
106+
echo "mkcert is already on PATH"
107+
echo "present=true" >> "$GITHUB_OUTPUT"
108+
else
109+
echo "present=false" >> "$GITHUB_OUTPUT"
110+
fi
129111
130-
# No `restore-keys`: a near-miss would restore some other version's binary,
131-
# which the checksum below would then reject and re-download anyway.
132-
- uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
133-
name: Setup mkcert cache
112+
- name: Fetch mkcert
113+
id: mkcert
114+
if: steps.mkcert-preinstalled.outputs.present != 'true'
115+
uses: ./.github/actions/fetch-cached-release-asset
134116
with:
135-
path: ${{ steps.mkcert.outputs.dir }}
136-
key: ${{ steps.mkcert.outputs.cache-key }}
117+
name: mkcert
118+
url: https://github.com/FiloSottile/mkcert/releases/download/v1.4.4/mkcert-v1.4.4-linux-amd64
119+
sha256: 6d31c65b03972c6dc4a14ab429f2928300518b26503f58723e532d1b0a3bbb52
137120

138121
# certutil, mkcert's only other requirement, already ships on the runner
139122
# image; the apt path below covers a future image that drops it.
140123
- name: Install mkcert
141124
shell: bash
142125
env:
143-
MKCERT_VERSION: ${{ steps.mkcert.outputs.version }}
144-
MKCERT_SHA256: ${{ steps.mkcert.outputs.sha256 }}
145-
MKCERT_CACHE_DIR: ${{ steps.mkcert.outputs.dir }}
146-
MKCERT_CACHE_KEY: ${{ steps.mkcert.outputs.cache-key }}
126+
# Empty when mkcert was already on PATH and the fetch above was skipped.
127+
MKCERT_ASSET: ${{ steps.mkcert.outputs.path }}
147128
run: |
148-
cached="${MKCERT_CACHE_DIR}/mkcert-v${MKCERT_VERSION}-linux-amd64"
149-
150-
# Verified on the cache-hit path too: a restored entry is as much a
151-
# supply-chain input as a fresh download, and the cache is writable
152-
# from any branch of this repository.
153-
checksum_ok() {
154-
[ -f "$1" ] && printf '%s %s\n' "$MKCERT_SHA256" "$1" | sha256sum -c --status -
155-
}
156-
157-
if command -v mkcert >/dev/null 2>&1; then
158-
echo "mkcert is already on PATH"
159-
else
160-
if checksum_ok "$cached"; then
161-
echo "mkcert v${MKCERT_VERSION} restored from the cache"
162-
else
163-
# A present-but-wrong file is not an alarm to stop on — the fresh
164-
# copy's checksum below is fatal, so nothing unverified is
165-
# installed either way. It is worth an annotation rather than a log
166-
# line, though, because the state is silent and does not clear
167-
# itself: the entry was saved under the same key this job just hit,
168-
# and `actions/cache` skips its save after an exact hit, so the
169-
# download below cannot replace it. Every job in the repository
170-
# then restores it, rejects it, and goes to the CDN, indefinitely,
171-
# while still passing. Only evicting the entry ends that, so the
172-
# annotation names it.
173-
if [ -e "$cached" ]; then
174-
echo "::warning title=Cached mkcert failed its checksum::Evict the '${MKCERT_CACHE_KEY}' cache entry: it cannot be overwritten, so every job is downloading mkcert from the release CDN until it is gone."
175-
rm -f "$cached"
176-
fi
177-
url="https://github.com/FiloSottile/mkcert/releases/download/v${MKCERT_VERSION}/mkcert-v${MKCERT_VERSION}-linux-amd64"
178-
# Downloaded outside the cache directory so a half-written file
179-
# cannot be what gets saved.
180-
#
181-
# `--max-time` resets on each retry, so it bounds one attempt
182-
# rather than the whole loop — without it a single stalled
183-
# connection would eat the entire window and never reach a second
184-
# attempt. The stall it cuts short exits 28, which curl counts as
185-
# transient, so the retry loop picks it up unaided.
186-
#
187-
# `--retry-all-errors` adds the connection-level failures — reset,
188-
# refused, DNS, TLS — that curl does not count as transient; a 504
189-
# or a stalled transfer it would retry regardless. It retries
190-
# permanent failures too, so a version pinned above to a tag that
191-
# does not exist spends the whole window below before failing, in
192-
# every job that runs this action.
193-
timeout 6m curl -fsSL \
194-
--connect-timeout 20 --max-time 90 \
195-
--retry 12 --retry-delay 20 --retry-max-time 240 --retry-all-errors \
196-
-o /tmp/mkcert.download "$url"
197-
printf '%s %s\n' "$MKCERT_SHA256" /tmp/mkcert.download | sha256sum -c -
198-
mkdir -p "$MKCERT_CACHE_DIR"
199-
mv /tmp/mkcert.download "$cached"
200-
fi
201-
sudo install -m 0755 "$cached" /usr/local/bin/mkcert
129+
if [ -n "$MKCERT_ASSET" ]; then
130+
sudo install -m 0755 "$MKCERT_ASSET" /usr/local/bin/mkcert
202131
fi
203132
204133
if ! command -v certutil >/dev/null 2>&1; then

‎.github/actions/install-grafanactl/action.yml‎

Lines changed: 41 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,54 @@
11
name: Install grafanactl
22
description: |
3-
Download and install the pinned grafanactl release on the Ubuntu runner,
4-
verifying the SHA-256 checksum from the release's grafanactl_checksums.txt
5-
before placing the binary on PATH. Used by the observability-apply-staging,
6-
observability-apply-production, and observability-diff workflows.
3+
Install the pinned grafanactl release on the Ubuntu runner, verifying its
4+
SHA-256 before placing the binary on PATH. Used by the observability diff,
5+
preview, preview-sweep and apply workflows.
76
8-
inputs:
7+
outputs:
98
version:
10-
description: grafanactl release version (without the leading 'v', e.g. "0.1.10")
11-
required: true
9+
description: The grafanactl version installed.
10+
value: ${{ steps.pin.outputs.version }}
1211

1312
runs:
1413
using: composite
1514
steps:
16-
- shell: bash
17-
env:
18-
GRAFANACTL_VERSION: ${{ inputs.version }}
15+
# grafanactl is still pre-1.0 — bump as needed. The version and the
16+
# checksum are a single pin and have to move together, so they live here
17+
# rather than in the five workflows that install it: a version supplied
18+
# apart from its checksum would fail verification rather than install what
19+
# was asked for. The checksum is this asset's line in the release's
20+
# grafanactl_checksums.txt.
21+
- name: Pin grafanactl
22+
id: pin
23+
shell: bash
1924
run: |
2025
set -euo pipefail
21-
TARBALL="grafanactl_Linux_x86_64.tar.gz"
22-
CHECKSUMS="grafanactl_checksums.txt"
23-
RELEASE_URL="https://github.com/grafana/grafanactl/releases/download/v${GRAFANACTL_VERSION}"
24-
TMPDIR="$(mktemp -d)"
25-
trap 'rm -rf "$TMPDIR"' EXIT
26-
27-
curl -fsSL "${RELEASE_URL}/${TARBALL}" -o "${TMPDIR}/${TARBALL}"
28-
curl -fsSL "${RELEASE_URL}/${CHECKSUMS}" -o "${TMPDIR}/${CHECKSUMS}"
26+
version=0.1.10
27+
{
28+
echo "version=${version}"
29+
echo "sha256=7f785c8be5ab6210579ae2bc0fddb9cdf180a2d59d928935adba8e581b69cfa9"
30+
echo "url=https://github.com/grafana/grafanactl/releases/download/v${version}/grafanactl_Linux_x86_64.tar.gz"
31+
} >> "$GITHUB_OUTPUT"
2932
30-
( cd "$TMPDIR" && grep " ${TARBALL}\$" "${CHECKSUMS}" | sha256sum -c - )
33+
- name: Fetch grafanactl
34+
id: asset
35+
uses: ./.github/actions/fetch-cached-release-asset
36+
with:
37+
name: grafanactl
38+
url: ${{ steps.pin.outputs.url }}
39+
sha256: ${{ steps.pin.outputs.sha256 }}
3140

32-
tar -xzf "${TMPDIR}/${TARBALL}" -C "$TMPDIR" grafanactl
33-
sudo install -m 0755 "${TMPDIR}/grafanactl" /usr/local/bin/grafanactl
41+
# The tarball is what gets cached and checksummed, so it is unpacked afresh
42+
# each run: that keeps the pinned value the one the release publishes,
43+
# verifiable against upstream without unpacking anything.
44+
- name: Install grafanactl
45+
shell: bash
46+
env:
47+
GRAFANACTL_TARBALL: ${{ steps.asset.outputs.path }}
48+
run: |
49+
set -euo pipefail
50+
unpacked="$(mktemp -d)"
51+
trap 'rm -rf "$unpacked"' EXIT
52+
tar -xzf "$GRAFANACTL_TARBALL" -C "$unpacked" grafanactl
53+
sudo install -m 0755 "${unpacked}/grafanactl" /usr/local/bin/grafanactl
3454
grafanactl --version

‎.github/workflows/observability-apply-production.yml‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,6 @@ jobs:
6969
# for the rationale). The pre-flight check below asserts the expected
7070
# hostname for paranoia.
7171
EXPECTED_GRAFANA_HOSTNAME: dashboard.boxel.ai
72-
GRAFANACTL_VERSION: "0.1.10"
7372
# Worker CloudWatch log group + owning AWS account, substituted into
7473
# provisioning/alerting/worker-status-group.json at push time
7574
# (CS-11107). Hardcoded here rather than fetched from SSM: the log
@@ -127,9 +126,8 @@ jobs:
127126
fi
128127
129128
- name: Install grafanactl
129+
id: grafanactl
130130
uses: ./.github/actions/install-grafanactl
131-
with:
132-
version: ${{ env.GRAFANACTL_VERSION }}
133131

134132
- name: Configure AWS credentials
135133
uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
@@ -192,5 +190,5 @@ jobs:
192190
echo "- Target: production (server URL from packages/observability/grafanactl/config.yaml)" >> "$GITHUB_STEP_SUMMARY"
193191
echo "- Dispatched ref: \`${{ github.ref }}\`" >> "$GITHUB_STEP_SUMMARY"
194192
echo "- Triggered by: \`${{ github.actor }}\`" >> "$GITHUB_STEP_SUMMARY"
195-
echo "- grafanactl: \`$GRAFANACTL_VERSION\`" >> "$GITHUB_STEP_SUMMARY"
193+
echo "- grafanactl: \`${{ steps.grafanactl.outputs.version }}\`" >> "$GITHUB_STEP_SUMMARY"
196194
echo "- Status: ${{ job.status }}" >> "$GITHUB_STEP_SUMMARY"

0 commit comments

Comments
 (0)