From 4c73692ea255a0331fb6892c78857c0852aaee4c Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 19 Oct 2026 18:18:19 -0400 Subject: [PATCH 01/23] feat(ci): add COSE package repository publishing Consume verified release assets, sign with an isolated OpenPGP subkey, and publish incremental apt/yum repositories through conditional S3 writes. Include protected maintenance commands, offline CI coverage, native package checks, and operator documentation. Production publication remains gated until migration cutover. AI assistance: GPT-6 (Codex). --- .../package-repository-tools/action.yml | 14 + .github/workflows/changes.yml | 12 + .../package-repositories-maintenance.yml | 90 ++++ .github/workflows/package-repositories.yml | 237 ++++++++++ .github/workflows/publish.yml | 12 - .github/workflows/test.yml | 18 + Cargo.toml | 1 - distribution/repositories/.dockerignore | 6 + distribution/repositories/Dockerfile | 11 + distribution/repositories/engine.py | 278 ++++++++++++ distribution/repositories/inputs.py | 169 ++++++++ distribution/repositories/production.json | 19 + distribution/repositories/repository.py | 403 ++++++++++++++++++ distribution/repositories/requirements.txt | 8 + distribution/repositories/storage.py | 396 +++++++++++++++++ distribution/repositories/tests/client.sh | 32 ++ .../repositories/tests/package_fixture.py | 117 +++++ .../repositories/tests/publication_fixture.py | 123 ++++++ .../repositories/tests/test_inputs.py | 86 ++++ .../repositories/tests/test_storage.py | 244 +++++++++++ distribution/repositories/vector-packages.asc | 65 +++ docs/README.md | 3 + docs/reference/package-repositories.md | 167 ++++++++ scripts/verify-install.sh | 26 +- vdev/src/commands/release/mod.rs | 3 + vdev/src/commands/release/repositories.rs | 248 +++++++++++ 26 files changed, 2773 insertions(+), 15 deletions(-) create mode 100644 .github/actions/package-repository-tools/action.yml create mode 100644 .github/workflows/package-repositories-maintenance.yml create mode 100644 .github/workflows/package-repositories.yml create mode 100644 distribution/repositories/.dockerignore create mode 100644 distribution/repositories/Dockerfile create mode 100644 distribution/repositories/engine.py create mode 100644 distribution/repositories/inputs.py create mode 100644 distribution/repositories/production.json create mode 100644 distribution/repositories/repository.py create mode 100644 distribution/repositories/requirements.txt create mode 100644 distribution/repositories/storage.py create mode 100644 distribution/repositories/tests/client.sh create mode 100644 distribution/repositories/tests/package_fixture.py create mode 100644 distribution/repositories/tests/publication_fixture.py create mode 100644 distribution/repositories/tests/test_inputs.py create mode 100644 distribution/repositories/tests/test_storage.py create mode 100644 distribution/repositories/vector-packages.asc create mode 100644 docs/reference/package-repositories.md create mode 100644 vdev/src/commands/release/repositories.rs diff --git a/.github/actions/package-repository-tools/action.yml b/.github/actions/package-repository-tools/action.yml new file mode 100644 index 0000000000000..77289e8e2d058 --- /dev/null +++ b/.github/actions/package-repository-tools/action.yml @@ -0,0 +1,14 @@ +name: Restore package repository tools +description: Restore the publisher image and vdev built by this workflow run. +runs: + using: composite + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: package-repository-tools + path: ${{ runner.temp }}/package-repository-tools + - shell: bash + run: | + gzip -dc "$RUNNER_TEMP/package-repository-tools/image.tar.gz" | docker load + chmod +x "$RUNNER_TEMP/package-repository-tools/vdev" + echo "VDEV=$RUNNER_TEMP/package-repository-tools/vdev" >> "$GITHUB_ENV" diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index 41c559182ed6a..80eb2c6dd17ed 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -52,6 +52,8 @@ on: value: ${{ jobs.source.outputs.website_tests }} install: value: ${{ jobs.source.outputs.install }} + package_repositories: + value: ${{ jobs.source.outputs.package_repositories }} actionlint: value: ${{ jobs.source.outputs.actionlint }} prettier: @@ -187,6 +189,7 @@ jobs: markdown: ${{ steps.filter.outputs.markdown }} scripts: ${{ steps.filter.outputs.scripts }} install: ${{ steps.filter.outputs.install }} + package_repositories: ${{ steps.filter.outputs.package_repositories }} k8s: ${{ steps.filter.outputs.k8s }} website_only: ${{ steps.filter.outputs.website == 'true' && steps.filter.outputs.not_website == 'false' }} website_tests: ${{ steps.website_filter.outputs.website_tests }} @@ -283,6 +286,15 @@ jobs: - ".github/workflows/install-sh.yml" - "distribution/install.sh" - ".github/workflows/changes.yml" + package_repositories: + - "distribution/repositories/**" + - "vdev/**" + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - ".github/actions/package-repository-tools/**" + - ".github/workflows/package-repositories*.yml" + - ".github/workflows/changes.yml" k8s: - "src/sources/kubernetes_logs/**" - "lib/k8s-e2e-tests/**" diff --git a/.github/workflows/package-repositories-maintenance.yml b/.github/workflows/package-repositories-maintenance.yml new file mode 100644 index 0000000000000..20e20b24368b8 --- /dev/null +++ b/.github/workflows/package-repositories-maintenance.yml @@ -0,0 +1,90 @@ +name: Package Repository Maintenance + +on: + workflow_dispatch: + inputs: + operation: + description: Inspect state, restore signed metadata, or recover an abandoned lock + type: choice + options: [status, rollback, recover-lock] + default: status + snapshot: + description: Exact archived snapshot SHA-256 for rollback + type: string + expected_version: + description: Exact control-object S3 version reported by status, for lock recovery + type: string + confirm_owner_stopped: + description: The previous workflow/process and its in-flight writes have stopped + type: boolean + default: false + +permissions: {} + +concurrency: + group: package-repositories + cancel-in-progress: false + +jobs: + maintain: + if: github.repository_id == '146327667' && github.ref == 'refs/heads/master' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + environment: package-repositories-publishing + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Validate the requested maintenance operation + env: + OPERATION: ${{ inputs.operation }} + SNAPSHOT: ${{ inputs.snapshot }} + EXPECTED_VERSION: ${{ inputs.expected_version }} + OWNER_STOPPED: ${{ inputs.confirm_owner_stopped }} + run: | + case "$OPERATION" in + status) ;; + rollback) [[ "$SNAPSHOT" =~ ^[0-9a-f]{64}$ ]] ;; + recover-lock) test -n "$EXPECTED_VERSION" && test "$OWNER_STOPPED" = true ;; + *) exit 1 ;; + esac + - name: Build the reviewed maintenance tools + run: | + cargo build --locked --package vdev + target/debug/vdev release repositories build-image + - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 + with: + role-to-assume: ${{ vars.PACKAGE_REPOSITORIES_ROLE_ARN }} + aws-region: us-east-1 + role-duration-seconds: 900 + role-session-name: package-maintenance-${{ github.run_id }} + allowed-account-ids: "486234852809" + unset-current-credentials: true + - name: Run maintenance and record the resulting state + env: + PACKAGE_REPOSITORIES_ROLE_ARN: ${{ vars.PACKAGE_REPOSITORIES_ROLE_ARN }} + OPERATION: ${{ inputs.operation }} + SNAPSHOT: ${{ inputs.snapshot }} + EXPECTED_VERSION: ${{ inputs.expected_version }} + run: | + case "$OPERATION" in + status) ;; + rollback) + target/debug/vdev release repositories rollback "$SNAPSHOT" + ;; + recover-lock) + target/debug/vdev release repositories recover-lock \ + --expected-version "$EXPECTED_VERSION" --confirm-owner-stopped + ;; + esac + target/debug/vdev release repositories status > target/package-repository-status.json + cat target/package-repository-status.json + { + echo '```json' + cat target/package-repository-status.json + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/package-repositories.yml b/.github/workflows/package-repositories.yml new file mode 100644 index 0000000000000..c458f45a23bd9 --- /dev/null +++ b/.github/workflows/package-repositories.yml @@ -0,0 +1,237 @@ +name: Package Repositories + +on: + workflow_run: + workflows: [Release Suite] + types: [completed] + workflow_dispatch: + inputs: + run_id: + description: Successful Release Suite run ID for the published tag + type: string + required: true + tag: + description: Immutable published tag (v0.x.y or an explicitly selected beta) + type: string + required: true + channel: + description: Repository channel + type: choice + options: [stable, beta] + default: stable + +permissions: {} + +# All channels share the apt pool and one S3 publication control object. +concurrency: + group: package-repositories + cancel-in-progress: false + +jobs: + inputs: + if: >- + github.repository_id == '146327667' && + github.ref == 'refs/heads/master' && + vars.PACKAGE_REPOSITORIES_ENABLED == 'true' && + (github.event_name == 'workflow_dispatch' || + (github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + startsWith(github.event.workflow_run.head_branch, 'v0.') && + !contains(github.event.workflow_run.head_branch, '-'))) + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + actions: read + outputs: + tooling_commit: ${{ steps.source.outputs.commit }} + steps: + # workflow_run executes reviewed default-branch code, never release/tag code. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - id: source + run: echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + - name: Build the reviewed tools + run: | + cargo build --locked --package vdev + target/debug/vdev release repositories build-image + target/debug/vdev release repositories test + - name: Freeze and verify release inputs + env: + GH_TOKEN: ${{ github.token }} + SOURCE_RUN: ${{ inputs.run_id || github.event.workflow_run.id }} + SOURCE_TAG: ${{ inputs.tag || github.event.workflow_run.head_branch }} + CHANNEL: ${{ inputs.channel || 'stable' }} + run: target/debug/vdev release repositories fetch --run "$SOURCE_RUN" --tag "$SOURCE_TAG" --channel "$CHANNEL" + - name: Save the exact tools used by this run + run: | + mkdir -p target/package-repository-tools + cp target/debug/vdev target/package-repository-tools/vdev + docker save vector-package-publisher:local | gzip > target/package-repository-tools/image.tar.gz + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: package-repository-tools + path: target/package-repository-tools/ + retention-days: 1 + if-no-files-found: error + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: package-repository-inputs + path: target/package-repositories/ + retention-days: 1 + if-no-files-found: error + + base: + needs: inputs + runs-on: ubuntu-24.04 + environment: package-repositories-publishing + timeout-minutes: 20 + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.inputs.outputs.tooling_commit }} + persist-credentials: false + - uses: ./.github/actions/package-repository-tools + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: package-repository-inputs + path: target/package-repositories + - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 + with: + role-to-assume: ${{ vars.PACKAGE_REPOSITORIES_ROLE_ARN }} + aws-region: us-east-1 + role-duration-seconds: 900 + allowed-account-ids: "486234852809" + - name: Read the active generation or restore a signed retry + env: + PACKAGE_REPOSITORIES_ROLE_ARN: ${{ vars.PACKAGE_REPOSITORIES_ROLE_ARN }} + run: '"$VDEV" release repositories base' + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: package-repository-base + path: target/package-repositories/ + retention-days: 1 + if-no-files-found: error + + sign: + needs: [inputs, base] + runs-on: ubuntu-24.04 + environment: package-repositories-signing + timeout-minutes: 20 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.inputs.outputs.tooling_commit }} + persist-credentials: false + - uses: ./.github/actions/package-repository-tools + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: package-repository-base + path: target/package-repositories + - id: cached + run: | + if test -f target/package-repositories/result/manifest.json; then + echo 'ready=true' >> "$GITHUB_OUTPUT" + fi + - name: Sign packages and incremental repository metadata + if: steps.cached.outputs.ready != 'true' + env: + PACKAGE_SIGNING_SUBKEY: ${{ secrets.PACKAGE_SIGNING_SUBKEY }} + PACKAGE_SIGNING_PASSPHRASE: ${{ secrets.PACKAGE_SIGNING_PASSPHRASE }} + run: '"$VDEV" release repositories prepare' + # Private material only exists in the container's tmpfs, outside this path. + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: package-repository-signed + path: target/package-repositories/ + retention-days: 1 + if-no-files-found: error + + publish: + needs: [inputs, sign] + runs-on: ubuntu-24.04 + environment: package-repositories-publishing + timeout-minutes: 20 + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.inputs.outputs.tooling_commit }} + persist-credentials: false + - uses: ./.github/actions/package-repository-tools + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: package-repository-signed + path: target/package-repositories + - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 + with: + role-to-assume: ${{ vars.PACKAGE_REPOSITORIES_ROLE_ARN }} + aws-region: us-east-1 + role-duration-seconds: 900 + allowed-account-ids: "486234852809" + - name: Publish and verify S3 objects + env: + PACKAGE_REPOSITORIES_ROLE_ARN: ${{ vars.PACKAGE_REPOSITORIES_ROLE_ARN }} + run: '"$VDEV" release repositories publish' + + endpoints: + needs: [inputs, publish] + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.inputs.outputs.tooling_commit }} + persist-credentials: false + - uses: ./.github/actions/package-repository-tools + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: package-repository-signed + path: target/package-repositories + - name: Verify public HTTPS metadata and new package bytes + run: '"$VDEV" release repositories verify' + + clients: + needs: [inputs, endpoints] + runs-on: ${{ matrix.runner }} + timeout-minutes: 15 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + runner: [ubuntu-24.04, ubuntu-24.04-arm] + image: ["ubuntu:24.04", "almalinux:9"] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.inputs.outputs.tooling_commit }} + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: package-repository-inputs + path: target/package-repositories + - name: Install through the public repository with the pinned public key + env: + IMAGE: ${{ matrix.image }} + run: | + CHANNEL=$(jq -r .channel target/package-repositories/input.json) + TAG=$(jq -r .tag target/package-repositories/input.json) + export CHANNEL TAG + export APT_URL=https://apt.vector.dev/ + export YUM_URL=https://yum.vector.dev/ + docker run --rm --env CHANNEL --env TAG --env APT_URL --env YUM_URL \ + --mount "type=bind,source=$PWD/distribution/repositories/vector-packages.asc,target=/key.asc,readonly" \ + --mount "type=bind,source=$PWD/distribution/repositories/tests/client.sh,target=/test.sh,readonly" \ + "$IMAGE" bash /test.sh diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9c69185b22f87..c59f349429d3d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -296,7 +296,6 @@ jobs: - build-linux-cross env: VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }} - DD_PKG_VERSION: "latest" strategy: matrix: runner: @@ -319,11 +318,6 @@ jobs: git \ systemd \ make - - name: Install dd-pkg for linting - run: | - ARCH=$(uname -m) - case "$ARCH" in aarch64) ARCH_DL=arm64 ;; *) ARCH_DL="$ARCH" ;; esac - curl -sSL "https://dd-package-tools.s3.amazonaws.com/dd-pkg/${DD_PKG_VERSION}/dd-pkg_Linux_${ARCH_DL}.tar.gz" | tar -xz -C /usr/local/bin dd-pkg - name: Fix Git safe directories issue when in containers (actions/checkout#760) run: git config --global --add safe.directory /__w/vector/vector - name: Checkout Vector @@ -351,7 +345,6 @@ jobs: - build-linux-cross env: VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }} - DD_PKG_VERSION: "latest" strategy: matrix: runner: @@ -380,11 +373,6 @@ jobs: if ! command -v curl &> /dev/null ; then yum install -y curl fi - - name: Install dd-pkg for linting - run: | - ARCH=$(uname -m) - case "$ARCH" in aarch64) ARCH_DL=arm64 ;; *) ARCH_DL="$ARCH" ;; esac - curl -sSL "https://dd-package-tools.s3.amazonaws.com/dd-pkg/${DD_PKG_VERSION}/dd-pkg_Linux_${ARCH_DL}.tar.gz" | tar -xz -C /usr/local/bin dd-pkg - name: Fix Git safe directories issue when in containers (actions/checkout#760) run: git config --global --add safe.directory /__w/vector/vector - name: Checkout Vector diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4177a9d334037..ef7dadd519f0c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -69,6 +69,23 @@ jobs: cargo-cache: true - run: make check-clippy + check-package-repositories: + name: Check package repository tooling + runs-on: ubuntu-24.04 + timeout-minutes: 15 + if: ${{ needs.changes.outputs.package_repositories == 'true' || needs.changes.outputs.test-yml == 'true' }} + needs: changes + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # Offline regressions need neither signing material nor AWS access. + - run: | + cargo test --locked --package vdev commands::release::repositories:: + cargo build --locked --package vdev + target/debug/vdev release repositories build-image + target/debug/vdev release repositories test + test: needs: changes permissions: @@ -214,6 +231,7 @@ jobs: - check-fmt - check-actionlint - check-clippy + - check-package-repositories - test - check-scripts - check-events diff --git a/Cargo.toml b/Cargo.toml index e8f1b03c82ee6..775bcc5dab0ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,7 +68,6 @@ assets = [ ] license-file = ["target/debian-license.txt"] extended-description-file = "target/debian-extended-description.txt" -recommends = "datadog-signing-keys (>= 1:1.4.0)" [package.metadata.deb.systemd-units] unit-scripts = "distribution/systemd/" diff --git a/distribution/repositories/.dockerignore b/distribution/repositories/.dockerignore new file mode 100644 index 0000000000000..856e6bdc83ef4 --- /dev/null +++ b/distribution/repositories/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!requirements.txt +!*.py +!tests/ +!tests/** diff --git a/distribution/repositories/Dockerfile b/distribution/repositories/Dockerfile new file mode 100644 index 0000000000000..cff4f3e635318 --- /dev/null +++ b/distribution/repositories/Dockerfile @@ -0,0 +1,11 @@ +FROM ubuntu:24.04@sha256:68ee22004a9109b44973313b02ff7b1de250f1de0f5ecb628d73f579aaff33ac +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + apt-utils ca-certificates createrepo-c dpkg-dev gnupg python3 python3-venv rpm \ + && rm -rf /var/lib/apt/lists/* +COPY requirements.txt /app/requirements.txt +RUN python3 -m venv /opt/publisher && /opt/publisher/bin/pip install \ + --no-cache-dir --require-hashes --only-binary=:all: -r /app/requirements.txt +COPY *.py /app/ +COPY tests /app/tests +WORKDIR /app +ENTRYPOINT ["/opt/publisher/bin/python", "/app/engine.py"] diff --git a/distribution/repositories/engine.py b/distribution/repositories/engine.py new file mode 100644 index 0000000000000..e8fb1da1c0864 --- /dev/null +++ b/distribution/repositories/engine.py @@ -0,0 +1,278 @@ +"""Container implementation for the typed `vdev release repositories` commands.""" + +import argparse +import gzip +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import sys +import tarfile +from urllib.request import urlopen + +from inputs import fetch, require, verify_input +from repository import PublicKey, lint_package, prepare, repomd_locations, run, verify_metadata +from storage import (Busy, Conflict, Publisher, S3Store, Snapshot, byte_entry, + canonical, matches, mutable, safe_relative, sha256_file) + + +def configuration(path): + config = json.loads(path.read_text()) + require(config["schema"] == 1, "Unsupported publisher configuration") + require(config["repository"] == "vectordotdev/vector" and config["repository_id"] == 146327667, "Wrong source repository") + require(re.fullmatch(r"[0-9]{12}", config["account"]), "Invalid AWS account") + require(re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", config["bucket"]), "Invalid bucket") + paths = [config[field] for field in ("live_prefix", "archive_prefix", "bundle_prefix", "control_key")] + for name in paths: + require(re.fullmatch(r"[a-zA-Z0-9_./-]+", name) and not name.startswith("/") + and all(part not in ("", ".", "..") for part in name.rstrip("/").split("/")), "Invalid S3 path") + require(all(config[field].endswith("/") for field in ("live_prefix", "archive_prefix", "bundle_prefix")), "Prefixes must end in / ") + require(all(not a.startswith(b) and not b.startswith(a) for i, a in enumerate(paths) for b in paths[i+1:]), + "Live, archive, bundle and control paths must be disjoint") + if config["account"] == "486234852809": + require(config["bucket"] == "dd-cose-releases" and config["live_prefix"] == "vector/repositories/", + "Unexpected production repository destination") + require(all(not value.startswith("vector/") for value in paths[1:]), "Internal state must be outside the public vector/ prefix") + for name in ("primary_fingerprint", "signing_fingerprint"): + require(re.fullmatch(r"[0-9A-F]{40}", config[name]), "Invalid pinned key fingerprint") + require(Path(config["public_key"]).name == config["public_key"], "Public key must be beside configuration") + config["public_key_path"] = str(path.parent / config["public_key"]) + config["role"] = os.environ.get("PACKAGE_REPOSITORIES_ROLE_ARN", "") + return config + + +def store_for(config): + require(re.fullmatch(r"arn:aws:iam::" + config["account"] + r":role/[A-Za-z0-9_+=,.@-]+", config["role"]), + "Set the configured publishing role ARN explicitly") + return S3Store(config) + + +def bundle_key(config, snapshot): + require(snapshot.input_id is not None, "Snapshot has no input identity") + return config["bundle_prefix"] + "bundles/" + snapshot.input_id + "/" + (snapshot.base or "initial") + ".tar.gz" + + +def rpm_cache_key(config, asset): + return config["bundle_prefix"] + "rpms/" + asset["sha256"] + ".rpm" + + +def checked_download(store, key, path, entry=None): + path.parent.mkdir(parents=True, exist_ok=True) + head = store.head(key) + require(head is not None and head["ContentLength"] < 5 * 1024**3, "Missing or oversized S3 input") + actual = store.download(key, path) + require(matches(head, actual) and (entry is None or actual == entry), "Downloaded S3 object failed checksum verification") + return actual + + +def unpack_bundle(path, destination): + destination.mkdir() + total = 0 + names = set() + with tarfile.open(path, "r:gz") as archive: + for member in archive: + require(member.isfile() and member.name not in names, "Bundle has a duplicate or nonregular member") + names.add(member.name) + require(member.name in {"input.json", "manifest.json"} or member.name.startswith("files/"), "Unexpected bundle member") + if member.name.startswith("files/"): + safe_relative(member.name[6:]) + total += member.size + require(total < 5 * 1024**3, "Bundle exceeds the extraction budget") + target = destination / member.name + target.parent.mkdir(parents=True, exist_ok=True) + with archive.extractfile(member) as source, target.open("xb") as output: + shutil.copyfileobj(source, output) + require({"input.json", "manifest.json"}.issubset(names), "Incomplete signed bundle") + return Snapshot.local(destination / "files") + + +def pack_bundle(source, target): + with target.open("xb") as raw, gzip.GzipFile(fileobj=raw, mode="wb", mtime=0, filename="") as compressed: + with tarfile.open(fileobj=compressed, mode="w|") as archive: + for path in sorted(source.rglob("*")): + require(not path.is_symlink(), "Bundle contains a symlink") + if path.is_file(): + info = archive.gettarinfo(str(path), arcname=path.relative_to(source).as_posix()) + info.uid = info.gid = info.mtime = 0 + info.uname = info.gname = "" + info.mode = 0o644 + with path.open("rb") as data: + archive.addfile(info, data) + + +def base(config, work): + manifest, input_id = verify_input(work) + store = store_for(config) + publisher = Publisher(store, config) + state, _ = publisher.load_state() + require(not state["held"], "Publisher is locked; inspect status before retrying") + active = publisher.archived(state["active"]) if state["active"] else None + require(active is not None or not config.get("require_existing_repository", True), + "Historical repository backfill is required before release publication") + if state["pending"]: + pending = publisher.archived(state["pending"]) + require(pending.input_id == input_id, "Resume or roll back the pending publication first") + candidate = pending + elif active and active.input_id == input_id: + candidate = active + else: + candidate = Snapshot({"apt/dists/stable/InRelease": byte_entry(b"")}, + base=active.id if active else None, input_id=input_id) + key = bundle_key(config, candidate) + if store.head(key): + archive = work / "cached.tar.gz" + checked_download(store, key, archive) + snapshot = unpack_bundle(archive, work / "result") + require(snapshot.input_id == input_id and snapshot.base == candidate.base, "Cached bundle belongs to another input or base") + require((work / "result/input.json").read_bytes() == (work / "input.json").read_bytes(), "Cached source manifest differs") + archive.unlink() + print("Restored the saved signed publication; signing is skipped") + return + require(not state["pending"] and not (active and active.input_id == input_id), "Required retry bundle is missing") + (work / "base.json").write_bytes(active.data if active else b"null\n") + destination = work / "base" + destination.mkdir() + if active: + for name, entry in active.entries.items(): + if mutable(name): + key = publisher.archives + active.id + "/files/" + name + checked_download(store, key, destination / name, entry) + for root in destination.glob("yum/*/vector-0/*"): + for relative in repomd_locations(root): + name = (root / relative).relative_to(destination).as_posix() + require(name in active.entries, "RPM metadata is missing from the active manifest") + checked_download(store, publisher.live + name, destination / name, active.entries[name]) + reuse = work / "reuse" + reuse.mkdir() + for name, asset in manifest["assets"].items(): + if asset.get("format") == "rpm" and store.head(rpm_cache_key(config, asset)): + checked_download(store, rpm_cache_key(config, asset), reuse / name) + print("Downloaded current metadata and any previously signed RPMs") + + +def check_result(config, work): + manifest, input_id = verify_input(work) + snapshot = Snapshot.local(work / "result/files") + require(snapshot.input_id == input_id and (work / "result/input.json").read_bytes() == (work / "input.json").read_bytes(), + "Signed output belongs to a different immutable input") + public = PublicKey(config, "/run/keys/check-result") + try: + verify_metadata(snapshot, snapshot.source, public) + rpmdb = public.home / "rpmdb" + run(["rpm", "--dbpath", rpmdb, "--import", public.key]) + for name, asset in manifest["assets"].items(): + if asset.get("format") == "deb": + require(snapshot.entries[f"apt/pool/v/ve/{name}"] == {"size": asset["size"], "sha256": asset["sha256"]}, + "Debian package changed after input verification") + elif asset.get("format") == "rpm": + filename = f"yum/{manifest['channel']}/vector-0/{asset['arch']}/{name}" + lint_package(snapshot.source / filename, asset) + require("digests signatures OK" in run(["rpm", "--dbpath", rpmdb, "--checksig", snapshot.source / filename]).decode(), + "RPM signature verification failed") + finally: + public.close() + return manifest, snapshot + + +def publish(config, work): + manifest, snapshot = check_result(config, work) + store = store_for(config) + publisher = Publisher(store, config) + require(publisher.load_state()[0]["active"] is not None or not config.get("require_existing_repository", True), + "Historical repository backfill is required before release publication") + # Freeze signed bytes before any live write. Conditional single PUTs are atomic; + # an ambiguous response is resolved by reading the same immutable key on retry. + for name, asset in manifest["assets"].items(): + if asset.get("format") == "rpm": + relative = f"yum/{manifest['channel']}/vector-0/{asset['arch']}/{name}" + with (snapshot.source / relative).open("rb") as data: + publisher.ensure_immutable(rpm_cache_key(config, asset), data, snapshot.entries[relative]) + archive = work / "prepared.tar.gz" + if archive.exists(): + archive.unlink() + pack_bundle(work / "result", archive) + require(archive.stat().st_size < 5 * 1024**3, "Prepared bundle exceeds the single-PUT limit") + with archive.open("rb") as data: + publisher.ensure_immutable(bundle_key(config, snapshot), data, + {"size": archive.stat().st_size, "sha256": sha256_file(archive)}) + print("Published snapshot " + publisher.publish(snapshot)) + + +def verify(config, work): + manifest, snapshot = check_result(config, work) + names = {name for name in snapshot.entries if mutable(name)} + names.update(path.relative_to(snapshot.source).as_posix() for path in snapshot.source.rglob("*") if path.is_file()) + for name in sorted(names): + repository, relative = name.split("/", 1) + url = config[repository + "_url"] + require(url.startswith("https://") and url.endswith("/"), "Verification requires HTTPS repository roots") + expected = snapshot.entries[name] + digest = hashlib.sha256() + size = 0 + with urlopen(url + relative, timeout=90) as response: + require(response.geturl().startswith("https://"), "Repository redirected away from HTTPS") + while data := response.read(1024 * 1024): + size += len(data) + require(size <= expected["size"], "Endpoint returned an oversized object") + digest.update(data) + require({"sha256": digest.hexdigest(), "size": size} == expected, "Endpoint differs from published snapshot: " + name) + print(f"Verified {len(names)} endpoint objects for {manifest['tag']}") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, required=True) + commands = parser.add_subparsers(dest="command", required=True) + download = commands.add_parser("fetch") + download.add_argument("--run", type=int, required=True) + download.add_argument("--tag", required=True) + download.add_argument("--channel", choices=("stable", "beta"), required=True) + for name in ("base", "prepare", "publish", "verify", "status"): + commands.add_parser(name) + test_options = commands.add_parser("test").add_mutually_exclusive_group() + test_options.add_argument("--packages", action="store_true") + test_options.add_argument("--s3", action="store_true") + commands.add_parser("rollback").add_argument("snapshot") + recovery = commands.add_parser("recover-lock") + recovery.add_argument("--expected-version", required=True) + recovery.add_argument("--confirm-owner-stopped", action="store_true") + args = parser.parse_args() + config = configuration(args.config) + work = Path("/work") + if args.command == "fetch": + fetch(config, work, args.run, args.tag, args.channel) + print("Verified six repository packages and their published checksums") + elif args.command == "base": + base(config, work) + elif args.command == "prepare": + print("Prepared snapshot " + prepare(config, work)) + elif args.command == "publish": + publish(config, work) + elif args.command == "verify": + verify(config, work) + elif args.command == "test": + import unittest + suite = unittest.defaultTestLoader.discover("tests") + require(unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful(), "Publisher regressions failed") + if args.packages: + from tests.package_fixture import prepare_fixture + prepare_fixture(config, work) + elif args.s3: + from tests.publication_fixture import publish_fixture + publish_fixture(config, work) + else: + publisher = Publisher(store_for(config), config) + if args.command == "status": + state, head = publisher.load_state() + print(json.dumps({"state": state, "version": head["VersionId"] if head else None}, indent=2)) + elif args.command == "rollback": + print(publisher.publish(publisher.archived(args.snapshot), rollback=True)) + else: + publisher.recover_lock(args.expected_version, args.confirm_owner_stopped) + print("Lock released; resume the pending snapshot or roll back to the last completed snapshot") + + +if __name__ == "__main__": + main() diff --git a/distribution/repositories/inputs.py b/distribution/repositories/inputs.py new file mode 100644 index 0000000000000..b94f2ac7f7cf6 --- /dev/null +++ b/distribution/repositories/inputs.py @@ -0,0 +1,169 @@ +"""Unprivileged admission and download of already-built GitHub release assets.""" + +import hashlib +import json +import os +from pathlib import Path +import re +from urllib.parse import quote +from urllib.request import HTTPRedirectHandler, Request, build_opener, urlopen + +from storage import PublicationError, canonical, sha256_file + +DEB_ARCHES = ("amd64", "arm64", "armhf") +RPM_ARCHES = ("x86_64", "aarch64", "armv7hl") +TAG = re.compile(r"v0\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)(?:-([0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*))?") + + +def require(condition, message): + if not condition: + raise PublicationError(message) + + +def versions(tag, channel): + match = TAG.fullmatch(tag) + require(match is not None, "Expected a Vector 0.x semantic release tag without build metadata") + require(channel in {"stable", "beta"}, "Invalid repository channel") + require(channel != "stable" or match[1] is None, "Stable cannot publish a prerelease") + raw = tag[1:] + native = raw.replace("-", "~", 1) + return raw, native + "-1", native.replace("-", ".") + "-1" + + +def expected_packages(tag, channel): + _, deb, rpm = versions(tag, channel) + return {**{f"vector_{deb}_{arch}.deb": {"format": "deb", "arch": arch, "version": deb} + for arch in DEB_ARCHES}, + **{f"vector-{rpm}.{arch}.rpm": {"format": "rpm", "arch": arch, "version": rpm} + for arch in RPM_ARCHES}} + + +def validate_run(config, run, tag, commit): + require(run.get("repository", {}).get("id") == config["repository_id"], "Wrong source repository") + require(run.get("head_repository", {}).get("id") == config["repository_id"], "Fork runs cannot publish") + require(run.get("path") == config["workflow"], "Wrong source workflow") + require(run.get("event") == "push" and run.get("head_branch") == tag, "Source must be a tag push") + require(run.get("status") == "completed" and run.get("conclusion") == "success", "Source run did not succeed") + require(run.get("head_sha") == commit and re.fullmatch(r"[0-9a-f]{40}", commit), "Run and tag commit differ") + + +def select_assets(release, tag, channel): + raw, deb, _ = versions(tag, channel) + require(release.get("tag_name") == tag and not release.get("draft") and release.get("published_at"), + "Release must be published and match the tag") + require(channel != "stable" or not release.get("prerelease"), "Stable release is marked prerelease") + expected = expected_packages(tag, channel) + checksums = f"vector-{raw}-SHA256SUMS" + wanted = set(expected) | {checksums} + allowed_packages = set(expected) | {f"vector_{deb}_armel.deb"} + selected = {} + seen = set() + for asset in release.get("assets", []): + name = asset["name"] + require(name not in seen, "Duplicate release asset name") + seen.add(name) + require(not name.endswith((".deb", ".rpm")) or name in allowed_packages, "Unexpected Linux package: " + name) + if name not in wanted: + continue + digest = asset.get("digest", "") + require(isinstance(digest, str) and re.fullmatch(r"sha256:[0-9a-f]{64}", digest), "Asset lacks a GitHub SHA-256 digest") + require(type(asset.get("size")) is int and 0 < asset["size"] < 1024**3, "Invalid asset size") + require(type(asset.get("id")) is int and asset["id"] > 0, "Invalid asset ID") + selected[name] = {"id": asset["id"], "size": asset["size"], "sha256": digest[7:], **expected.get(name, {})} + require(set(selected) == wanted, "Release is missing required packages or SHA256SUMS") + return selected + + +def parse_checksums(data): + result = {} + for line in data.decode("utf-8").splitlines(): + match = re.fullmatch(r"([0-9a-f]{64}) [ *]([^/\\\s]+)", line) + require(match is not None, "Malformed SHA256SUMS entry") + require(match[2] not in result, "Duplicate SHA256SUMS entry") + result[match[2]] = match[1] + return result + + +def github(endpoint): + class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, request, response, code, message, headers, url): + raise PublicationError("GitHub API redirects are not allowed") + + request = Request("https://api.github.com/" + endpoint, + headers={"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", + **({"Authorization": "Bearer " + os.environ["GH_TOKEN"]} if os.environ.get("GH_TOKEN") else {})}) + with build_opener(NoRedirect).open(request, timeout=90) as response: + require(response.geturl().startswith("https://api.github.com/"), "Unexpected API redirect") + return json.load(response) + + +def fetch(config, work, run_id, tag, channel): + expected_packages(tag, channel) + repo = config["repository"] + require(repo == "vectordotdev/vector" and config["repository_id"] == 146327667, + "Release inputs must come from vectordotdev/vector") + run = github(f"repos/{repo}/actions/runs/{run_id}") + ref = github(f"repos/{repo}/git/ref/tags/{quote(tag, safe='')}")["object"] + for _ in range(4): + if ref["type"] == "commit": + break + require(ref["type"] == "tag", "Release ref does not identify a commit") + ref = github(f"repos/{repo}/git/tags/{ref['sha']}")["object"] + require(ref["type"] == "commit", "Too many annotated tag indirections") + validate_run(config, run, tag, ref["sha"]) + endpoint = f"repos/{repo}/releases/tags/{quote(tag, safe='')}" + release = github(endpoint) + assets = select_assets(release, tag, channel) + manifest = {"schema": 1, "repository": repo, "repository_id": config["repository_id"], + "run": run_id, "commit": ref["sha"], "tag": tag, "channel": channel, + "release": release["id"], "assets": assets} + destination = work / "input" + destination.mkdir(exist_ok=True) + require({p.name for p in destination.iterdir()} <= set(assets), "Unexpected files in the input directory") + if (work / "input.json").exists(): + require((work / "input.json").read_bytes() == canonical(manifest), "Previously frozen inputs have changed") + for name, asset in assets.items(): + if (destination / name).exists(): + require((destination / name).is_file() and not (destination / name).is_symlink() + and (destination / name).stat().st_size == asset["size"] + and sha256_file(destination / name) == asset["sha256"], "Previously downloaded asset differs") + continue + # Public downloads deliberately carry no GitHub credential through redirects. + url = f"https://github.com/{repo}/releases/download/{quote(tag, safe='')}/{quote(name, safe='')}" + temporary = work / "download.partial" + require(not temporary.is_symlink(), "Invalid temporary download file") + with urlopen(url, timeout=90) as source, temporary.open("wb") as target: + size = 0 + while data := source.read(1024 * 1024): + size += len(data) + require(size <= asset["size"], "Asset exceeded its declared size") + target.write(data) + require(size == asset["size"] and sha256_file(temporary) == asset["sha256"], "Asset digest/size mismatch") + temporary.replace(destination / name) + after = github(endpoint) + require(after["id"] == release["id"] and select_assets(after, tag, channel) == assets, + "Release assets changed during download") + (work / "input.json").write_bytes(canonical(manifest)) + verify_input(work) + return manifest + + +def verify_input(work): + data = (work / "input.json").read_bytes() + manifest = json.loads(data) + require(canonical(manifest) == data and manifest["schema"] == 1, "Invalid input manifest") + require(manifest["repository"] == "vectordotdev/vector" and manifest["repository_id"] == 146327667, "Wrong input repository") + expected = expected_packages(manifest["tag"], manifest["channel"]) + checksum_name = f"vector-{manifest['tag'][1:]}-SHA256SUMS" + require(set(manifest["assets"]) == set(expected) | {checksum_name}, "Input package set changed") + require({p.name for p in (work / "input").iterdir()} == set(manifest["assets"]), "Unexpected input files") + for name, asset in manifest["assets"].items(): + path = work / "input" / name + require(path.is_file() and not path.is_symlink(), "Input must be a regular file") + require(path.stat().st_size == asset["size"] and sha256_file(path) == asset["sha256"], "Input asset changed: " + name) + for field, value in expected.get(name, {}).items(): + require(asset.get(field) == value, "Input native identity changed") + sums = parse_checksums((work / "input" / checksum_name).read_bytes()) + for name in expected: + require(sums.get(name) == manifest["assets"][name]["sha256"], "SHA256SUMS disagrees with GitHub digest") + return manifest, hashlib.sha256(data).hexdigest() diff --git a/distribution/repositories/production.json b/distribution/repositories/production.json new file mode 100644 index 0000000000000..1f6b36d9dec78 --- /dev/null +++ b/distribution/repositories/production.json @@ -0,0 +1,19 @@ +{ + "schema": 1, + "require_existing_repository": true, + "repository": "vectordotdev/vector", + "repository_id": 146327667, + "workflow": ".github/workflows/release.yml", + "account": "486234852809", + "bucket": "dd-cose-releases", + "region": "us-east-1", + "live_prefix": "vector/repositories/", + "archive_prefix": "package-repositories/snapshots/", + "bundle_prefix": "package-repositories/prepared/", + "control_key": "package-repositories/control.json", + "public_key": "vector-packages.asc", + "primary_fingerprint": "FBC83025C7F26E39DBCCDCD9ED518768DAA61968", + "signing_fingerprint": "E49714E41E4A57313AACFC333C9F4A9B1ACBAD72", + "apt_url": "https://apt.vector.dev/", + "yum_url": "https://yum.vector.dev/" +} diff --git a/distribution/repositories/repository.py b/distribution/repositories/repository.py new file mode 100644 index 0000000000000..6d9f647a7308c --- /dev/null +++ b/distribution/repositories/repository.py @@ -0,0 +1,403 @@ +"""Native package validation and incremental apt/yum metadata construction.""" + +import gzip +import hashlib +import os +from pathlib import Path +import shutil +import subprocess +import tarfile +import tempfile +import time +from datetime import datetime, timezone +import xml.etree.ElementTree as ET + +from inputs import DEB_ARCHES, RPM_ARCHES, require, verify_input +from storage import Snapshot, byte_entry, mutable, safe_relative, sha256_file + + +def run(args, *, cwd=None, data=None, env=None): + result = subprocess.run([str(arg) for arg in args], cwd=cwd, input=data, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, check=False) + require(result.returncode == 0, f"{args[0]} failed: " + result.stderr.decode(errors="replace")[-4000:]) + return result.stdout + + +def fields(paragraph): + result = {} + for line in paragraph.splitlines(): + if line.startswith((" ", "\t")): + continue + key, separator, value = line.partition(":") + require(separator and key not in result, "Malformed or duplicate package metadata field") + result[key] = value.strip() + return result + + +def merge_packages(previous, incoming): + packages = {} + for content in (previous, incoming): + for paragraph in content.decode().strip().split("\n\n"): + if not paragraph: + continue + item = fields(paragraph) + require(item.get("Package") == "vector", "Non-Vector Debian package") + identity = (item["Package"], item["Version"], item["Architecture"]) + old = packages.get(identity) + require(old is None or fields(old) == item, "Conflicting Debian package identity") + packages[identity] = paragraph + return ("\n\n".join(packages[key] for key in sorted(packages)) + ("\n\n" if packages else "")).encode() + + +def lint_package(path, identity): + if identity["format"] == "deb": + for field, expected in (("Package", "vector"), ("Version", identity["version"]), ("Architecture", identity["arch"])): + require(run(["dpkg-deb", "-f", path, field]).decode().strip() == expected, "Debian native identity mismatch") + process = subprocess.Popen(["dpkg-deb", "--fsys-tarfile", str(path)], stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL) + try: + size = 0 + with tarfile.open(fileobj=process.stdout, mode="r|") as archive: + for member in archive: + size += member.size + require(size < 8 * 1024**3, "Unreasonably large package payload") + require(member.issym() or member.islnk() or not member.mode & 0o002, + "World-writable Debian package entry: " + member.name) + require(process.wait() == 0, "Cannot inspect Debian payload") + finally: + process.stdout.close() + if process.poll() is None: + process.kill() + process.wait() + else: + actual = run(["rpm", "-qp", "--qf", "%{NAME} %{VERSION}-%{RELEASE} %{ARCH}", path]).decode() + require(actual == f"vector {identity['version']} {identity['arch']}", "RPM native identity mismatch") + listing = run(["rpm", "-qp", "--qf", "[%{FILEMODES:perms} %{FILENAMES}\\n]", path]).decode() + for line in listing.splitlines(): + require(len(line) >= 10, "Invalid RPM file listing") + require(line[0] == "l" or line[8] != "w", "World-writable RPM package entry: " + line[11:]) + + +def key_fingerprints(listing, primary_type, subkey_type): + result = [] + pending = None + for line in listing.decode().splitlines(): + parts = line.split(":") + if parts[0] in (primary_type, subkey_type): + pending = parts + elif parts[0] == "fpr" and pending is not None: + result.append((pending, parts[9])) + pending = None + return result + + +class PublicKey: + def __init__(self, config, directory): + self.config = config + self.home = Path(directory) + self.home.mkdir(mode=0o700, parents=True) + self.env = {**os.environ, "GNUPGHOME": str(self.home)} + self.key = Path(config["public_key_path"]) + listing = self.gpg("--with-colons", "--show-keys", str(self.key)) + keys = key_fingerprints(listing, "pub", "sub") + require([fpr for parts, fpr in keys if parts[0] == "pub"] == [config["primary_fingerprint"]], "Wrong public primary key") + require(config["signing_fingerprint"] in [fpr for parts, fpr in keys if parts[0] == "sub"], "Wrong public signing subkey") + self.gpg("--import", str(self.key)) + + def gpg(self, *args, data=None): + return run(["gpg", "--batch", "--no-tty", *args], data=data, env=self.env) + + def verify(self, signature, data=None): + args = ["--status-fd", "1", "--verify", str(signature)] + if data is not None: + args.append(str(data)) + status = self.gpg(*args).decode() + require(any(line.startswith("[GNUPG:] VALIDSIG " + self.config["signing_fingerprint"] + " ") + for line in status.splitlines()), "Signature did not use the pinned signing subkey") + + def close(self): + run(["gpgconf", "--kill", "all"], env=self.env) + shutil.rmtree(self.home) + + +class Signer(PublicKey): + def __init__(self, config, directory, private_key, passphrase): + super().__init__(config, directory) + require(private_key and passphrase and "\n" not in passphrase and "\r" not in passphrase, + "An encrypted signing subkey and a single-line passphrase are required") + self.password = self.home / "passphrase" + self.password.write_text(passphrase + "\n") + self.password.chmod(0o600) + self.gpg("--import", data=private_key.encode()) + keys = key_fingerprints(self.gpg("--with-colons", "--list-secret-keys"), "sec", "ssb") + require(len(keys) == 2, "Expected exactly one primary stub and one signing subkey") + primary, subkey = keys + require(primary[1] == config["primary_fingerprint"] and primary[0][0] == "sec" + and len(primary[0]) > 14 and primary[0][14] == "#", "Usable primary secret key is forbidden in CI") + require(subkey[1] == config["signing_fingerprint"] and subkey[0][0] == "ssb" + and len(subkey[0]) > 14 and subkey[0][14] == "+" and "s" in subkey[0][11], + "Wrong or unavailable signing subkey") + for parts, _ in keys: + require(parts[1] not in ("r", "e", "d") and parts[6] and int(parts[6]) > time.time(), "Signing identity is expired or revoked") + challenge = self.home / "challenge" + challenge.write_text("Vector repository signing capability check\n") + run(["gpgconf", "--kill", "gpg-agent"], env=self.env) + wrong = subprocess.run(["gpg", "--batch", "--no-tty", "--pinentry-mode", "loopback", + "--passphrase-fd", "0", "--local-user", config["signing_fingerprint"] + "!", + "--output", str(self.home / "wrong-passphrase.asc"), "--detach-sign", str(challenge)], + input=os.urandom(32).hex().encode() + b"\n", env=self.env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False) + require(wrong.returncode != 0, "Signing subkey must be passphrase-protected") + self.sign(challenge) + self.verify(challenge.with_suffix(".asc"), challenge) + self.rpmdb = self.home / "rpmdb" + run(["rpm", "--dbpath", self.rpmdb, "--import", self.key]) + + def sign(self, source, *, clear=False, destination=None): + destination = destination or Path(str(source) + ".asc") + self.gpg("--yes", "--pinentry-mode", "loopback", "--passphrase-file", str(self.password), + "--local-user", self.config["signing_fingerprint"] + "!", "--digest-algo", "SHA256", + "--armor", "--output", str(destination), "--clearsign" if clear else "--detach-sign", str(source)) + + def rpm(self, package): + run(["rpmsign", "--define", "_gpg_name " + self.config["signing_fingerprint"] + "!", + "--define", "_gpg_path " + str(self.home), "--define", "__gpg /usr/bin/gpg", + "--define", "_gpg_digest_algo sha256", "--define", + "_gpg_sign_cmd_extra_args --pinentry-mode loopback --passphrase-file " + str(self.password), + "--addsign", package], env=self.env) + output = run(["rpm", "--dbpath", self.rpmdb, "--checksig", package]).decode() + require("digests signatures OK" in output, "Signed RPM failed native verification") + + +def repomd_locations(root): + document = ET.fromstring((root / "repodata/repomd.xml").read_bytes()) + result = [] + for entry in document.findall("{*}data"): + location = entry.find("{*}location") + require(location is not None and set(location.attrib) == {"href"}, "Unexpected RPM metadata base URL") + name = location.attrib["href"] + safe_relative("yum/metadata/" + name) + require(name.startswith("repodata/") and len(Path(name).parts) == 2, "Invalid RPM metadata location") + result.append(name) + require(result, "Empty RPM root metadata") + return result + + +def check_apt_index(data, snapshot): + for paragraph in data.decode().strip().split("\n\n"): + if not paragraph: + continue + item = fields(paragraph) + require(item.get("Package") == "vector", "Foreign package in apt history") + name = "apt/" + item["Filename"] + safe_relative(name) + require(snapshot.entries.get(name) == {"sha256": item["SHA256"], "size": int(item["Size"])}, + "Debian history does not match the snapshot manifest") + + +def verify_metadata(snapshot, root, public): + for name in snapshot.entries: + if name.endswith("/InRelease"): + suite = (root / name).parent + public.verify(suite / "InRelease") + public.verify(suite / "Release.gpg", suite / "Release") + release = (suite / "Release").read_bytes() + require(public.gpg("--decrypt", str(suite / "InRelease")) == release, "Inline and detached apt roots disagree") + algorithm = None + checked = set() + for line in release.decode().splitlines(): + if not line.startswith(" "): + algorithm = "sha256" if line == "SHA256:" else None + elif algorithm: + digest, size, relative = line.split() + safe_relative("apt/index/" + relative) + entry = {"sha256": digest, "size": int(size)} + filename = (suite / relative).relative_to(root).as_posix() + require(snapshot.entries.get(filename) == entry, "Apt index differs from its signed root") + if relative.endswith("/Packages"): + require(byte_entry((suite / relative).read_bytes()) == entry, "Apt index checksum mismatch") + check_apt_index((suite / relative).read_bytes(), snapshot) + checked.add(filename) + require(checked, "Apt root has no SHA-256 package indexes") + elif name.endswith("/repomd.xml"): + package_root = (root / name).parent.parent + public.verify(Path(str(root / name) + ".asc"), root / name) + document = ET.fromstring((root / name).read_bytes()) + for item in document.findall("{*}data"): + location = item.find("{*}location") + require(location is not None and set(location.attrib) == {"href"}, "RPM metadata must use relative locations") + relative = location.attrib["href"] + safe_relative("yum/metadata/" + relative) + path = package_root / relative + entry = snapshot.entries.get(path.relative_to(root).as_posix()) + checksum = item.find("{*}checksum") + require(checksum is not None and checksum.attrib.get("type") in {"sha256", "sha512"}, "RPM metadata needs SHA-2 checksums") + data = path.read_bytes() + require(entry == byte_entry(data) and hashlib.new(checksum.attrib["type"], data).hexdigest() == checksum.text, + "RPM metadata differs from its signed root") + if item.attrib.get("type") == "primary": + require(path.suffix == ".gz", "Expected gzip RPM primary metadata") + primary = ET.fromstring(gzip.decompress(data)) + for package in primary.findall("{*}package"): + require(package.findtext("{*}name") == "vector", "Foreign package in RPM history") + url = package.find("{*}location") + checksum = package.find("{*}checksum") + size = package.find("{*}size") + require(url is not None and set(url.attrib) == {"href"}, "RPM package must use a relative location") + safe_relative("yum/package/" + url.attrib["href"]) + filename = (package_root / url.attrib["href"]).relative_to(root).as_posix() + require(checksum is not None and checksum.attrib.get("type") == "sha256" and size is not None, + "RPM package needs a SHA-256 checksum and size") + require(snapshot.entries.get(filename) == {"sha256": checksum.text, "size": int(size.attrib["package"])}, + "RPM package differs from its signed index") + + +def apt_indexes(root, channel, incoming, previous, signer): + suite = root / "apt/dists" / channel + architectures = (*DEB_ARCHES, "x86_64", "all", "i386") + for arch in architectures: + relative = f"apt/dists/{channel}/vector-0/binary-{arch}/Packages" + old = (previous / relative).read_bytes() if (previous / relative).exists() else b"" + added = incoming.get("amd64" if arch == "x86_64" else arch, b"") + data = merge_packages(old, added) + index = root / relative + index.parent.mkdir(parents=True, exist_ok=True) + index.write_bytes(data) + index.with_suffix(".gz").write_bytes(gzip.compress(data, mtime=0)) + for path in (index, index.with_suffix(".gz")): + for algorithm in ("sha256", "sha512"): + content = path.read_bytes() + hashed = path.parent / "by-hash" / algorithm.upper() / hashlib.new(algorithm, content).hexdigest() + hashed.parent.mkdir(parents=True, exist_ok=True) + hashed.write_bytes(content) + lines = ["Origin: Vector", "Label: Vector", f"Suite: {channel}", f"Codename: {channel}", + "Date: " + datetime.now(timezone.utc).strftime("%a, %d %b %Y %H:%M:%S +0000"), + "Architectures: " + " ".join(architectures), "Components: vector-0", "Acquire-By-Hash: yes"] + for algorithm in ("sha256", "sha512"): + lines.append(algorithm.upper() + ":") + for arch in architectures: + for filename in ("Packages", "Packages.gz"): + relative = f"vector-0/binary-{arch}/{filename}" + data = (suite / relative).read_bytes() + lines.append(f" {hashlib.new(algorithm, data).hexdigest()} {len(data)} {relative}") + release = suite / "Release" + release.write_text("\n".join(lines) + "\n") + signer.sign(release, clear=True, destination=suite / "InRelease") + signer.sign(release, destination=suite / "Release.gpg") + signer.verify(suite / "InRelease") + signer.verify(suite / "Release.gpg", release) + + +def prepare(config, work): + private_key = os.environ.pop("PACKAGE_SIGNING_SUBKEY", "") + passphrase = os.environ.pop("PACKAGE_SIGNING_PASSPHRASE", "") + manifest, input_id = verify_input(work) + require(not (work / "result").exists(), "Prepared output already exists; reuse it or select a new work directory") + base_data = (work / "base.json").read_bytes() + previous = Snapshot.decode(base_data) if base_data != b"null\n" else None + base_files = work / "base" + # Check every downloaded base file before a private key is imported. + for path in base_files.rglob("*"): + require(not path.is_symlink(), "Base metadata cannot contain symlinks") + if path.is_file(): + name = path.relative_to(base_files).as_posix() + require(previous is not None and previous.entries.get(name) == byte_entry(path.read_bytes()), "Base metadata changed") + for name, identity in manifest["assets"].items(): + if "format" in identity: + lint_package(work / "input" / name, identity) + public = PublicKey(config, "/run/keys/verify") + try: + if previous: + verify_metadata(previous, base_files, public) + finally: + public.close() + signer = None + try: + signer = Signer(config, "/run/keys/sign", private_key, passphrase) + private_key = passphrase = None + result = work / "result" + root = result / "files" + root.mkdir(parents=True) + entries = dict(previous.entries) if previous else {} + if previous: + for source in base_files.rglob("*"): + if source.is_file(): + path = root / source.relative_to(base_files) + path.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, path) + incoming = {} + channel = manifest["channel"] + for name, identity in manifest["assets"].items(): + if "format" not in identity: + continue + relative = (f"apt/pool/v/ve/{name}" if identity["format"] == "deb" + else f"yum/{channel}/vector-0/{identity['arch']}/{name}") + if relative in entries: + require(identity["format"] == "deb" and entries[relative] == + {"size": identity["size"], "sha256": identity["sha256"]}, + "Package is already present; reuse its saved signed publication") + destination = root / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(work / "input" / name, destination) + if identity["format"] == "rpm": + cached = work / "reuse" / name + if cached.exists(): + require(cached.is_file() and not cached.is_symlink(), "Invalid saved signed RPM") + shutil.copyfile(cached, destination) + lint_package(destination, identity) + require("digests signatures OK" in run(["rpm", "--dbpath", signer.rpmdb, "--checksig", destination]).decode(), + "Saved RPM does not have a valid COSE signature") + else: + signer.rpm(destination) + else: + incoming[identity["arch"]] = run(["apt-ftparchive", "packages", relative[4:]], cwd=root / "apt") + for selected in ("stable", "beta"): + if previous is None or selected == channel: + apt_indexes(root, selected, incoming if selected == channel else {}, base_files, signer) + for arch in RPM_ARCHES: + relative = f"yum/{selected}/vector-0/{arch}" + target = root / relative + target.mkdir(parents=True, exist_ok=True) + # Previously copied mutable roots are not input to createrepo. + shutil.rmtree(target / "repodata", ignore_errors=True) + run(["createrepo_c", "--no-database", "--checksum", "sha256", "--compress-type", "gz", target]) + old = base_files / relative + if (old / "repodata/repomd.xml").exists(): + with tempfile.TemporaryDirectory() as temporary: + run(["mergerepo_c", "--all", "--omit-baseurl", "--no-database", "--compress-type", "gz", + "--repo", old.as_uri(), "--repo", target.as_uri(), "--outputdir", temporary]) + shutil.rmtree(target / "repodata") + shutil.copytree(Path(temporary) / "repodata", target / "repodata") + repomd_locations(target) + signer.sign(target / "repodata/repomd.xml") + signer.verify(target / "repodata/repomd.xml.asc", target / "repodata/repomd.xml") + for path in root.rglob("*"): + if path.is_file(): + name = path.relative_to(root).as_posix() + entry = {"size": path.stat().st_size, "sha256": sha256_file(path)} + require(mutable(name) or name not in entries or entries[name] == entry, "Immutable repository object changed") + entries[name] = entry + snapshot = Snapshot(entries, root, base=previous.id if previous else None, input_id=input_id) + # Metadata inherited from an untouched channel may refer to files held only + # in the base workspace. Validate a metadata-only view without copying RPMs. + with tempfile.TemporaryDirectory() as temporary: + view = Path(temporary) + for source in (base_files, root): + for path in source.rglob("*"): + if path.is_file() and not path.name.endswith((".deb", ".rpm")): + destination = view / path.relative_to(source) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(path, destination) + verify_metadata(snapshot, view, signer) + (result / "manifest.json").write_bytes(snapshot.data) + (result / "input.json").write_bytes((work / "input.json").read_bytes()) + Snapshot.local(root) + return snapshot.id + finally: + if signer is not None: + signer.close() + else: + # A failed key check can leave an initialized agent; stop it before cleanup. + if Path("/run/keys/sign").exists(): + run(["gpgconf", "--kill", "all"], env={**os.environ, "GNUPGHOME": "/run/keys/sign"}) + shutil.rmtree("/run/keys/sign") diff --git a/distribution/repositories/requirements.txt b/distribution/repositories/requirements.txt new file mode 100644 index 0000000000000..d27f4c469d3e6 --- /dev/null +++ b/distribution/repositories/requirements.txt @@ -0,0 +1,8 @@ +# Pinned AWS SDK and transitive dependencies; wheels from PyPI. +boto3==1.43.114 --hash=sha256:d9cac2eb921ce674970cef1c9ad750f85ee3a846aedcf188d18368fb9eb6da23 +botocore==1.43.115 --hash=sha256:9ceda640c868bc15f5399129e8cfdef8cbb4b350e8a99547c9b5f477cfb9619a +jmespath==1.1.0 --hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64 +python-dateutil==2.9.0.post0 --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 +s3transfer==0.19.3 --hash=sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553 +six==1.17.0 --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 +urllib3==2.8.0 --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 diff --git a/distribution/repositories/storage.py b/distribution/repositories/storage.py new file mode 100644 index 0000000000000..cb2e2236eae33 --- /dev/null +++ b/distribution/repositories/storage.py @@ -0,0 +1,396 @@ +"""Conditional S3 publication protocol, invoked through vdev release repositories.""" + +import base64 +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta, timezone +from email.utils import parsedate_to_datetime +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import sys +import uuid + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + + +class PublicationError(RuntimeError): + pass + + +class Busy(PublicationError): + pass + + +class Conflict(PublicationError): + pass + + +class Interrupted(PublicationError): + """A deliberate test interruption after all preceding S3 requests finished.""" + + +MUTABLE_NAMES = {"Packages", "Packages.gz", "Release", "Release.gpg", "InRelease", + "repomd.xml", "repomd.xml.asc"} +ROOT_ORDER = {"Release": 0, "Release.gpg": 1, "InRelease": 2, + "repomd.xml": 3, "repomd.xml.asc": 4} + + +def canonical(value): + return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode() + + +def sha256_file(path): + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def safe_relative(name): + if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9_./+~\-]+", name): + raise PublicationError("Invalid manifest path") + path = PurePosixPath(name) + if path.is_absolute() or str(path) != name or ".." in path.parts: + raise PublicationError("Noncanonical manifest path") + if len(path.parts) < 3 or path.parts[0] not in {"apt", "yum"}: + raise PublicationError("Manifest path is outside the repository roots") + return path + + +def mutable(name): + return safe_relative(name).name in MUTABLE_NAMES + + +class Snapshot: + def __init__(self, entries, source=None, *, base=None, input_id=None): + if not isinstance(entries, dict) or not entries: + raise PublicationError("Empty or invalid snapshot manifest") + for name, entry in entries.items(): + safe_relative(name) + if not isinstance(entry, dict) or set(entry) != {"sha256", "size"}: + raise PublicationError("Invalid manifest entry") + if not re.fullmatch(r"[0-9a-f]{64}", entry["sha256"]): + raise PublicationError("Invalid manifest checksum") + if type(entry["size"]) is not int or not 0 <= entry["size"] <= 5 * 1024**3: + raise PublicationError("Only single-PUT objects up to 5 GiB are supported") + if not any(mutable(name) for name in entries): + raise PublicationError("Snapshot has no repository metadata") + self.entries = entries + self.source = source + for value in (base, input_id): + if value is not None and not re.fullmatch(r"[0-9a-f]{64}", value): + raise PublicationError("Invalid snapshot identity") + self.base = base + self.input_id = input_id + self.data = canonical({"schema": 2, "base": base, "input": input_id, "files": entries}) + self.id = hashlib.sha256(self.data).hexdigest() + + @classmethod + def decode(cls, data, expected_id=None): + value = json.loads(data) + if set(value) != {"schema", "base", "input", "files"} or value["schema"] != 2: + raise PublicationError("Unsupported snapshot schema") + snapshot = cls(value["files"], base=value["base"], input_id=value["input"]) + if data != snapshot.data or (expected_id is not None and snapshot.id != expected_id): + raise PublicationError("Snapshot manifest is not canonical or has the wrong digest") + return snapshot + + @classmethod + def local(cls, source): + source = Path(source) + if source.is_symlink() or not source.is_dir(): + raise PublicationError("Snapshot must be a regular directory") + snapshot = cls.decode((source.parent / "manifest.json").read_bytes()) + entries = {} + for path in sorted(source.rglob("*")): + if path.is_symlink(): + raise PublicationError("Snapshot symlinks are not allowed") + if path.is_dir(): + continue + if not path.is_file(): + raise PublicationError("Snapshot contains a nonregular file") + name = path.relative_to(source).as_posix() + safe_relative(name) + entries[name] = {"sha256": sha256_file(path), "size": path.stat().st_size} + for name, entry in entries.items(): + if snapshot.entries.get(name) != entry: + raise PublicationError("Snapshot content differs from recorded manifest: " + name) + for name in snapshot.entries: + if name not in entries and (mutable(name) or snapshot.base is None): + raise PublicationError("Snapshot is missing a required local file: " + name) + snapshot.source = source + return snapshot + + +class S3Store: + def __init__(self, config): + # Capture one fixed STS session. Never refresh credentials while owning a lock. + required = ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN") + if not all(os.environ.get(name) for name in required): + raise PublicationError("A fixed temporary AWS session is required") + session = boto3.Session( + aws_access_key_id=os.environ[required[0]], + aws_secret_access_key=os.environ[required[1]], + aws_session_token=os.environ[required[2]], + region_name=config["region"], + ) + # Disable automatic retries: an ambiguous mutable PUT must retain its lock. + sdk = Config(retries={"total_max_attempts": 1}, max_pool_connections=16, + connect_timeout=10, read_timeout=90, + s3={"us_east_1_regional_endpoint": "regional"}) + identity = session.client("sts", config=sdk).get_caller_identity() + role = config["role"].rsplit("/", 1)[1] + expected = f'arn:aws:sts::{config["account"]}:assumed-role/{role}/' + if identity["Account"] != config["account"] or not identity["Arn"].startswith(expected): + raise PublicationError("Wrong AWS account or role") + actual_role = session.client("iam", config=sdk).get_role(RoleName=role)["Role"] + if actual_role["MaxSessionDuration"] > 3600: + raise PublicationError("Role exceeds the one-hour recovery credential bound") + self.client = session.client("s3", config=sdk.merge(Config(signature_version="s3v4"))) + self.bucket = config["bucket"] + self.live_prefix = config["live_prefix"] + self.owner = identity["Arn"] + self.server_time = parsedate_to_datetime(identity["ResponseMetadata"]["HTTPHeaders"]["date"]) + + def response_time(self, response): + self.server_time = parsedate_to_datetime(response["ResponseMetadata"]["HTTPHeaders"]["date"]) + + def head(self, key): + try: + result = self.client.head_object(Bucket=self.bucket, Key=key, ChecksumMode="ENABLED") + except ClientError as exc: + self.response_time(exc.response) + status = exc.response["ResponseMetadata"]["HTTPStatusCode"] + if status == 404: + return None + if status == 403: + # Prefix-scoped ListBucket may hide missing keys behind HEAD 403. + # An authorized exact-prefix listing distinguishes absence from denial. + listed = self.client.list_objects_v2(Bucket=self.bucket, Prefix=key, MaxKeys=1) + self.response_time(listed) + if not any(item["Key"] == key for item in listed.get("Contents", [])): + return None + raise + self.response_time(result) + return result + + def read(self, key): + result = self.client.get_object(Bucket=self.bucket, Key=key, ChecksumMode="ENABLED") + self.response_time(result) + with result["Body"] as stream: + data = stream.read() + return data, result + + def put(self, key, body, entry, *, previous=None, immutable=False): + options = {"IfNoneMatch": "*"} if previous is None else {"IfMatch": previous["ETag"]} + result = self.client.put_object( + Bucket=self.bucket, Key=key, Body=body, ContentLength=entry["size"], + ChecksumSHA256=base64.b64encode(bytes.fromhex(entry["sha256"])).decode(), + ServerSideEncryption="AES256", ContentType=content_type(key), + CacheControl=(("public" if key.startswith(self.live_prefix) else "private") + + (",max-age=31536000,immutable" if immutable + else ",max-age=0,no-cache,must-revalidate")), **options, + ) + self.response_time(result) + return result + + def download(self, key, destination): + result = self.client.get_object(Bucket=self.bucket, Key=key, ChecksumMode="ENABLED") + self.response_time(result) + digest = hashlib.sha256() + with result["Body"] as source, destination.open("wb") as target: + for block in iter(lambda: source.read(1024 * 1024), b""): + digest.update(block) + target.write(block) + return {"sha256": digest.hexdigest(), "size": destination.stat().st_size} + + +def content_type(key): + if key.endswith(".deb"): + return "application/vnd.debian.binary-package" + if key.endswith(".rpm"): + return "application/x-rpm" + if key.endswith(".json"): + return "application/json" + if key.endswith(".gz"): + return "application/gzip" + if key.endswith(".xml"): + return "application/xml" + return "application/octet-stream" + + +def matches(head, entry): + return bool(head and head.get("ChecksumSHA256") == base64.b64encode( + bytes.fromhex(entry["sha256"])).decode() and head["ContentLength"] == entry["size"]) + + +def byte_entry(data): + return {"sha256": hashlib.sha256(data).hexdigest(), "size": len(data)} + + +class Publisher: + def __init__(self, store, config, hook=None): + self.store = store + self.live = config["live_prefix"] + self.archives = config["archive_prefix"] + self.lock_key = config["control_key"] + self.hook = hook or (lambda phase: None) + self.lock_head = None + self.state = None + + def load_state(self): + head = self.store.head(self.lock_key) + if head is None: + return {"schema": 1, "held": False, "active": None, "pending": None}, None + data, current = self.store.read(self.lock_key) + state = json.loads(data) + if state.get("schema") != 1 or type(state.get("held")) is not bool: + raise PublicationError("Invalid publication control object") + return state, current + + def write_state(self): + self.state["nonce"] = uuid.uuid4().hex # Prevent an ABA match on lock state. + data = canonical(self.state) + self.lock_head = self.store.put(self.lock_key, data, byte_entry(data), previous=self.lock_head) + + def acquire(self): + self.state, self.lock_head = self.load_state() + if self.state["held"]: + raise Busy("Publisher busy; locks are never stolen on timeout") + self.state.update(held=True, owner=self.store.owner, + recover_after=(self.store.server_time + timedelta(minutes=65)).isoformat()) + try: + self.write_state() + except ClientError as exc: + if exc.response["ResponseMetadata"]["HTTPStatusCode"] in (409, 412): + raise Busy("Another publisher acquired the lock") from None + raise + self.hook("locked") + + def release(self): + self.state["held"] = False + self.write_state() + + def recover_lock(self, expected_version, owner_stopped): + self.state, self.lock_head = self.load_state() + if not owner_stopped: + raise PublicationError("Confirm that the previous workflow/process has stopped") + if not self.state["held"] or self.lock_head["VersionId"] != expected_version: + raise Conflict("Lock version changed or is already released") + if self.store.server_time < datetime.fromisoformat(self.state["recover_after"]): + raise Busy("Recovery waits 65 minutes so the previous fixed STS session expires") + # Conditional replacement cannot unlock a different owner's newer lock. + self.release() + + def ensure_immutable(self, key, body, entry): + old = self.store.head(key) + if old is not None: + if not matches(old, entry): + raise Conflict("Immutable object conflict: " + key) + return + self.store.put(key, body, entry, immutable=True) + + def archive(self, snapshot): + prefix = self.archives + snapshot.id + "/" + for name, entry in sorted(snapshot.entries.items()): + if mutable(name): + with (snapshot.source / name).open("rb") as stream: + self.ensure_immutable(prefix + "files/" + name, stream, entry) + self.ensure_immutable(prefix + "manifest.json", snapshot.data, byte_entry(snapshot.data)) + + def archived(self, snapshot_id): + if not re.fullmatch(r"[0-9a-f]{64}", snapshot_id): + raise PublicationError("Invalid snapshot ID") + data, _ = self.store.read(self.archives + snapshot_id + "/manifest.json") + return Snapshot.decode(data, snapshot_id) + + def preflight(self, snapshot, rollback): + active = self.state["active"] + if not rollback and active != snapshot.id: + if active != snapshot.base: + raise Conflict("Repository advanced after preparation; regenerate against the current base") + if active: + previous = self.archived(active) + for name, entry in previous.entries.items(): + if not mutable(name) and snapshot.entries.get(name) != entry: + raise Conflict("Publication would remove or replace historical data: " + name) + names = sorted(snapshot.entries) + with ThreadPoolExecutor(max_workers=16) as workers: + heads = list(workers.map(self.store.head, (self.live + name for name in names))) + for name, old in zip(names, heads): + entry = snapshot.entries[name] + if not mutable(name): + if old is not None and not matches(old, entry): + raise Conflict("Immutable object conflict: " + name) + if rollback and old is None: + raise Conflict("Rollback payload/index no longer exists: " + name) + if old is None and (snapshot.source is None or not (snapshot.source / name).is_file()): + raise Conflict("Inherited payload/index no longer exists: " + name) + + def publish(self, snapshot, *, rollback=False): + self.acquire() + try: + pending = self.state["pending"] + if pending and snapshot.id != pending and not ( + rollback and snapshot.id == self.state["active"] + ): + raise Conflict("Recover the pending snapshot or roll back to the last completed one") + self.preflight(snapshot, rollback) + # Nothing above changes client-visible metadata. + if not rollback: + self.archive(snapshot) + metadata = {} + for name, entry in snapshot.entries.items(): + if mutable(name): + data, _ = self.store.read(self.archives + snapshot.id + "/files/" + name) + if byte_entry(data) != entry: + raise Conflict("Archived metadata failed digest verification: " + name) + metadata[name] = data + self.state["pending"] = snapshot.id + self.write_state() + for name, entry in sorted(snapshot.entries.items()): + if not mutable(name) and not rollback: + # Preflight verified inherited immutable objects under the lock. + if not (snapshot.source / name).exists(): + continue + if matches(self.store.head(self.live + name), entry): + continue + with (snapshot.source / name).open("rb") as stream: + self.ensure_immutable(self.live + name, stream, entry) + self.hook("payloads") + indexes = sorted(name for name in snapshot.entries + if PurePosixPath(name).name in {"Packages", "Packages.gz"}) + roots = sorted((name for name in snapshot.entries if mutable(name) and name not in indexes), + key=lambda name: (ROOT_ORDER[PurePosixPath(name).name], name)) + for name in indexes + roots: + entry = snapshot.entries[name] + key = self.live + name + previous = self.store.head(key) + if matches(previous, entry): + continue + self.store.put(key, metadata[name], entry, previous=previous) + self.hook("written:" + name) + names = sorted(snapshot.entries) + with ThreadPoolExecutor(max_workers=16) as workers: + heads = list(workers.map(self.store.head, (self.live + name for name in names))) + for name, head in zip(names, heads): + if not matches(head, snapshot.entries[name]): + raise PublicationError("Published object failed final verification: " + name) + self.state.update(active=snapshot.id, pending=None) + self.write_state() + except (Conflict, Interrupted): + # These are deliberate, synchronous stops with no outstanding AWS request. + self.release() + raise + except BaseException: + # A lost response can hide a completed PUT. Never unlock automatically. + print("Publication stopped; the lock remains held for operator recovery", file=sys.stderr) + raise + else: + self.release() + return snapshot.id diff --git a/distribution/repositories/tests/client.sh b/distribution/repositories/tests/client.sh new file mode 100644 index 0000000000000..f071f809e8020 --- /dev/null +++ b/distribution/repositories/tests/client.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail +version=${TAG#v} +version=${version/-/\~} +if command -v apt-get >/dev/null; then + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y ca-certificates gnupg + gpg --batch --dearmor --output /usr/share/keyrings/vector.gpg /key.asc + echo "deb [signed-by=/usr/share/keyrings/vector.gpg] $APT_URL $CHANNEL vector-0" > /etc/apt/sources.list.d/vector.list + apt-get update + apt-get install -y "vector=$version-1" + test "$(dpkg-query -W -f='${Version}' vector)" = "$version-1" +else + version=${version//-/.} + rpm --import /key.asc + cat > /etc/yum.repos.d/vector.repo <&2 + exit 1 + ;; +esac + install_package () { case "$1" in *.deb) @@ -70,5 +94,3 @@ getent group vector || (echo "vector group missing" && exit 1) vector --version || (echo "vector --version failed" && exit 1) grep -q "FOO=bar" "/etc/default/vector" || (echo "/etc/default/vector has incorrect contents" && exit 1) grep -q "foo: bar" "/etc/vector/vector.yaml" || (echo "user-provided /etc/vector/vector.yaml was not preserved on reinstall" && exit 1) - -dd-pkg lint "$package" diff --git a/vdev/src/commands/release/mod.rs b/vdev/src/commands/release/mod.rs index 1f4fea8d2c4c9..5d8689bb88bd2 100644 --- a/vdev/src/commands/release/mod.rs +++ b/vdev/src/commands/release/mod.rs @@ -2,6 +2,7 @@ mod channel; mod generate_cue; mod github; mod prepare; +mod repositories; mod workflow; use anyhow::{Result, ensure}; @@ -40,6 +41,7 @@ enum Commands { GenerateCue(generate_cue::Cli), Github(github::Cli), Prepare(prepare::Cli), + Repositories(repositories::Cli), Workflow(workflow::Cli), /// Uploads archives and packages to AWS S3 S3(ScriptArgs), @@ -53,6 +55,7 @@ impl Cli { Commands::GenerateCue(cli) => cli.exec(), Commands::Github(cli) => cli.exec(), Commands::Prepare(cli) => cli.exec(), + Commands::Repositories(cli) => cli.exec(), Commands::Workflow(cli) => cli.exec(), Commands::S3(args) => args.exec("release-s3.sh"), } diff --git a/vdev/src/commands/release/repositories.rs b/vdev/src/commands/release/repositories.rs new file mode 100644 index 0000000000000..908220ceab0ab --- /dev/null +++ b/vdev/src/commands/release/repositories.rs @@ -0,0 +1,248 @@ +use std::{path::PathBuf, process::Command}; + +use anyhow::{Context as _, Result, ensure}; + +use crate::app::{self, CommandExt as _}; + +/// Build, sign, publish, and recover the apt/yum repositories. +#[derive(clap::Args, Debug)] +pub(super) struct Cli { + /// Container built with `build-image` from the reviewed publisher source. + #[arg(long, default_value = "vector-package-publisher:local", global = true)] + image: String, + /// Configuration and public key directory; never contains a private key. + #[arg( + long, + default_value = "distribution/repositories/production.json", + global = true + )] + config: PathBuf, + /// Persistent workspace exchanged between input, signing, and publishing jobs. + #[arg(long, default_value = "target/package-repositories", global = true)] + work: PathBuf, + #[command(subcommand)] + command: Commands, +} + +#[derive(clap::Subcommand, Debug)] +enum Commands { + /// Build the tools image locally; no registry publication is required. + BuildImage, + /// Validate a completed Release Suite run and freeze its release assets. + Fetch { + #[arg(long)] + run: u64, + #[arg(long)] + tag: String, + #[arg(long, value_enum, default_value = "stable")] + channel: Channel, + }, + /// Read current metadata or restore the previously signed retry bundle. + Base, + /// Validate inputs, import the signing subkey, and generate signed metadata. + Prepare, + /// Save the signed retry bundle and publish with conditional S3 writes. + Publish, + /// Verify the configured public HTTPS endpoints against the signed snapshot. + Verify, + /// Print the publication state and its S3 version for operator inspection. + Status, + /// Restore archived metadata; this does not downgrade installed packages. + Rollback { snapshot: String }, + /// Release an abandoned lock after the fixed credential safety window. + RecoverLock { + #[arg(long)] + expected_version: String, + /// Confirm the previous process and its in-flight writes have stopped. + #[arg(long)] + confirm_owner_stopped: bool, + }, + /// Run the publisher's offline protocol and metadata regressions. + Test { + /// Prepare two releases in the workspace with an ephemeral test key. + #[arg(long, conflicts_with = "s3")] + packages: bool, + /// Exercise prepared fixtures against the isolated private S3 sandbox. + #[arg(long)] + s3: bool, + }, +} + +#[derive(clap::ValueEnum, Clone, Copy, Debug)] +enum Channel { + Stable, + Beta, +} + +impl Cli { + pub fn exec(self) -> Result<()> { + app::set_repo_dir()?; + if matches!(self.command, Commands::BuildImage) { + return Command::new("docker") + .args(["build", "--tag", &self.image, "distribution/repositories"]) + .check_run(); + } + std::fs::create_dir_all(&self.work)?; + let work = self.work.canonicalize()?; + let config = self + .config + .canonicalize() + .context("Publisher configuration is missing")?; + let directory = config.parent().context("Configuration has no parent")?; + ensure!(config.is_file(), "Configuration must be a regular file"); + let mut command = Command::new("docker"); + command.args(["run", "--rm", "--init"]); + let key_mount = { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt as _; + + let owner = work.metadata()?; + command.args(["--user", &format!("{}:{}", owner.uid(), owner.gid())]); + format!( + "/run/keys:rw,noexec,nosuid,size=16m,mode=0700,uid={},gid={}", + owner.uid(), + owner.gid() + ) + } + #[cfg(not(unix))] + { + "/run/keys:rw,noexec,nosuid,size=16m,mode=0700".to_owned() + } + }; + command.args([ + "--cap-drop=ALL", + "--security-opt=no-new-privileges", + "--tmpfs", + &key_mount, + "--env", + "HOME=/run/keys", + "--mount", + &format!("type=bind,source={},target=/work", work.display()), + "--mount", + &format!( + "type=bind,source={},target=/config,readonly", + directory.display() + ), + ]); + // Forward names, never values: command logging must not expose credentials. + for name in self.command.credentials() { + command.args(["--env", name]); + } + command.arg(&self.image).args([ + "--config", + &format!( + "/config/{}", + config + .file_name() + .context("Missing configuration filename")? + .to_string_lossy() + ), + ]); + self.command.add_arguments(&mut command); + command.check_run() + } +} + +impl Commands { + fn credentials(&self) -> &'static [&'static str] { + match self { + Self::Fetch { .. } => &["GH_TOKEN"], + Self::Prepare => &["PACKAGE_SIGNING_SUBKEY", "PACKAGE_SIGNING_PASSPHRASE"], + Self::Base + | Self::Publish + | Self::Status + | Self::Rollback { .. } + | Self::RecoverLock { .. } + | Self::Test { s3: true, .. } => &[ + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "PACKAGE_REPOSITORIES_ROLE_ARN", + ], + _ => &[], + } + } + + fn add_arguments(self, command: &mut Command) { + match self { + Commands::Fetch { run, tag, channel } => { + command.args([ + "fetch", + "--run", + &run.to_string(), + "--tag", + &tag, + "--channel", + match channel { + Channel::Stable => "stable", + Channel::Beta => "beta", + }, + ]); + } + Commands::Base => { + command.arg("base"); + } + Commands::Prepare => { + command.arg("prepare"); + } + Commands::Publish => { + command.arg("publish"); + } + Commands::Verify => { + command.arg("verify"); + } + Commands::Status => { + command.arg("status"); + } + Commands::Rollback { snapshot } => { + command.args(["rollback", &snapshot]); + } + Commands::RecoverLock { + expected_version, + confirm_owner_stopped, + } => { + command.args(["recover-lock", "--expected-version", &expected_version]); + if confirm_owner_stopped { + command.arg("--confirm-owner-stopped"); + } + } + Commands::Test { packages, s3 } => { + command.arg("test"); + if packages { + command.arg("--packages"); + } + if s3 { + command.arg("--s3"); + } + } + Commands::BuildImage => unreachable!(), + } + } +} + +#[cfg(test)] +mod tests { + use super::Commands; + + #[test] + fn signing_and_publishing_receive_disjoint_credentials() { + let signing = Commands::Prepare.credentials(); + let publishing = Commands::Publish.credentials(); + assert_eq!( + signing, + ["PACKAGE_SIGNING_SUBKEY", "PACKAGE_SIGNING_PASSPHRASE"] + ); + assert!(publishing.contains(&"AWS_SESSION_TOKEN")); + assert!(signing.iter().all(|name| !publishing.contains(name))); + assert!(Commands::Verify.credentials().is_empty()); + assert!( + Commands::Test { + packages: true, + s3: false + } + .credentials() + .is_empty() + ); + } +} From e0eb2fba636c37a9fb28bd637a943a3d3ce0c80e Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 20 Oct 2026 09:38:43 -0400 Subject: [PATCH 02/23] feat(ci): add COSE package repository setup and migration --- .github/workflows/package-repositories.yml | 4 +- distribution/repositories/.dockerignore | 2 + distribution/repositories/repository.py | 5 +- distribution/repositories/setup.sh | 312 ++++++++++++++++++ distribution/repositories/tests/client.sh | 16 +- .../repositories/tests/installer-client.sh | 237 +++++++++++++ .../repositories/tests/installer_fixture.py | 103 ++++++ .../tests/legacy-apt-identity.json | 9 + .../repositories/tests/package_fixture.py | 2 + .../repositories/tests/test_inputs.py | 22 +- docs/reference/package-repositories.md | 71 ++++ 11 files changed, 764 insertions(+), 19 deletions(-) create mode 100644 distribution/repositories/setup.sh create mode 100644 distribution/repositories/tests/installer-client.sh create mode 100644 distribution/repositories/tests/installer_fixture.py create mode 100644 distribution/repositories/tests/legacy-apt-identity.json diff --git a/.github/workflows/package-repositories.yml b/.github/workflows/package-repositories.yml index c458f45a23bd9..2cad16562698c 100644 --- a/.github/workflows/package-repositories.yml +++ b/.github/workflows/package-repositories.yml @@ -222,7 +222,7 @@ jobs: with: name: package-repository-inputs path: target/package-repositories - - name: Install through the public repository with the pinned public key + - name: Configure and install through the public repository with pinned trust env: IMAGE: ${{ matrix.image }} run: | @@ -232,6 +232,6 @@ jobs: export APT_URL=https://apt.vector.dev/ export YUM_URL=https://yum.vector.dev/ docker run --rm --env CHANNEL --env TAG --env APT_URL --env YUM_URL \ - --mount "type=bind,source=$PWD/distribution/repositories/vector-packages.asc,target=/key.asc,readonly" \ + --mount "type=bind,source=$PWD/distribution/repositories/setup.sh,target=/setup.sh,readonly" \ --mount "type=bind,source=$PWD/distribution/repositories/tests/client.sh,target=/test.sh,readonly" \ "$IMAGE" bash /test.sh diff --git a/distribution/repositories/.dockerignore b/distribution/repositories/.dockerignore index 856e6bdc83ef4..68a5289c7a3be 100644 --- a/distribution/repositories/.dockerignore +++ b/distribution/repositories/.dockerignore @@ -4,3 +4,5 @@ !*.py !tests/ !tests/** + +**/__pycache__ diff --git a/distribution/repositories/repository.py b/distribution/repositories/repository.py index 6d9f647a7308c..7ebd2cc9ed2b3 100644 --- a/distribution/repositories/repository.py +++ b/distribution/repositories/repository.py @@ -270,7 +270,10 @@ def apt_indexes(root, channel, incoming, previous, signer): hashed = path.parent / "by-hash" / algorithm.upper() / hashlib.new(algorithm, content).hexdigest() hashed.parent.mkdir(parents=True, exist_ok=True) hashed.write_bytes(content) - lines = ["Origin: Vector", "Label: Vector", f"Suite: {channel}", f"Codename: {channel}", + # Preserve the existing public repository identity during the key/hosting + # migration. Adding Origin/Label or changing the empty Suite would require + # a separate apt release-information approval on clients with cached indexes. + lines = ["Suite:", f"Codename: {channel}", "Date: " + datetime.now(timezone.utc).strftime("%a, %d %b %Y %H:%M:%S +0000"), "Architectures: " + " ".join(architectures), "Components: vector-0", "Acquire-By-Hash: yes"] for algorithm in ("sha256", "sha512"): diff --git a/distribution/repositories/setup.sh b/distribution/repositories/setup.sh new file mode 100644 index 0000000000000..027d040819736 --- /dev/null +++ b/distribution/repositories/setup.sh @@ -0,0 +1,312 @@ +#!/usr/bin/env bash +# Configure Vector's apt/yum repository. This script does not install Vector. +set -euo pipefail + +fail() { printf 'Vector repository setup: %s\n' "$*" >&2; exit 1; } +usage() { + cat <<'EOF' +Usage: bash setup.sh [--migrate] [--cose-only] + +With no existing Vector source, configure the stable repository with COSE trust. +With an existing vector.list or vector.repo, preserve its settings and signing +keys while adding COSE trust. --migrate requires an existing source. +After cutover, --cose-only removes legacy trust from Vector's own configuration. +Do not use --cose-only while the repository is still signed by Datadog. + +Fresh setup options: + VECTOR_DIST_CHANNEL stable (default) or beta + VECTOR_MAJOR_VERSION 0 (default) + VEC_REPO_URL vector.dev (default repository domain) + TESTING_APT_URL HTTPS apt host/path or URL + TESTING_YUM_URL HTTPS yum host/path or URL + TESTING_APT_REPO_VERSION " vector-" + TESTING_YUM_VERSION_PATH "/vector-" + +Both modes: + VECTOR_PUBLIC_KEY_URL HTTPS armored public-key URL + VECTOR_PUBLIC_KEY_FINGERPRINT Full primary fingerprint for an owned mirror + TESTING_KEYS_URL Compatibility alias: HTTPS host/path containing vector-packages.asc + CSM_MIGRATE=true Disable the old timber-vector source, with backup + +Existing sources keep their channel, URL and options; fresh setup overrides are +rejected during migration. Package and repository signatures remain required. +Other source filenames and deb822 files require manual migration. +EOF +} + +require_existing=false +retain_legacy=true +while (($#)); do + case "$1" in + --migrate) require_existing=true ;; + --cose-only) retain_legacy=false ;; + --help|-h) usage; exit 0 ;; + *) fail "Unknown option: $1. Use --help." ;; + esac + shift +done + +[[ "$(uname -s)" == Linux ]] || fail "This script supports Linux apt/yum clients." +sudo_cmd=() +if ((EUID != 0)); then + command -v sudo >/dev/null || fail "Run this script as root or install sudo." + sudo_cmd=(sudo) + "${sudo_cmd[@]}" -v +fi + +if [[ -f /etc/debian_version ]] && command -v apt-get >/dev/null; then + family=apt + source_file=/etc/apt/sources.list.d/vector.list + key_file=/usr/share/keyrings/vector-archive-keyring.gpg + timber_file=/etc/apt/sources.list.d/timber-vector.list +elif command -v rpm >/dev/null && { command -v dnf >/dev/null || command -v yum >/dev/null; }; then + family=rpm + source_file=/etc/yum.repos.d/vector.repo + key_file=/etc/pki/rpm-gpg/RPM-GPG-KEY-vector + timber_file=/etc/yum.repos.d/timber-vector.repo + package_manager=yum + if command -v dnf >/dev/null; then package_manager=dnf; fi +else + fail "No supported apt/yum package manager found." +fi + +case "${CSM_MIGRATE:-false}" in true|false) ;; *) fail "CSM_MIGRATE must be true or false." ;; esac +[[ "${VEC_RPM_REPO_GPGCHECK:-1}" == 1 ]] || fail "Repository signature verification cannot be disabled." +[[ -z "${REPO_URL:-}" ]] || fail "Use VEC_REPO_URL for a custom repository domain." + +# Never follow a managed-file symlink or replace a directory/device. +regular_or_absent() { + [[ ! -L "$1" && ( ! -e "$1" || -f "$1" ) ]] || fail "Expected a regular file: $1" +} +for path in "$source_file" "$source_file.pre-cose" "$key_file" "$key_file.pre-cose"; do + regular_or_absent "$path" +done +existing=false +if [[ -f "$source_file" ]]; then existing=true; fi +if "$require_existing" && ! "$existing"; then fail "No $source_file found; migrate custom sources manually."; fi +if "$existing"; then + for option in VECTOR_DIST_CHANNEL VECTOR_MAJOR_VERSION VEC_REPO_URL TESTING_APT_URL TESTING_YUM_URL \ + TESTING_APT_REPO_VERSION TESTING_YUM_VERSION_PATH; do + [[ -z "${!option:-}" ]] || fail "$option applies to fresh setup only; existing source settings are preserved." + done +fi + +# Refuse ambiguous parallel Vector definitions instead of creating conflicting +# Signed-By settings or rewriting a file belonging to another package. +shopt -s nullglob +if [[ "$family" == apt ]]; then + candidates=(/etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources) +else + candidates=(/etc/yum.repos.d/*.repo) +fi +for path in "${candidates[@]}"; do + [[ -f "$path" && "$path" != "$source_file" ]] || continue + if [[ "$path" == "$timber_file" && "${CSM_MIGRATE:-false}" == true ]]; then continue; fi + if awk '!/^[[:space:]]*[#;]/ && /vector-[0-9]+|^\[vector\]/ { found=1 } END { exit !found }' "$path"; then + fail "Additional Vector source found in $path; migrate it manually before running setup." + fi +done +if [[ "${CSM_MIGRATE:-false}" == true && -e "$timber_file" ]]; then + regular_or_absent "$timber_file" + regular_or_absent "$timber_file.pre-cose" + [[ ! -e "$timber_file.pre-cose" ]] || fail "A timber source backup already exists; inspect it before retrying." + grep -q 'repositories.timber.io' "$timber_file" || fail "Unrecognized timber source; leaving it unchanged." +fi + +work=$(mktemp -d) +cleanup() { + if command -v gpgconf >/dev/null; then GNUPGHOME="$work/gnupg" gpgconf --kill all >/dev/null 2>&1 || true; fi + rm -rf "$work" +} +trap cleanup EXIT +mkdir -m 700 "$work/gnupg" +export GNUPGHOME="$work/gnupg" + +https_url() { + local value=$1 + if [[ "$value" != *://* ]]; then value="https://$value"; fi + [[ "$value" =~ ^https://[A-Za-z0-9][A-Za-z0-9._:/%+~-]*$ ]] || fail "Expected an HTTPS URL without credentials, query, or whitespace." + printf '%s' "${value%/}" +} +default_key_url=https://dd-cose-releases.s3.us-east-1.amazonaws.com/vector/keys/vector-packages.asc +if [[ -n "${TESTING_KEYS_URL:-}" ]]; then default_key_url="$(https_url "$TESTING_KEYS_URL")/vector-packages.asc"; fi +key_url=$(https_url "${VECTOR_PUBLIC_KEY_URL:-$default_key_url}") +fingerprint=${VECTOR_PUBLIC_KEY_FINGERPRINT:-FBC83025C7F26E39DBCCDCD9ED518768DAA61968} +[[ "$fingerprint" =~ ^[0-9A-F]{40}$ ]] || fail "The primary fingerprint must contain exactly 40 uppercase hexadecimal characters." +old_keyrings=() +old_key_urls=() + +if "$existing" && [[ "$family" == apt ]]; then + mapfile -t active < <(awk 'NF && !/^[[:space:]]*#/ { print }' "$source_file") + ((${#active[@]} == 1)) || fail "$source_file must contain exactly one active deb source; migrate custom files manually." + apt_line='^[[:space:]]*deb[[:space:]]+\[([^]]+)\][[:space:]]+(https://[^[:space:]]+)[[:space:]]+(stable|beta)[[:space:]]+(vector-[0-9]+)[[:space:]]*(#.*)?$' + [[ "${active[0]}" =~ $apt_line ]] || fail "Unrecognized apt source; expected HTTPS, stable/beta, and a Vector component with Signed-By." + options=${BASH_REMATCH[1]} + signed_by= + for option in $options; do + case "${option,,}" in + trusted=*|allow-insecure=*|allow-weak=*|allow-downgrade-to-insecure=*) + case "${option##*=}" in no|false|0) ;; *) fail "Remove insecure apt trust overrides before migration." ;; esac + ;; + esac + if [[ "$option" == signed-by=* ]]; then + [[ -z "$signed_by" ]] || fail "Multiple Signed-By options require manual migration." + signed_by=${option#signed-by=} + fi + done + [[ -n "$signed_by" ]] || fail "An explicit Signed-By keyring is required for migration." + if "$retain_legacy"; then + IFS=, read -r -a old_keyrings <<< "$signed_by" + for path in "${old_keyrings[@]}"; do + [[ "$path" =~ ^/[A-Za-z0-9_./-]+\.(gpg|asc)$ && -f "$path" && -r "$path" ]] || + fail "Cannot preserve Signed-By entry $path; migrate fingerprint selectors or unavailable keys manually." + done + fi + awk -v key="$key_file" '/^[[:space:]]*deb[[:space:]]/ { sub(/signed-by=[^] \t]+/, "signed-by=" key) } { print }' \ + "$source_file" > "$work/source" +elif "$existing"; then + if [[ $(grep -c '^[[:space:]]*\[' "$source_file") != 1 ]] || + ! grep -Eq '^[[:space:]]*\[vector\][[:space:]]*$' "$source_file"; then + fail "$source_file must contain only the [vector] repository; migrate custom files manually." + fi + # Snapshot old trust locally; a migrated client must not depend on Datadog's + # key endpoint remaining available after retirement. + if "$retain_legacy"; then + awk ' + /^[[:space:]]*gpgkey[[:space:]]*=/ { sub(/^[^=]*=[[:space:]]*/, ""); keys=1; print; next } + keys && /^[[:space:]]+[^#;]/ && !/^[[:space:]]*[a-zA-Z_]+[[:space:]]*=/ { print; next } + /^[[:space:]]*([#;]|$)/ { next } + { keys=0 } + ' "$source_file" > "$work/old-key-urls" + while read -r line; do + line=${line%%#*} + for url in $line; do + case "$url" in + https://*) old_key_urls+=("$(https_url "$url")") ;; + file:///*) + path=${url#file://} + [[ "$path" =~ ^/[A-Za-z0-9_./-]+$ && -f "$path" && -r "$path" ]] || fail "Cannot read the old RPM key $path." + old_keyrings+=("$path") + ;; + *) fail "Old RPM keys must use HTTPS or a readable local file." ;; + esac + done + done < "$work/old-key-urls" + ((${#old_key_urls[@]} + ${#old_keyrings[@]} > 0)) || fail "No existing RPM gpgkey found; preserve legacy trust manually." + fi + # DNF maintains a separate metadata keyring, so configure gpgkey as well as + # importing the combined public keys into the RPM database. + awk -v key="file://$key_file" ' + BEGIN { keys=0; seen=0 } + /^[[:space:]]*gpgkey[[:space:]]*=/ { + if (seen++) exit 2 + print "gpgkey=" key + keys=1 + next + } + keys && /^[[:space:]]+[^#;]/ && !/^[[:space:]]*[a-zA-Z_]+[[:space:]]*=/ { next } + /^[[:space:]]*([#;]|$)/ { print; next } + { keys=0 } + /^[[:space:]]*(gpgcheck|repo_gpgcheck)[[:space:]]*=/ { next } + { print } + END { + if (!seen) print "gpgkey=" key + print "gpgcheck=1" + print "repo_gpgcheck=1" + }' "$source_file" > "$work/source" || fail "Duplicate gpgkey settings require manual migration." +else + channel=${VECTOR_DIST_CHANNEL:-stable} + major=${VECTOR_MAJOR_VERSION:-0} + [[ "$channel" == stable || "$channel" == beta ]] || fail "VECTOR_DIST_CHANNEL must be stable or beta." + [[ "$major" =~ ^(0|[1-9][0-9]*)$ ]] || fail "VECTOR_MAJOR_VERSION must be a nonnegative integer." + domain=${VEC_REPO_URL:-vector.dev} + [[ "$domain" =~ ^[A-Za-z0-9][A-Za-z0-9.-]*$ ]] || fail "VEC_REPO_URL must be a domain name." + if [[ "$family" == apt ]]; then + url=$(https_url "${TESTING_APT_URL:-apt.$domain}") + selection=${TESTING_APT_REPO_VERSION:-$channel vector-$major} + [[ "$selection" =~ ^(stable|beta)\ vector-(0|[1-9][0-9]*)$ ]] || fail "Invalid apt channel/component override." + printf 'deb [signed-by=%s] %s/ %s\n' "$key_file" "$url" "$selection" > "$work/source" + else + url=$(https_url "${TESTING_YUM_URL:-yum.$domain}") + selection=${TESTING_YUM_VERSION_PATH:-$channel/vector-$major} + [[ "$selection" =~ ^(stable|beta)/vector-(0|[1-9][0-9]*)$ ]] || fail "Invalid yum channel/component override." + cat > "$work/source" </dev/null || ! command -v gpg >/dev/null || [[ ! -s /etc/ssl/certs/ca-certificates.crt && ! -s /etc/pki/tls/certs/ca-bundle.crt ]]; then + if [[ "$family" == apt ]]; then + # A stale Vector key must not prevent repairing its trust. Distro package + # installation still performs its normal authentication checks. + "${sudo_cmd[@]}" apt-get update || printf 'Some indexes could not refresh; attempting prerequisite installation.\n' >&2 + "${sudo_cmd[@]}" env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates curl gnupg + else + "${sudo_cmd[@]}" "$package_manager" -y --disablerepo=vector install ca-certificates curl gnupg2 + fi +fi + +curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' \ + --connect-timeout 15 --max-time 120 --retry 3 "$key_url" -o "$work/key.asc" +gpg --batch --quiet --import "$work/key.asc" +mapfile -t imported < <(gpg --batch --with-colons --list-keys | awk '$1 ~ /^pub:/ { pub=1; next } pub && /^fpr:/ { split($0,f,":"); print f[10]; pub=0 }') +[[ ${#imported[@]} == 1 && "${imported[0]}" == "$fingerprint" ]] || fail "Downloaded public key does not match the pinned primary fingerprint." +gpg --batch --with-colons --list-keys "$fingerprint" > "$work/key.info" +awk -F: -v now="$(date +%s)" '$1=="pub" { if ($2=="r" || $2=="e" || ($7!="" && $7!=0 && $7<=now)) exit 1 }' \ + "$work/key.info" || fail "The pinned public key is expired or revoked." +gpg --batch --armor --export "$fingerprint" > "$work/cose.asc" + +for path in "${old_keyrings[@]}"; do gpg --batch --quiet --import "$path"; done +for url in "${old_key_urls[@]}"; do + curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' \ + --connect-timeout 15 --max-time 120 --retry 3 "$url" -o "$work/old-key.asc" + gpg --batch --quiet --import "$work/old-key.asc" +done +if [[ "$family" == apt ]]; then + gpg --batch --export > "$work/key" +else + gpg --batch --armor --export > "$work/key" +fi +[[ -s "$work/key" ]] || fail "Empty public keyring." + +backup() { + if [[ -f "$1" && ! -e "$1.pre-cose" ]]; then "${sudo_cmd[@]}" cp -p "$1" "$1.pre-cose"; fi +} +atomic_install() { + local source=$1 destination=$2 temporary + "${sudo_cmd[@]}" install -d -m 755 "$(dirname "$destination")" + temporary=$("${sudo_cmd[@]}" mktemp "$(dirname "$destination")/.vector-setup.XXXXXX") + if ! "${sudo_cmd[@]}" install -o root -g root -m 644 "$source" "$temporary" || + ! "${sudo_cmd[@]}" mv -fT "$temporary" "$destination"; then + "${sudo_cmd[@]}" rm -f "$temporary" + fail "Could not install $destination." + fi +} +backup "$source_file" +backup "$key_file" +atomic_install "$work/key" "$key_file" +atomic_install "$work/source" "$source_file" +if [[ "${CSM_MIGRATE:-false}" == true && -f "$timber_file" ]]; then + "${sudo_cmd[@]}" mv "$timber_file" "$timber_file.pre-cose" +fi + +printf 'Vector trust and repository configuration written. Refreshing only the Vector repository.\n' +if [[ "$family" == apt ]]; then + "${sudo_cmd[@]}" apt-get update -o Dir::Etc::sourcelist=sources.list.d/vector.list \ + -o Dir::Etc::sourceparts=- -o APT::Get::List-Cleanup=0 -o APT::Update::Error-Mode=any || + fail "Configuration was saved, but the repository refresh failed. Check the error above and rerun." +else + "${sudo_cmd[@]}" rpm --import "$key_file" + "${sudo_cmd[@]}" "$package_manager" -y --disablerepo='*' --enablerepo=vector clean metadata + "${sudo_cmd[@]}" "$package_manager" -y --disablerepo='*' --enablerepo=vector makecache || + fail "Configuration was saved, but the repository refresh failed. Check the error above and rerun." +fi +printf 'Vector repository setup complete. Install or upgrade Vector separately with your package manager.\n' diff --git a/distribution/repositories/tests/client.sh b/distribution/repositories/tests/client.sh index f071f809e8020..fa9c7c6c6f4f0 100644 --- a/distribution/repositories/tests/client.sh +++ b/distribution/repositories/tests/client.sh @@ -2,27 +2,13 @@ set -euo pipefail version=${TAG#v} version=${version/-/\~} +VECTOR_DIST_CHANNEL="$CHANNEL" TESTING_APT_URL="$APT_URL" TESTING_YUM_URL="$YUM_URL" bash /setup.sh if command -v apt-get >/dev/null; then export DEBIAN_FRONTEND=noninteractive - apt-get update -qq - apt-get install -y ca-certificates gnupg - gpg --batch --dearmor --output /usr/share/keyrings/vector.gpg /key.asc - echo "deb [signed-by=/usr/share/keyrings/vector.gpg] $APT_URL $CHANNEL vector-0" > /etc/apt/sources.list.d/vector.list - apt-get update apt-get install -y "vector=$version-1" test "$(dpkg-query -W -f='${Version}' vector)" = "$version-1" else version=${version//-/.} - rpm --import /key.asc - cat > /etc/yum.repos.d/vector.repo </dev/null || true + if [[ -n "${source_file:-}" && -f "$source_file" ]]; then cat "$source_file"; fi + if command -v apt-cache >/dev/null; then apt-cache policy vector || true; fi + fi + [[ -z "$server_pid" ]] || kill "$server_pid" 2>/dev/null || true + rm -rf "$work" /run/installer-keys/* +} +trap cleanup EXIT +trap 'printf "Installer test failed at line %s\n" "$LINENO" >&2' ERR +mkdir -p "$work/bundle" "$work/http/keys" /run/installer-keys +tar -xf /bundle.tar -C "$work/bundle" +cp "$work/bundle/public/"* "$work/http/keys/" +select_view() { + ln -s "$work/bundle/views/$1" "$work/http/next" + mv -Tf "$work/http/next" "$work/http/current" +} +select_view legacy +openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj /CN=localhost \ + -addext 'subjectAltName=DNS:localhost,IP:127.0.0.1' \ + -keyout /run/installer-keys/tls.key -out /run/installer-keys/tls.crt > "$work/command.log" 2>&1 +if command -v apt-get >/dev/null; then + family=apt + install -m 644 /run/installer-keys/tls.crt /usr/local/share/ca-certificates/vector-installer-test.crt + update-ca-certificates > "$work/command.log" 2>&1 +else + family=rpm + install -m 644 /run/installer-keys/tls.crt /etc/pki/ca-trust/source/anchors/vector-installer-test.crt + update-ca-trust > "$work/command.log" 2>&1 +fi +python3 - "$work/http" > "$work/http.log" 2>&1 <<'PY' & +import http.server, os, socketserver, ssl, sys +class Handler(http.server.SimpleHTTPRequestHandler): + protocol_version = "HTTP/1.1" + def send_head(self): + # The archived views were copied together and may share a filesystem + # mtime. Always return the selected bytes, while apt retains its cached + # Release identity for comparison. CDN caching is a separate test. + if "If-Modified-Since" in self.headers: + del self.headers["If-Modified-Since"] + return super().send_head() +# AlmaLinux 8's system Python predates ThreadingHTTPServer and the directory +# parameter. Keep the fixture compatible without changing the client runtime. +class Server(socketserver.ThreadingMixIn, http.server.HTTPServer): + daemon_threads = True +os.chdir(sys.argv[1]) +server = Server(("127.0.0.1", 18766), Handler) +context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) +context.load_cert_chain("/run/installer-keys/tls.crt", "/run/installer-keys/tls.key") +server.socket = context.wrap_socket(server.socket, server_side=True) +server.serve_forever() +PY +server_pid=$! +for _ in $(seq 1 30); do curl -fsS https://localhost:18766/ >/dev/null 2>&1 && break; sleep .2; done +curl -fsS https://localhost:18766/ >/dev/null +export VECTOR_PUBLIC_KEY_URL=https://localhost:18766/keys/cose.asc +VECTOR_PUBLIC_KEY_FINGERPRINT=$(cat "$work/bundle/public/fingerprint") +export VECTOR_PUBLIC_KEY_FINGERPRINT +base=https://localhost:18766/current +pass() { printf 'PASS: %s %s: %s\n' "$family" "$scenario" "$*"; } +setup() { bash /setup.sh "$@"; } +expect_failure() { + local pattern=$1; shift + if "$@" > "$work/command.log" 2>&1; then printf 'Unexpected success: %s\n' "$*" >&2; exit 1; fi + grep -Eiq "$pattern" "$work/command.log" || { cat "$work/command.log"; exit 1; } +} +if [[ "$family" == apt ]]; then + source_file=/etc/apt/sources.list.d/vector.list + shared_key=/usr/share/keyrings/datadog-archive-keyring.gpg + owned_key=/usr/share/keyrings/vector-archive-keyring.gpg + timber_file=/etc/apt/sources.list.d/timber-vector.list + refresh() { apt-get -o Dir::Etc::sourcelist=sources.list.d/vector.list -o Dir::Etc::sourceparts=- -o APT::Get::List-Cleanup=0 -o APT::Update::Error-Mode=any update; } + install_version() { DEBIAN_FRONTEND=noninteractive apt-get -y --allow-downgrades install "vector=$1-1"; } + # Resolve normal distro dependencies separately; later refreshes retain the + # actual cached Vector Release information across the simulated cutover. + apt-get update > "$work/command.log" 2>&1 +else + source_file=/etc/yum.repos.d/vector.repo + shared_key=/etc/pki/rpm-gpg/LEGACY-INSTALLER-TEST + owned_key=/etc/pki/rpm-gpg/RPM-GPG-KEY-vector + timber_file=/etc/yum.repos.d/timber-vector.repo + refresh() { + dnf -y --disablerepo='*' --enablerepo=vector clean metadata + dnf -y --disablerepo='*' --enablerepo=vector --refresh makecache + } + install_version() { dnf -y install "vector-$1-1"; } +fi + +if [[ "$scenario" == fresh ]]; then + select_view upgraded + expect_failure 'requires an existing|No .* found' setup --migrate + expect_failure 'cannot be disabled' env VEC_RPM_REPO_GPGCHECK=0 bash /setup.sh + expect_failure 'HTTPS URL' env VECTOR_PUBLIC_KEY_URL=http://localhost/key bash /setup.sh + expect_failure 'pinned primary fingerprint' env VECTOR_PUBLIC_KEY_URL=https://localhost:18766/keys/legacy.asc bash /setup.sh + [[ ! -e "$source_file" && ! -e "$owned_key" ]] + pass 'invalid trust inputs fail before creating repository files' + + # Both legacy override spellings accept HTTPS URLs and retain strict checks. + VECTOR_DIST_CHANNEL=beta TESTING_APT_URL="$base/apt" TESTING_YUM_URL="$base/yum" setup > "$work/command.log" 2>&1 + grep -q beta "$source_file" + setup --migrate > "$work/command.log" 2>&1 + grep -q beta "$source_file" + pass 'fresh beta setup and repeated migration retain the channel' + rm "$source_file" "$source_file.pre-cose" + TESTING_APT_REPO_VERSION='stable vector-0' TESTING_YUM_VERSION_PATH=stable/vector-0 \ + TESTING_APT_URL="$base/apt" TESTING_YUM_URL="$base/yum" setup > "$work/command.log" 2>&1 + if [[ "$family" == apt ]]; then + [[ $(gpg --batch --with-colons --show-keys "$owned_key" | grep -c '^pub:') == 1 ]] + else + grep -qx 'gpgcheck=1' "$source_file" + grep -qx 'repo_gpgcheck=1' "$source_file" + if grep -q legacy "$source_file"; then exit 1; fi + fi + install_version 0.60.0 > "$work/command.log" 2>&1 + vector --version | grep -F 'vector 0.60.0 ' + getent passwd vector >/dev/null + getent group vector >/dev/null + pass 'fresh stable installation uses only COSE trust over verified HTTPS' + exit 0 +fi + +if [[ "$family" == apt ]]; then + install -m 644 "$work/bundle/public/legacy.gpg" "$shared_key" + printf '# existing Vector settings\ndeb [arch=%s signed-by=%s] %s/apt/ stable vector-0\n' \ + "$(dpkg --print-architecture)" "$shared_key" "$base" > "$source_file" +else + install -m 644 "$work/bundle/public/legacy.asc" "$shared_key" + rpm --import "$shared_key" + cat > "$source_file" < "$work/command.log" 2>&1 +install_version 0.59.0 > "$work/command.log" 2>&1 +vector --version | grep -F 'vector 0.59.0 ' +mkdir -p /etc/vector +printf '# preserve installer migration configuration\n' > /etc/vector/vector.yaml +account=$(id vector) +pass 'legacy installation succeeds with cached metadata and legacy-only trust' + +expect_failure 'pinned primary fingerprint' env VECTOR_PUBLIC_KEY_URL=https://localhost:18766/keys/legacy.asc bash /setup.sh --migrate +expect_failure 'fresh setup only' env VECTOR_DIST_CHANNEL=beta bash /setup.sh --migrate +if [[ "$family" == apt ]]; then + printf 'Types: deb\nURIs: %s/apt\nSuites: stable\nComponents: vector-0\n' "$base" > /etc/apt/sources.list.d/custom-vector.sources + expect_failure 'Additional Vector source' setup --migrate + rm /etc/apt/sources.list.d/custom-vector.sources +else + # shellcheck disable=SC2016 + printf '[other-vector]\nbaseurl=%s/yum/stable/vector-0/$basearch/\n' "$base" > /etc/yum.repos.d/custom-vector.repo + expect_failure 'Additional Vector source' setup --migrate + rm /etc/yum.repos.d/custom-vector.repo +fi +[[ "$source_hash" == "$(sha256sum "$source_file")" && ! -e "$owned_key" && ! -e "$source_file.pre-cose" ]] +pass 'wrong key, channel override and ambiguous custom sources leave existing files untouched' + +if [[ "$scenario" == late ]]; then + select_view cose + expect_failure 'NO_PUBKEY|public key|GPG|signature' refresh + pass 'a client that missed the announcement rejects the new key before migration' +else + # The opt-in legacy Cloudsmith file is disabled with a recoverable backup. + printf '# old source at https://repositories.timber.io/public/vector\n' > "$timber_file" + export CSM_MIGRATE=true +fi +setup --migrate > "$work/command.log" 2>&1 +[[ "$shared_hash" == "$(sha256sum "$shared_key")" ]] +[[ $(sha256sum "$source_file.pre-cose" | cut -d ' ' -f 1) == $(sha256sum "$work/original-source" | cut -d ' ' -f 1) ]] +first_source_hash=$(sha256sum "$source_file") +first_key_hash=$(sha256sum "$owned_key") +# Subsequent migration and metadata verification must no longer need the old +# remote key endpoint. The shared key on disk remains available to other apps. +rm "$work/http/keys/legacy.asc" +setup --migrate > "$work/command.log" 2>&1 +[[ "$first_source_hash" == "$(sha256sum "$source_file")" && "$first_key_hash" == "$(sha256sum "$owned_key")" ]] +if [[ "$family" == apt ]]; then + [[ $(gpg --batch --with-colons --show-keys "$owned_key" | grep -c '^pub:') == 2 ]] + grep -q "arch=$(dpkg --print-architecture)" "$source_file" +else + grep -qx "gpgkey=file://$owned_key" "$source_file" + [[ $(gpg --batch --with-colons --show-keys "$owned_key" | grep -c '^pub:') == 2 ]] + grep -qx priority=7 "$source_file" + grep -qx exclude=other-package "$source_file" + grep -qx gpgcheck=1 "$source_file" + grep -qx repo_gpgcheck=1 "$source_file" +fi +pass 'migration is byte-idempotent, retains legacy trust and options, and leaves shared keys unchanged' +if [[ "$scenario" == early ]]; then + [[ ! -e "$timber_file" && -f "$timber_file.pre-cose" ]] + refresh > "$work/command.log" 2>&1 + pass 'pre-cutover migration still authenticates the legacy repository' + select_view cose + refresh > "$work/command.log" 2>&1 + pass 'cutover succeeds with cached apt identity and no release-information bypass' +fi +select_view upgraded +refresh > "$work/refresh.log" 2>&1 +install_version 0.60.0 > "$work/command.log" 2>&1 +vector --version | grep -F 'vector 0.60.0 ' +grep -q 'preserve installer migration configuration' /etc/vector/vector.yaml +[[ "$account" == "$(id vector)" ]] +[[ "$shared_hash" == "$(sha256sum "$shared_key")" ]] +pass 'upgrade to 0.60.0 over verified HTTPS preserves configuration and account' + +# After cutover, legacy trust can be retired from the Vector source even if the +# old key endpoint has gone away. Other packages' shared key files stay intact. +install -m 644 "$work/original-source" "$source_file" +if [[ "$family" == apt ]]; then + sed -i "s|$shared_key|/usr/share/keyrings/vector-unavailable-test.gpg|" "$source_file" +fi +setup --migrate --cose-only > "$work/command.log" 2>&1 +[[ $(gpg --batch --with-colons --show-keys "$owned_key" | grep -c '^pub:') == 1 ]] +[[ "$shared_hash" == "$(sha256sum "$shared_key")" ]] +refresh > "$work/command.log" 2>&1 +pass 'post-cutover migration from original settings needs no legacy key source and preserves shared keys' diff --git a/distribution/repositories/tests/installer_fixture.py b/distribution/repositories/tests/installer_fixture.py new file mode 100644 index 0000000000000..3411cb0edc603 --- /dev/null +++ b/distribution/repositories/tests/installer_fixture.py @@ -0,0 +1,103 @@ +"""Private installer fixtures, with a separate ephemeral legacy signing identity.""" + +import json +import os +from datetime import timedelta +from email.utils import format_datetime, parsedate_to_datetime +from pathlib import Path +import shutil +import tarfile + +from inputs import require +from repository import Signer, key_fingerprints, prepare +from storage import Snapshot, canonical, sha256_file + + +def installer_fixture(config, work, gpg, password): + gpg("--quick-generate-key", "Vector simulated legacy repository", "rsa4096", "cert", "2d", protected=True) + primary = key_fingerprints(gpg("--with-colons", "--list-keys", "Vector simulated legacy repository"), "pub", "sub")[0][1] + gpg("--quick-add-key", primary, "rsa4096", "sign", "2d", protected=True) + subkey = key_fingerprints(gpg("--with-colons", "--list-keys", primary), "pub", "sub")[1][1] + private = gpg("--armor", "--export-secret-subkeys", subkey + "!", protected=True).decode() + public = work / "installer/public" + public.mkdir(parents=True) + (public / "legacy.asc").write_bytes(gpg("--armor", "--export", primary)) + (public / "legacy.gpg").write_bytes(gpg("--export", primary)) + shutil.copyfile(config["public_key_path"], public / "cose.asc") + (public / "fingerprint").write_text(config["primary_fingerprint"] + "\n") + legacy_config = {**config, "primary_fingerprint": primary, "signing_fingerprint": subkey, + "public_key_path": str(public / "legacy.asc")} + legacy = work / "installer-legacy" + legacy.mkdir() + shutil.copyfile(work / "one/input.json", legacy / "input.json") + shutil.copytree(work / "one/input", legacy / "input", copy_function=os.link) + (legacy / "base").mkdir() + (legacy / "base.json").write_bytes(b"null\n") + os.environ.update(PACKAGE_SIGNING_SUBKEY=private, PACKAGE_SIGNING_PASSPHRASE=password) + prepare(legacy_config, legacy) + # Model the observed Datadog Release fields, without pretending that this + # fixture has Datadog's signing key or its non-Vector package payloads. + observed = json.loads((Path(__file__).parent / "legacy-apt-identity.json").read_bytes()) + signer = Signer(legacy_config, "/run/keys/legacy-metadata", private, password) + try: + for channel in ("stable", "beta"): + suite = legacy / "result/files/apt/dists" / channel + release = suite / "Release" + lines = [line for line in release.read_text().splitlines() + if not line.startswith(("Origin:", "Label:", "Suite:", "Components:", "Acquire-By-Hash:", "Date:"))] + first = (work / "one/result/files/apt/dists" / channel / "Release").read_text() + date = next(line[6:] for line in first.splitlines() if line.startswith("Date: ")) + # Apt rejects Release date regressions while retaining its cached + # indexes. This simulated legacy publication must precede cutover. + lines.insert(0, "Date: " + format_datetime(parsedate_to_datetime(date) - timedelta(hours=1))) + lines.insert(0, ("Suite: " + observed["Suite"]).rstrip()) + lines.insert(1, "Components: " + observed["Components"]) + release.write_text("\n".join(lines) + "\n") + signer.sign(release, clear=True, destination=suite / "InRelease") + signer.sign(release, destination=suite / "Release.gpg") + finally: + signer.close() + private = None + original = Snapshot.decode((legacy / "result/manifest.json").read_bytes()) + entries = {p.relative_to(legacy / "result/files").as_posix(): {"size": p.stat().st_size, "sha256": sha256_file(p)} + for p in (legacy / "result/files").rglob("*") if p.is_file()} + (legacy / "result/manifest.json").write_bytes(Snapshot(entries, input_id=original.input_id).data) + + views = work / "installer/views" + for view, sources in ( + ("legacy", [legacy / "result/files"]), + ("cose", [work / "one/result/files"]), + ("upgraded", [work / "one/result/files", work / "two/result/files"]), + ): + for source in sources: + for path in source.rglob("*"): + if path.is_file(): + target = views / view / path.relative_to(source) + target.parent.mkdir(parents=True, exist_ok=True) + # Copies permit client-side tamper tests without modifying + # another view through a shared inode. + shutil.copyfile(path, target) + for channel in ("stable", "beta"): + release = (views / "cose/apt/dists" / channel / "Release").read_text() + require("Origin:" not in release and "Label:" not in release and "Suite:\n" in release, + "The new apt repository would require release-information approval") + output = work / "installer-bundles" + output.mkdir() + for deb_arch, rpm_arch in (("amd64", "x86_64"), ("arm64", "aarch64")): + with tarfile.open(output / (deb_arch + ".tar"), "w") as archive: + for path in sorted((work / "installer").rglob("*")): + if not path.is_file(): + continue + if path.name.endswith(".deb") and not path.name.endswith("_" + deb_arch + ".deb"): + continue + if path.name.endswith(".rpm") and not path.name.endswith("." + rpm_arch + ".rpm"): + continue + archive.add(path, arcname=path.relative_to(work / "installer").as_posix()) + (work / "installer-report.json").write_bytes(canonical({ + "primary_fingerprint": config["primary_fingerprint"], + "legacy_fingerprint": primary, + "checks": ["independent ephemeral legacy signing key", "legacy apt identity fields captured from live metadata", + "unchanged apt Origin, Label, Suite and Codename", "legacy metadata predates COSE publications", + "three signed repository views for native HTTPS tests"], + })) + print("PASS: installer fixtures preserve the observed apt identity with independent legacy and COSE keys", flush=True) diff --git a/distribution/repositories/tests/legacy-apt-identity.json b/distribution/repositories/tests/legacy-apt-identity.json new file mode 100644 index 0000000000000..6125005a85fe4 --- /dev/null +++ b/distribution/repositories/tests/legacy-apt-identity.json @@ -0,0 +1,9 @@ +{ + "source": "https://apt.vector.dev/dists/stable/Release", + "observed": "2026-10-20", + "Origin": null, + "Label": null, + "Suite": "", + "Codename": "stable", + "Components": "main 6 7 observability-pipelines-worker-1 vector-0 observability-pipelines-worker-2 agentless-scanner datadog-security-cli datadog-bits-cli" +} diff --git a/distribution/repositories/tests/package_fixture.py b/distribution/repositories/tests/package_fixture.py index a39c0a54210eb..a81d85ad8e5f6 100644 --- a/distribution/repositories/tests/package_fixture.py +++ b/distribution/repositories/tests/package_fixture.py @@ -100,6 +100,8 @@ def gpg(*args, protected=False): require(promoted.entries["yum/beta/" + relative] == snapshot.entries["yum/stable/" + relative], "Selecting another channel changed RPM signatures") shutil.rmtree(beta) + from tests.installer_fixture import installer_fixture + installer_fixture(config, work, gpg, password) config.pop("public_key_path") config.pop("role", None) (directory / "staging.json").write_bytes(canonical(config)) diff --git a/distribution/repositories/tests/test_inputs.py b/distribution/repositories/tests/test_inputs.py index 7d7f473771e8d..e8ad967ecf5ed 100644 --- a/distribution/repositories/tests/test_inputs.py +++ b/distribution/repositories/tests/test_inputs.py @@ -1,5 +1,6 @@ import copy import io +import json import sys import tarfile import tempfile @@ -9,7 +10,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from engine import unpack_bundle from inputs import expected_packages, parse_checksums, select_assets, validate_run, versions -from repository import merge_packages +from repository import apt_indexes, merge_packages from storage import PublicationError @@ -64,6 +65,25 @@ def test_incremental_apt_merge_keeps_historical_native_versions(self): with self.assertRaises(PublicationError): merge_packages(old, old.replace(b"pool/v/ve", b"pool/replaced")) + def test_apt_identity_matches_observed_legacy_repository(self): + # Identity fields are cached by apt and changes require explicit user + # approval. Compare with observed input rather than the desired output. + observed = json.loads((Path(__file__).parent / "legacy-apt-identity.json").read_bytes()) + class UnsignedFixture: + def sign(self, source, **kwargs): + pass + def verify(self, *args): + pass + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + apt_indexes(root, "stable", {}, root / "previous", UnsignedFixture()) + release = {key: value.strip() for line in + (root / "apt/dists/stable/Release").read_text().splitlines() + if ":" in line and not line.startswith(" ") + for key, value in [line.split(":", 1)]} + for field in ("Origin", "Label", "Suite", "Codename"): + self.assertEqual(release.get(field), observed[field], field) + def test_signed_bundle_rejects_links_traversal_and_duplicates(self): for names, linked in ((["files/apt/../escape"], False), (["files/apt/pool/x"], True), (["input.json", "input.json"], False)): diff --git a/docs/reference/package-repositories.md b/docs/reference/package-repositories.md index e659f1f315460..47c75b3f2a7ec 100644 --- a/docs/reference/package-repositories.md +++ b/docs/reference/package-repositories.md @@ -88,6 +88,68 @@ headers; the actual CDN caching and retry behavior must be validated before cutover. Workflow success also requires public HTTPS checksum verification and native apt/DNF installation on amd64 and arm64. +## Client setup and migration + +`distribution/repositories/setup.sh` is the standalone apt/yum setup script. +It configures a repository without installing or upgrading Vector. A new source +defaults to stable and COSE-only trust. An existing `vector.list` or `vector.repo` +keeps its URL, channel, architecture restrictions and other options while adding +COSE trust. `--migrate` requires an existing managed source. + +A pre-cutover migration retains the old public keys in Vector's own key file. +It leaves shared Datadog keyrings and RPM database entries unchanged. On RPM +clients, old remote public keys are copied into the local Vector key file so +subsequent refreshes no longer need Datadog's key endpoint. Both `gpgcheck` and +`repo_gpgcheck` are required, including on custom mirrors. The old RHEL/CentOS +8.1 exception is not retained. + +Only after hosting has switched to COSE, `--migrate --cose-only` can remove legacy +trust from Vector's own configuration. This mode also supports late migration +when the old key files or endpoints are unavailable. It never deletes shared +Datadog key files, RPM keys or keys cached by DNF. + +The default public key URL is +`https://dd-cose-releases.s3.us-east-1.amazonaws.com/vector/keys/vector-packages.asc`. +The script verifies its full primary fingerprint before changing managed files. +It uses HTTPS with certificate verification for key downloads, backs up original +managed files with `.pre-cose`, installs replacements atomically, and refreshes +only the Vector repository. A failed final refresh leaves the saved configuration +available for inspection and retry. The script rejects ambiguous source layouts, +symlinked managed files, unavailable legacy trust and unsupported overrides +before replacing source files. Custom filenames and deb822 sources require +manual migration. + +Fresh setup retains `VECTOR_DIST_CHANNEL`, `VECTOR_MAJOR_VERSION`, +`VEC_REPO_URL`, `TESTING_APT_URL`, `TESTING_YUM_URL`, +`TESTING_APT_REPO_VERSION` and `TESTING_YUM_VERSION_PATH`. +`TESTING_KEYS_URL` is a compatibility alias for the HTTPS directory containing +`vector-packages.asc`. An owned staging mirror can supply +`VECTOR_PUBLIC_KEY_URL` and an explicit full +`VECTOR_PUBLIC_KEY_FINGERPRINT`. These are public trust settings, not secrets. +Existing sources reject fresh-setup overrides rather than switching their +channel on a repeated invocation. `CSM_MIGRATE=true` disables the known +`timber-vector` file with a backup. + +The publisher keeps the existing apt identity: no `Origin` or `Label`, an empty +`Suite`, and the channel as `Codename`. This permits the signing-key and hosting +transition without a separate release-information approval. The observed +legacy fields are recorded in `tests/legacy-apt-identity.json`; a regression +check compares the publisher's output with that observation. + +Before announcement, deploy the reviewed script at +`vector/setup.sh` and the public key at `vector/keys/vector-packages.asc` +in the COSE bucket, then publish the migration guide. These paths are separate +from the package publisher's scope and need an appropriately scoped deployment. +Keep the live `setup.vector.dev` redirect and normal installation pages unchanged +until the repository hosting cutover. At cutover, point `setup.vector.dev` at the +new script and update those installation pages together. + +The protected production client job runs this setup script before installing a +release. Activation therefore requires the public key endpoint to be available. +The real-package fixture also creates separate ephemeral legacy and COSE signing +identities and installer bundles for disposable HTTPS client tests. Private keys +stay in tmpfs; client bundles contain only public keys and signed packages. + ## Maintainer commands Build the tools once from reviewed source: @@ -165,3 +227,12 @@ Rollback restores retained signed metadata and checks that its immutable objects still exist. It does not downgrade installed packages or restore Datadog-only client trust. Verify the public endpoints and native clients after rollback. Never delete a control object to bypass the recovery checks. + +Apt can retain newer cached indexes when a restored Release has an older Date. +The original rollback client tests cleared those caches, so they do not prove +that existing clients adopt the restored snapshot. Before activation, validate a +recovery design that accounts for cached Release dates, such as publishing a +freshly dated and signed generation for the intended package set. Do not disable +apt's date or signature checks globally. This remains an open cutover requirement. +The behavior is visible in +[apt's metadata verification](https://sources.debian.org/src/apt/2.2.4/apt-pkg/acquire-item.cc/). From ee156ee75a7098b2a4eb014d0aec32500fbed5b9 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 20 Oct 2026 10:22:07 -0400 Subject: [PATCH 03/23] fix(ci): refresh signed metadata when rolling back package repositories --- .../package-repositories-maintenance.yml | 121 ++++++++- distribution/repositories/bundles.py | 52 ++++ distribution/repositories/engine.py | 78 ++---- distribution/repositories/repository.py | 32 ++- distribution/repositories/rollback.py | 236 ++++++++++++++++++ distribution/repositories/storage.py | 30 ++- .../repositories/tests/package_fixture.py | 5 +- .../repositories/tests/publication_fixture.py | 77 +++++- .../repositories/tests/rollback_fixture.py | 81 ++++++ .../repositories/tests/test_rollback.py | 55 ++++ .../repositories/tests/test_storage.py | 37 ++- docs/reference/package-repositories.md | 66 +++-- vdev/src/commands/release/repositories.rs | 70 +++++- 13 files changed, 823 insertions(+), 117 deletions(-) create mode 100644 distribution/repositories/bundles.py create mode 100644 distribution/repositories/rollback.py create mode 100644 distribution/repositories/tests/rollback_fixture.py create mode 100644 distribution/repositories/tests/test_rollback.py diff --git a/.github/workflows/package-repositories-maintenance.yml b/.github/workflows/package-repositories-maintenance.yml index 20e20b24368b8..8e5aeccea991f 100644 --- a/.github/workflows/package-repositories-maintenance.yml +++ b/.github/workflows/package-repositories-maintenance.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: operation: - description: Inspect state, restore signed metadata, or recover an abandoned lock + description: Inspect state, restore a package selection, or recover an abandoned lock type: choice options: [status, rollback, recover-lock] default: status @@ -56,6 +56,7 @@ jobs: run: | cargo build --locked --package vdev target/debug/vdev release repositories build-image + target/debug/vdev release repositories test - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 with: role-to-assume: ${{ vars.PACKAGE_REPOSITORIES_ROLE_ARN }} @@ -64,7 +65,7 @@ jobs: role-session-name: package-maintenance-${{ github.run_id }} allowed-account-ids: "486234852809" unset-current-credentials: true - - name: Run maintenance and record the resulting state + - name: Prepare rollback or run maintenance, and record the current state env: PACKAGE_REPOSITORIES_ROLE_ARN: ${{ vars.PACKAGE_REPOSITORIES_ROLE_ARN }} OPERATION: ${{ inputs.operation }} @@ -74,7 +75,7 @@ jobs: case "$OPERATION" in status) ;; rollback) - target/debug/vdev release repositories rollback "$SNAPSHOT" + target/debug/vdev release repositories rollback base "$SNAPSHOT" ;; recover-lock) target/debug/vdev release repositories recover-lock \ @@ -88,3 +89,117 @@ jobs: cat target/package-repository-status.json echo '```' } >> "$GITHUB_STEP_SUMMARY" + - name: Save the reviewed tools for the rollback jobs + if: inputs.operation == 'rollback' + run: | + mkdir -p target/package-repository-tools + cp target/debug/vdev target/package-repository-tools/vdev + docker save vector-package-publisher:local | gzip > target/package-repository-tools/image.tar.gz + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: inputs.operation == 'rollback' + with: + name: package-repository-tools + path: target/package-repository-tools/ + retention-days: 1 + if-no-files-found: error + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: inputs.operation == 'rollback' + with: + name: package-repository-rollback-base + path: target/package-repositories/ + retention-days: 1 + if-no-files-found: error + + sign: + needs: maintain + if: inputs.operation == 'rollback' + runs-on: ubuntu-24.04 + environment: package-repositories-signing + timeout-minutes: 20 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: ./.github/actions/package-repository-tools + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: package-repository-rollback-base + path: target/package-repositories + - id: cached + run: | + if test -f target/package-repositories/result/manifest.json; then + echo 'ready=true' >> "$GITHUB_OUTPUT" + fi + - name: Sign fresh apt roots for the selected archived indexes + if: steps.cached.outputs.ready != 'true' + env: + PACKAGE_SIGNING_SUBKEY: ${{ secrets.PACKAGE_SIGNING_SUBKEY }} + PACKAGE_SIGNING_PASSPHRASE: ${{ secrets.PACKAGE_SIGNING_PASSPHRASE }} + run: '"$VDEV" release repositories rollback prepare' + # Private material stays in the container's tmpfs, outside this artifact. + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: package-repository-rollback-signed + path: target/package-repositories/ + retention-days: 1 + if-no-files-found: error + + publish: + needs: sign + runs-on: ubuntu-24.04 + environment: package-repositories-publishing + timeout-minutes: 20 + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: ./.github/actions/package-repository-tools + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: package-repository-rollback-signed + path: target/package-repositories + - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 + with: + role-to-assume: ${{ vars.PACKAGE_REPOSITORIES_ROLE_ARN }} + aws-region: us-east-1 + role-duration-seconds: 900 + allowed-account-ids: "486234852809" + unset-current-credentials: true + - name: Publish the fresh rollback against its expected repository state + env: + PACKAGE_REPOSITORIES_ROLE_ARN: ${{ vars.PACKAGE_REPOSITORIES_ROLE_ARN }} + run: | + "$VDEV" release repositories rollback publish + "$VDEV" release repositories status > target/package-repository-status.json + cat target/package-repository-status.json + { + echo '```json' + cat target/package-repository-status.json + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + endpoints: + needs: publish + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: ./.github/actions/package-repository-tools + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: package-repository-rollback-signed + path: target/package-repositories + - name: Verify the selected indexes and fresh roots through public HTTPS + run: '"$VDEV" release repositories rollback verify' diff --git a/distribution/repositories/bundles.py b/distribution/repositories/bundles.py new file mode 100644 index 0000000000000..75bef64170521 --- /dev/null +++ b/distribution/repositories/bundles.py @@ -0,0 +1,52 @@ +"""Checksum-verified transport of signed publication workspaces.""" + +import gzip +import shutil +import tarfile + +from inputs import require +from storage import Snapshot, matches, safe_relative + + +def checked_download(store, key, path, entry=None): + path.parent.mkdir(parents=True, exist_ok=True) + head = store.head(key) + require(head is not None and head["ContentLength"] < 5 * 1024**3, "Missing or oversized S3 input") + actual = store.download(key, path) + require(matches(head, actual) and (entry is None or actual == entry), "Downloaded S3 object failed checksum verification") + return actual + + +def unpack_bundle(path, destination): + destination.mkdir() + total = 0 + names = set() + with tarfile.open(path, "r:gz") as archive: + for member in archive: + require(member.isfile() and member.name not in names, "Bundle has a duplicate or nonregular member") + names.add(member.name) + require(member.name in {"input.json", "manifest.json"} or member.name.startswith("files/"), "Unexpected bundle member") + if member.name.startswith("files/"): + safe_relative(member.name[6:]) + total += member.size + require(total < 5 * 1024**3, "Bundle exceeds the extraction budget") + target = destination / member.name + target.parent.mkdir(parents=True, exist_ok=True) + with archive.extractfile(member) as source, target.open("xb") as output: + shutil.copyfileobj(source, output) + require({"input.json", "manifest.json"}.issubset(names), "Incomplete signed bundle") + return Snapshot.local(destination / "files") + + +def pack_bundle(source, target): + with target.open("xb") as raw, gzip.GzipFile(fileobj=raw, mode="wb", mtime=0, filename="") as compressed: + with tarfile.open(fileobj=compressed, mode="w|") as archive: + for path in sorted(source.rglob("*")): + require(not path.is_symlink(), "Bundle contains a symlink") + if path.is_file(): + info = archive.gettarinfo(str(path), arcname=path.relative_to(source).as_posix()) + info.uid = info.gid = info.mtime = 0 + info.uname = info.gname = "" + info.mode = 0o644 + with path.open("rb") as data: + archive.addfile(info, data) diff --git a/distribution/repositories/engine.py b/distribution/repositories/engine.py index e8fb1da1c0864..0c28970ef7c12 100644 --- a/distribution/repositories/engine.py +++ b/distribution/repositories/engine.py @@ -1,21 +1,20 @@ """Container implementation for the typed `vdev release repositories` commands.""" import argparse -import gzip import hashlib import json import os from pathlib import Path import re -import shutil import sys -import tarfile from urllib.request import urlopen +from bundles import checked_download, pack_bundle, unpack_bundle +import rollback from inputs import fetch, require, verify_input from repository import PublicKey, lint_package, prepare, repomd_locations, run, verify_metadata from storage import (Busy, Conflict, Publisher, S3Store, Snapshot, byte_entry, - canonical, matches, mutable, safe_relative, sha256_file) + mutable, sha256_file) def configuration(path): @@ -58,50 +57,6 @@ def rpm_cache_key(config, asset): return config["bundle_prefix"] + "rpms/" + asset["sha256"] + ".rpm" -def checked_download(store, key, path, entry=None): - path.parent.mkdir(parents=True, exist_ok=True) - head = store.head(key) - require(head is not None and head["ContentLength"] < 5 * 1024**3, "Missing or oversized S3 input") - actual = store.download(key, path) - require(matches(head, actual) and (entry is None or actual == entry), "Downloaded S3 object failed checksum verification") - return actual - - -def unpack_bundle(path, destination): - destination.mkdir() - total = 0 - names = set() - with tarfile.open(path, "r:gz") as archive: - for member in archive: - require(member.isfile() and member.name not in names, "Bundle has a duplicate or nonregular member") - names.add(member.name) - require(member.name in {"input.json", "manifest.json"} or member.name.startswith("files/"), "Unexpected bundle member") - if member.name.startswith("files/"): - safe_relative(member.name[6:]) - total += member.size - require(total < 5 * 1024**3, "Bundle exceeds the extraction budget") - target = destination / member.name - target.parent.mkdir(parents=True, exist_ok=True) - with archive.extractfile(member) as source, target.open("xb") as output: - shutil.copyfileobj(source, output) - require({"input.json", "manifest.json"}.issubset(names), "Incomplete signed bundle") - return Snapshot.local(destination / "files") - - -def pack_bundle(source, target): - with target.open("xb") as raw, gzip.GzipFile(fileobj=raw, mode="wb", mtime=0, filename="") as compressed: - with tarfile.open(fileobj=compressed, mode="w|") as archive: - for path in sorted(source.rglob("*")): - require(not path.is_symlink(), "Bundle contains a symlink") - if path.is_file(): - info = archive.gettarinfo(str(path), arcname=path.relative_to(source).as_posix()) - info.uid = info.gid = info.mtime = 0 - info.uname = info.gname = "" - info.mode = 0o644 - with path.open("rb") as data: - archive.addfile(info, data) - - def base(config, work): manifest, input_id = verify_input(work) store = store_for(config) @@ -200,8 +155,7 @@ def publish(config, work): print("Published snapshot " + publisher.publish(snapshot)) -def verify(config, work): - manifest, snapshot = check_result(config, work) +def verify_snapshot(config, snapshot): names = {name for name in snapshot.entries if mutable(name)} names.update(path.relative_to(snapshot.source).as_posix() for path in snapshot.source.rglob("*") if path.is_file()) for name in sorted(names): @@ -218,7 +172,12 @@ def verify(config, work): require(size <= expected["size"], "Endpoint returned an oversized object") digest.update(data) require({"sha256": digest.hexdigest(), "size": size} == expected, "Endpoint differs from published snapshot: " + name) - print(f"Verified {len(names)} endpoint objects for {manifest['tag']}") + print(f"Verified {len(names)} endpoint objects for snapshot {snapshot.id}") + + +def verify(config, work): + _, snapshot = check_result(config, work) + verify_snapshot(config, snapshot) def main(): @@ -234,7 +193,10 @@ def main(): test_options = commands.add_parser("test").add_mutually_exclusive_group() test_options.add_argument("--packages", action="store_true") test_options.add_argument("--s3", action="store_true") - commands.add_parser("rollback").add_argument("snapshot") + rollback_commands = commands.add_parser("rollback").add_subparsers(dest="phase", required=True) + rollback_commands.add_parser("base").add_argument("snapshot") + for name in ("prepare", "publish", "verify"): + rollback_commands.add_parser(name) recovery = commands.add_parser("recover-lock") recovery.add_argument("--expected-version", required=True) recovery.add_argument("--confirm-owner-stopped", action="store_true") @@ -252,6 +214,16 @@ def main(): publish(config, work) elif args.command == "verify": verify(config, work) + elif args.command == "rollback": + if args.phase == "base": + rollback.base(config, work, args.snapshot, store_for(config)) + elif args.phase == "prepare": + print("Prepared rollback snapshot " + rollback.prepare(config, work)) + elif args.phase == "publish": + rollback.publish(config, work, store_for(config)) + else: + _, snapshot = rollback.check_result(config, work) + verify_snapshot(config, snapshot) elif args.command == "test": import unittest suite = unittest.defaultTestLoader.discover("tests") @@ -267,8 +239,6 @@ def main(): if args.command == "status": state, head = publisher.load_state() print(json.dumps({"state": state, "version": head["VersionId"] if head else None}, indent=2)) - elif args.command == "rollback": - print(publisher.publish(publisher.archived(args.snapshot), rollback=True)) else: publisher.recover_lock(args.expected_version, args.confirm_owner_stopped) print("Lock released; resume the pending snapshot or roll back to the last completed snapshot") diff --git a/distribution/repositories/repository.py b/distribution/repositories/repository.py index 7ebd2cc9ed2b3..c3f6d4cb6ee69 100644 --- a/distribution/repositories/repository.py +++ b/distribution/repositories/repository.py @@ -9,7 +9,8 @@ import tarfile import tempfile import time -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone +from email.utils import format_datetime, parsedate_to_datetime import xml.etree.ElementTree as ET from inputs import DEB_ARCHES, RPM_ARCHES, require, verify_input @@ -252,6 +253,30 @@ def verify_metadata(snapshot, root, public): "RPM package differs from its signed index") +def release_date(data): + metadata = fields(data.decode()) + require("Date" in metadata and "Valid-Until" not in metadata, + "Expected a dated repository without Valid-Until; expiry needs a separate policy") + value = parsedate_to_datetime(metadata["Date"]) + require(value.tzinfo is not None, "Repository date must have a timezone") + return value.astimezone(timezone.utc) + + +def next_release_date(previous, now=None): + now = (now or datetime.now(timezone.utc)).replace(microsecond=0) + value = max([now, *(release_date(data) + timedelta(seconds=1) for data in previous)]) + # Apt normally tolerates ten seconds into the future. Keep a margin and + # fail on clock skew rather than publishing a root clients cannot accept. + require(value <= now + timedelta(seconds=5), "Repository date is too far ahead; check the clock and retry later") + return format_datetime(value) + + +def dated_release(data, date): + release_date(data) # Reject ambiguous/unsupported roots before changing one field. + return b"".join(("Date: " + date + "\n").encode() if line.startswith(b"Date:") else line + for line in data.splitlines(keepends=True)) + + def apt_indexes(root, channel, incoming, previous, signer): suite = root / "apt/dists" / channel architectures = (*DEB_ARCHES, "x86_64", "all", "i386") @@ -273,8 +298,9 @@ def apt_indexes(root, channel, incoming, previous, signer): # Preserve the existing public repository identity during the key/hosting # migration. Adding Origin/Label or changing the empty Suite would require # a separate apt release-information approval on clients with cached indexes. - lines = ["Suite:", f"Codename: {channel}", - "Date: " + datetime.now(timezone.utc).strftime("%a, %d %b %Y %H:%M:%S +0000"), + prior_release = previous / f"apt/dists/{channel}/Release" + date = next_release_date([prior_release.read_bytes()] if prior_release.exists() else []) + lines = ["Suite:", f"Codename: {channel}", "Date: " + date, "Architectures: " + " ".join(architectures), "Components: vector-0", "Acquire-By-Hash: yes"] for algorithm in ("sha256", "sha512"): lines.append(algorithm.upper() + ":") diff --git a/distribution/repositories/rollback.py b/distribution/repositories/rollback.py new file mode 100644 index 0000000000000..2ca2b23109144 --- /dev/null +++ b/distribution/repositories/rollback.py @@ -0,0 +1,236 @@ +"""Restore an archived package selection as a new, freshly signed generation.""" + +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import shutil + +from bundles import checked_download, pack_bundle, unpack_bundle +from inputs import require +from repository import (PublicKey, Signer, dated_release, next_release_date, + release_date, repomd_locations, verify_metadata) +from storage import Publisher, Snapshot, byte_entry, canonical, mutable, sha256_file + + +def plan_bytes(target, active, pending): + data = canonical({"schema": 1, "operation": "rollback", "target": target, + "active": active, "pending": pending}) + decode_plan(data) + return data + + +def decode_plan(data): + plan = json.loads(data) + require(set(plan) == {"schema", "operation", "target", "active", "pending"} + and plan["schema"] == 1 and plan["operation"] == "rollback" + and canonical(plan) == data, "Invalid rollback request") + for name in ("target", "active", "pending"): + value = plan[name] + require((name == "pending" and value is None) or + (isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value)), "Invalid rollback snapshot identity") + require(plan["pending"] is None or plan["target"] == plan["active"], + "Abandoning a pending publication must restore the last completed package selection") + return plan, hashlib.sha256(data).hexdigest() + + +def request_key(config, input_id): + return config["bundle_prefix"] + "rollback-requests/" + input_id + ".json" + + +def bundle_key(config, input_id): + return config["bundle_prefix"] + "rollbacks/" + input_id + ".tar.gz" + + +def source_ids(plan): + return sorted({plan[key] for key in ("target", "active", "pending") if plan[key]}) + + +def download_sources(publisher, work, plan): + for identity in source_ids(plan): + snapshot = publisher.archived(identity) + directory = work / "sources" / identity + root = directory / "files" + root.mkdir(parents=True) + (directory / "manifest.json").write_bytes(snapshot.data) + for name, entry in snapshot.entries.items(): + if mutable(name): + checked_download(publisher.store, publisher.archives + identity + "/files/" + name, root / name, entry) + # A pending generation may have stopped before uploading immutable RPM + # indexes. Its signed root dates still constrain the replacement. + if identity in {plan["active"], plan["target"]}: + for package_root in root.glob("yum/*/vector-0/*"): + for relative in repomd_locations(package_root): + name = (package_root / relative).relative_to(root).as_posix() + require(name in snapshot.entries, "RPM metadata is missing from the archived manifest") + checked_download(publisher.store, publisher.live + name, root / name, snapshot.entries[name]) + + +def base(config, work, target, store): + publisher = Publisher(store, config) + state, _ = publisher.load_state() + require(not state["held"] and state["active"], "Rollback requires an unlocked, initialized repository") + data = None + retry = False + candidate = publisher.archived(state["pending"] or state["active"]) + if candidate.input_id and store.head(request_key(config, candidate.input_id)): + saved, _ = store.read(request_key(config, candidate.input_id)) + plan, identity = decode_plan(saved) + require(identity == candidate.input_id, "Saved rollback request failed checksum verification") + if plan["target"] == target: + data = saved + retry = True + data = data or plan_bytes(target, state["active"], state["pending"]) + plan, input_id = decode_plan(data) + (work / "input.json").write_bytes(data) + download_sources(publisher, work, plan) + key = bundle_key(config, input_id) + if store.head(key): + archive = work / "cached.tar.gz" + checked_download(store, key, archive) + snapshot = unpack_bundle(archive, work / "result") + require(snapshot.input_id == input_id and snapshot.base == plan["active"] + and (work / "result/input.json").read_bytes() == data, "Saved rollback belongs to another request") + if retry: + require(snapshot.id == candidate.id, "Saved rollback differs from the pending/completed generation") + archive.unlink() + print("Restored the saved signed rollback; signing is skipped") + else: + require(candidate.input_id != input_id, "Required rollback retry bundle is missing") + print("Downloaded archived selections and all potentially exposed root dates") + + +def check_sources(config, work): + plan, input_id = decode_plan((work / "input.json").read_bytes()) + sources = {} + public = PublicKey(config, "/run/keys/rollback-sources") + try: + for identity in source_ids(plan): + directory = work / "sources" / identity + snapshot = Snapshot.decode((directory / "manifest.json").read_bytes(), identity) + root = directory / "files" + require(root.is_dir() and not root.is_symlink(), "Invalid rollback source directory") + present = set() + for path in root.rglob("*"): + require(not path.is_symlink(), "Rollback source contains a symlink") + if path.is_dir(): + continue + name = path.relative_to(root).as_posix() + require(path.is_file() and snapshot.entries.get(name) == + {"sha256": sha256_file(path), "size": path.stat().st_size}, "Rollback source differs from its manifest") + require(not name.endswith((".deb", ".rpm")), "Rollback does not require local package payloads") + present.add(name) + require({name for name in snapshot.entries if mutable(name)} <= present, "Rollback source is missing mutable metadata") + snapshot.source = root + sources[identity] = snapshot + if identity in {plan["active"], plan["target"]}: + verify_metadata(snapshot, root, public) + else: + for name in present: + if name.endswith("/Release"): + suite = (root / name).parent + public.verify(suite / "InRelease") + public.verify(suite / "Release.gpg", suite / "Release") + require(public.gpg("--decrypt", str(suite / "InRelease")) == (root / name).read_bytes(), + "Pending apt roots disagree") + elif name.endswith("/repomd.xml"): + public.verify(Path(str(root / name) + ".asc"), root / name) + selected = sources[plan["target"]] + mutable_names = {name for name in selected.entries if mutable(name)} + for snapshot in sources.values(): + require({name for name in snapshot.entries if mutable(name)} == mutable_names, + "Rollback across repository layout changes needs a separate migration") + # Completed generations preserve the immutable history. Never resurrect + # bytes absent from that history or allow two identities at the same URL. + active = sources[plan["active"]] + require(all(active.entries.get(name) == entry for name, entry in selected.entries.items() if not mutable(name)), + "Selected snapshot is not contained in the active immutable history") + finally: + public.close() + return plan, input_id, sources + + +def expected_entries(plan, sources): + entries = dict(sources[plan["target"]].entries) + entries.update({name: entry for name, entry in sources[plan["active"]].entries.items() if not mutable(name)}) + return entries + + +def prepare(config, work): + private_key = os.environ.pop("PACKAGE_SIGNING_SUBKEY", "") + passphrase = os.environ.pop("PACKAGE_SIGNING_PASSPHRASE", "") + plan, input_id, sources = check_sources(config, work) + selected = sources[plan["target"]] + root = work / "result/files" + shutil.copytree(selected.source, root) + entries = expected_entries(plan, sources) + signer = Signer(config, "/run/keys/rollback-signing", private_key, passphrase) + try: + for name in selected.entries: + if name.endswith("/Release"): + release = root / name + date = next_release_date([(snapshot.source / name).read_bytes() for snapshot in sources.values()]) + release.write_bytes(dated_release(release.read_bytes(), date)) + signer.sign(release, clear=True, destination=release.parent / "InRelease") + signer.sign(release, destination=release.parent / "Release.gpg") + for path in root.rglob("*"): + if path.is_file(): + entries[path.relative_to(root).as_posix()] = byte_entry(path.read_bytes()) + snapshot = Snapshot(entries, root, base=plan["active"], input_id=input_id) + (work / "result/manifest.json").write_bytes(snapshot.data) + (work / "result/input.json").write_bytes((work / "input.json").read_bytes()) + finally: + signer.close() + check_result(config, work) + return snapshot.id + + +def check_result(config, work): + plan, input_id, sources = check_sources(config, work) + snapshot = Snapshot.local(work / "result/files") + require(snapshot.input_id == input_id and snapshot.base == plan["active"] + and (work / "result/input.json").read_bytes() == (work / "input.json").read_bytes(), + "Signed rollback belongs to a different request") + selected = sources[plan["target"]] + expected = expected_entries(plan, sources) + require(snapshot.entries.keys() == expected.keys(), "Rollback changed the repository file set") + refreshed = {name for name in expected if name.endswith(("/Release", "/InRelease", "/Release.gpg"))} + require(all(snapshot.entries[name] == entry for name, entry in expected.items() if name not in refreshed), + "Rollback changed package bytes or the selected indexes") + for name in expected: + if name.endswith("/Release"): + data = (snapshot.source / name).read_bytes() + date = release_date(data) + require(all(date > release_date((source.source / name).read_bytes()) for source in sources.values()), + "Rollback date is not newer than every potentially exposed root") + require(date <= datetime.now(timezone.utc) + timedelta(seconds=5), "Rollback root is dated too far into the future") + require(dated_release(data, "DATE") == dated_release((selected.source / name).read_bytes(), "DATE"), + "Rollback changed apt identity or index checksums") + public = PublicKey(config, "/run/keys/rollback-result") + try: + verify_metadata(snapshot, snapshot.source, public) + finally: + public.close() + return plan, snapshot + + +def publish(config, work, store): + plan, snapshot = check_result(config, work) + publisher = Publisher(store, config) + for identity in source_ids(plan): + require(publisher.archived(identity).data == (work / "sources" / identity / "manifest.json").read_bytes(), + "Rollback source is not the saved archive") + archive = work / "prepared.tar.gz" + archive.unlink(missing_ok=True) + pack_bundle(work / "result", archive) + require(archive.stat().st_size < 5 * 1024**3, "Rollback bundle exceeds the single-PUT limit") + with archive.open("rb") as stream: + publisher.ensure_immutable(bundle_key(config, snapshot.input_id), stream, + {"sha256": sha256_file(archive), "size": archive.stat().st_size}) + data = (work / "input.json").read_bytes() + publisher.ensure_immutable(request_key(config, snapshot.input_id), data, byte_entry(data)) + identity = publisher.publish(snapshot, recovery=(plan["active"], plan["pending"])) + print("Published fresh rollback snapshot " + identity) + return identity diff --git a/distribution/repositories/storage.py b/distribution/repositories/storage.py index cb2e2236eae33..d06f500ff9446 100644 --- a/distribution/repositories/storage.py +++ b/distribution/repositories/storage.py @@ -309,9 +309,9 @@ def archived(self, snapshot_id): data, _ = self.store.read(self.archives + snapshot_id + "/manifest.json") return Snapshot.decode(data, snapshot_id) - def preflight(self, snapshot, rollback): + def preflight(self, snapshot): active = self.state["active"] - if not rollback and active != snapshot.id: + if active != snapshot.id: if active != snapshot.base: raise Conflict("Repository advanced after preparation; regenerate against the current base") if active: @@ -327,23 +327,26 @@ def preflight(self, snapshot, rollback): if not mutable(name): if old is not None and not matches(old, entry): raise Conflict("Immutable object conflict: " + name) - if rollback and old is None: - raise Conflict("Rollback payload/index no longer exists: " + name) if old is None and (snapshot.source is None or not (snapshot.source / name).is_file()): raise Conflict("Inherited payload/index no longer exists: " + name) - def publish(self, snapshot, *, rollback=False): + def publish(self, snapshot, *, recovery=None): self.acquire() try: - pending = self.state["pending"] - if pending and snapshot.id != pending and not ( - rollback and snapshot.id == self.state["active"] - ): + active, pending = self.state["active"], self.state["pending"] + if recovery is not None: + # Recovery may abandon a pending generation only from the exact + # state whose signed metadata was checked during preparation. + expected_active, expected_pending = recovery + allowed = {(expected_active, expected_pending), + (expected_active, snapshot.id), (snapshot.id, None)} + if snapshot.base != expected_active or (active, pending) not in allowed: + raise Conflict("Repository changed after rollback preparation; inspect and prepare again") + elif pending and snapshot.id != pending: raise Conflict("Recover the pending snapshot or roll back to the last completed one") - self.preflight(snapshot, rollback) + self.preflight(snapshot) # Nothing above changes client-visible metadata. - if not rollback: - self.archive(snapshot) + self.archive(snapshot) metadata = {} for name, entry in snapshot.entries.items(): if mutable(name): @@ -353,8 +356,9 @@ def publish(self, snapshot, *, rollback=False): metadata[name] = data self.state["pending"] = snapshot.id self.write_state() + self.hook("pending") for name, entry in sorted(snapshot.entries.items()): - if not mutable(name) and not rollback: + if not mutable(name): # Preflight verified inherited immutable objects under the lock. if not (snapshot.source / name).exists(): continue diff --git a/distribution/repositories/tests/package_fixture.py b/distribution/repositories/tests/package_fixture.py index a81d85ad8e5f6..91b62c1077174 100644 --- a/distribution/repositories/tests/package_fixture.py +++ b/distribution/repositories/tests/package_fixture.py @@ -100,6 +100,8 @@ def gpg(*args, protected=False): require(promoted.entries["yum/beta/" + relative] == snapshot.entries["yum/stable/" + relative], "Selecting another channel changed RPM signatures") shutil.rmtree(beta) + from tests.rollback_fixture import prepare_fixture as prepare_rollback_fixture + prepare_rollback_fixture(config, work, key, password) from tests.installer_fixture import installer_fixture installer_fixture(config, work, gpg, password) config.pop("public_key_path") @@ -109,7 +111,8 @@ def gpg(*args, protected=False): "primary_fingerprint": primary, "signing_fingerprint": subkey, "checks": ["primary secret rejected", "encrypted subkey signing", "SHA-256 RPM signatures", "apt/dnf metadata signatures", "all six architecture packages", "incremental retained history", - "historical payloads not copied", "channel selection reuses shared Debian bytes and RPM signatures"]})) + "historical payloads not copied", "channel selection reuses shared Debian bytes and RPM signatures", + "fresh rollback and recovery roots", "pending root date covered", "corrupt source rejected before key import"]})) print("PASS: incremental metadata retains both versions without copying old packages", flush=True) print("PASS: explicit beta selection reuses the existing package bytes and RPM signatures", flush=True) finally: diff --git a/distribution/repositories/tests/publication_fixture.py b/distribution/repositories/tests/publication_fixture.py index c9d693386ad0c..a8e122989eddf 100644 --- a/distribution/repositories/tests/publication_fixture.py +++ b/distribution/repositories/tests/publication_fixture.py @@ -8,6 +8,7 @@ from unittest.mock import patch import engine +import rollback from inputs import require from storage import Conflict, Interrupted, Publisher, Snapshot, canonical, mutable @@ -84,19 +85,83 @@ def hook(actual): engine.publish(config, two) capture("current") reports.append("incremental publication resumed after payload and RPM root/signature interruptions") - publisher.publish(publisher.archived(first.id), rollback=True) + # This historical view reproduces apt's rejection of an older Release date. + shutil.copytree(views / "initial", views / "stale-rollback", copy_function=os.link) + + def exercise_rollback(configuration, fixture, target, label, interrupt=False): + request = work / label + request.mkdir() + rollback.base(configuration, request, target, store) + require((request / "input.json").read_bytes() == (fixture / "input.json").read_bytes(), + "S3 preparation differs from the independently signed recovery request") + require(not (request / "result").exists(), "Unexpected rollback cache hit") + shutil.copytree(fixture / "result", request / "result", copy_function=os.link) + signed = Snapshot.local(request / "result/files") + if interrupt: + def factory(store, configuration): + def hook(phase): + if phase == "written:apt/dists/stable/InRelease": + raise Interrupted("intentional rollback interruption") + return Publisher(store, configuration, hook) + with patch.object(rollback, "Publisher", factory): + try: + rollback.publish(configuration, request, store) + except Interrupted: + pass + else: + raise AssertionError("Expected rollback interruption was not reached") + resumed = work / (label + "-resume") + resumed.mkdir() + rollback.base(configuration, resumed, target, store) + require(Snapshot.local(resumed / "result/files").id == signed.id, "Resume changed the signed rollback") + rollback.publish(configuration, resumed, store) + else: + rollback.publish(configuration, request, store) + live = Publisher(store, configuration).live + before = {name: store.head(live + name)["VersionId"] for name in signed.entries} + retry = work / (label + "-retry") + retry.mkdir() + rollback.base(configuration, retry, target, store) + require(Snapshot.local(retry / "result/files").id == signed.id, "Completed retry changed the signed rollback") + rollback.publish(configuration, retry, store) + require(before == {name: store.head(live + name)["VersionId"] for name in signed.entries}, + "Rollback retry replaced a live object") + return signed + + restored = exercise_rollback(config, work / "rollback", first.id, "restore", interrupt=True) capture("rollback") - engine.publish(config, two) + recovered = exercise_rollback(config, work / "recovered", second.id, "forward") capture("recovered") - reports.append("archived metadata rollback and re-publication preserved immutable history") - before = live_versions(second) + reports.append("fresh signed rollback and forward recovery preserve immutable history and advance apt dates") + reports.append("interrupted and completed rollback retries restore exact signed bytes without a key or extra live versions") + + pending_config = dict(config) + for field in ("live_prefix", "archive_prefix", "bundle_prefix"): + pending_config[field] += "pending-test/" + pending_config["control_key"] = config["control_key"].removesuffix(".json") + "-pending.json" + engine.publish(pending_config, one) + def pending_factory(store, configuration): + def hook(phase): + if phase == "pending": + raise Interrupted("stopped before any new payload or index upload") + return Publisher(store, configuration, hook) + with patch.object(engine, "Publisher", pending_factory): + try: + engine.publish(pending_config, two) + except Interrupted: + pass + else: + raise AssertionError("Expected early publication interruption was not reached") + exercise_rollback(pending_config, work / "rollback-pending", first.id, "abandon-pending") + reports.append("pending publication abandoned safely even when its immutable RPM indexes were never uploaded") + before = live_versions(recovered) try: publisher.publish(first) except Conflict: pass else: raise AssertionError("An old preparation replaced newer repository metadata") - require(live_versions(second) == before, "Stale input modified live data") + require(live_versions(recovered) == before, "Stale input modified live data") reports.append("stale prepared generation rejected before any live object changed") public = work / "public" public.mkdir() @@ -116,7 +181,7 @@ def hook(actual): if path.name.endswith(".rpm") and not path.name.endswith("." + rpm_arch + ".rpm"): continue archive.add(path, arcname="views/" + path.relative_to(views).as_posix()) - report = {"first": first.id, "second": second.id, "checks": reports, + report = {"first": first.id, "second": second.id, "rollback": restored.id, "recovered": recovered.id, "checks": reports, "state": publisher.load_state()[0]} (work / "publication-report.json").write_bytes(canonical(report)) for item in reports: diff --git a/distribution/repositories/tests/rollback_fixture.py b/distribution/repositories/tests/rollback_fixture.py new file mode 100644 index 0000000000000..73dafaaaf3d25 --- /dev/null +++ b/distribution/repositories/tests/rollback_fixture.py @@ -0,0 +1,81 @@ +"""Pre-sign recovery cases without exposing a private key to S3 publication.""" + +import os +from pathlib import Path +import shutil +from unittest.mock import patch + +from inputs import require +from repository import repomd_locations +import rollback +from storage import PublicationError, Snapshot, mutable + + +def seed(work, target, active, pending=None): + work.mkdir() + data = rollback.plan_bytes(target.id, active.id, pending.id if pending else None) + (work / "input.json").write_bytes(data) + for snapshot in {item.id: item for item in (target, active, pending) if item}.values(): + directory = work / "sources" / snapshot.id + root = directory / "files" + root.mkdir(parents=True) + (directory / "manifest.json").write_bytes(snapshot.data) + names = {name for name in snapshot.entries if mutable(name)} + if snapshot.id in {target.id, active.id}: + for package_root in snapshot.source.glob("yum/*/vector-0/*"): + names.update((package_root / name).relative_to(snapshot.source).as_posix() + for name in repomd_locations(package_root)) + for name in names: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(snapshot.source / name, path) + + +def prepare_fixture(config, work, key, password): + first = Snapshot.local(work / "one/result/files") + second = Snapshot.local(work / "two/result/files") + back = work / "rollback" + seed(back, first, second) + os.environ.update(PACKAGE_SIGNING_SUBKEY=key, PACKAGE_SIGNING_PASSPHRASE=password) + rollback.prepare(config, back) + restored = Snapshot.local(back / "result/files") + forward = work / "recovered" + seed(forward, second, restored) + os.environ.update(PACKAGE_SIGNING_SUBKEY=key, PACKAGE_SIGNING_PASSPHRASE=password) + rollback.prepare(config, forward) + interrupted = work / "rollback-pending" + seed(interrupted, first, first, second) + os.environ.update(PACKAGE_SIGNING_SUBKEY=key, PACKAGE_SIGNING_PASSPHRASE=password) + rollback.prepare(config, interrupted) + for name, entry in second.entries.items(): + if not mutable(name): + require(restored.entries[name] == entry, "Rollback removed immutable history") + require(not any(path.name.endswith((".deb", ".rpm")) for path in back.rglob("*")), + "Rollback copied package payloads") + + # A corrupt downloaded source must fail before importing the private key. + broken = work / "rollback-corrupt" + seed(broken, first, second) + path = broken / "sources" / first.id / "files/apt/dists/stable/Release" + path.write_bytes(path.read_bytes() + b"Corruption\n") + with patch.object(rollback, "Signer") as signer: + try: + rollback.prepare(config, broken) + except PublicationError: + pass + else: + raise AssertionError("Corrupt rollback source was accepted") + signer.assert_not_called() + shutil.rmtree(broken) + + # A valid signed bundle cannot be relabeled as another selection. + original = (back / "input.json").read_bytes() + (back / "input.json").write_bytes(rollback.plan_bytes(second.id, second.id, None)) + try: + rollback.check_result(config, back) + except PublicationError: + pass + else: + raise AssertionError("Rollback accepted a signed bundle for another selection") + (back / "input.json").write_bytes(original) + print("PASS: fresh signed rollback preserves package bytes, validates sources before key import, and binds the selected archive", flush=True) diff --git a/distribution/repositories/tests/test_rollback.py b/distribution/repositories/tests/test_rollback.py new file mode 100644 index 0000000000000..5a16338bb99de --- /dev/null +++ b/distribution/repositories/tests/test_rollback.py @@ -0,0 +1,55 @@ +from datetime import datetime, timedelta, timezone +from email.utils import format_datetime +from pathlib import Path +import sys +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from repository import dated_release, next_release_date, release_date +from rollback import decode_plan, plan_bytes +from storage import PublicationError + + +class RollbackDateTests(unittest.TestCase): + def setUp(self): + self.now = datetime(2026, 10, 20, 12, tzinfo=timezone.utc) + + def root(self, date): + return ("Suite:\nCodename: stable\nDate: " + format_datetime(date) + + "\nSHA256:\n abc 3 vector-0/binary-amd64/Packages\n").encode() + + def test_new_date_supersedes_cached_and_interrupted_roots_without_changing_identity(self): + old = self.root(self.now - timedelta(days=30)) + cached = self.root(self.now) + pending = self.root(self.now + timedelta(seconds=1)) + refreshed = dated_release(old, next_release_date([old, cached, pending], self.now)) + self.assertGreater(release_date(refreshed), release_date(pending)) + self.assertEqual(dated_release(refreshed, "DATE"), dated_release(old, "DATE")) + + def test_normal_release_immediately_after_rollback_also_advances(self): + rollback = self.root(self.now + timedelta(seconds=1)) + later = self.root(self.now + timedelta(seconds=2)) + self.assertEqual(next_release_date([rollback], self.now), format_datetime(release_date(later))) + + def test_future_clock_drift_is_rejected_instead_of_disabling_apt_checks(self): + with self.assertRaises(PublicationError): + next_release_date([self.root(self.now + timedelta(minutes=1))], self.now) + + def test_ambiguous_date_or_expiry_policy_is_rejected(self): + root = self.root(self.now) + for extra in (b"Date: Tue, 20 Oct 2026 12:00:01 +0000\n", b"Valid-Until: Wed, 21 Oct 2026 12:00:00 +0000\n"): + with self.assertRaises(PublicationError): + dated_release(root + extra, next_release_date([], self.now)) + + def test_pending_publication_can_only_be_abandoned_to_last_completed_selection(self): + active, pending, other = "a" * 64, "b" * 64, "c" * 64 + plan, _ = decode_plan(plan_bytes(active, active, pending)) + self.assertEqual(plan["pending"], pending) + with self.assertRaises(PublicationError): + plan_bytes(other, active, pending) + with self.assertRaises(PublicationError): + plan_bytes("../escape", active, None) + + +if __name__ == "__main__": + unittest.main() diff --git a/distribution/repositories/tests/test_storage.py b/distribution/repositories/tests/test_storage.py index 052bb40badfec..46fc7f9521209 100644 --- a/distribution/repositories/tests/test_storage.py +++ b/distribution/repositories/tests/test_storage.py @@ -172,9 +172,40 @@ def stop(phase): self.assertEqual(state["pending"], self.two.id) with self.assertRaises(Conflict): self.publisher.publish(self.snapshot("three", "three")) - self.publisher.publish(self.publisher.archived(self.one.id), rollback=True) + recovery = self.snapshot("recovery", "fresh signed old selection") + self.publisher.publish(recovery, recovery=(self.one.id, self.two.id)) + self.assertEqual(self.publisher.load_state()[0]["active"], recovery.id) self.assertIsNone(self.publisher.load_state()[0]["pending"]) + before = self.live() + self.publisher.publish(recovery, recovery=(self.one.id, self.two.id)) + self.assertEqual(self.live(), before) + with self.assertRaises(Conflict): + self.publisher.publish(self.two) + + def test_recovery_rejects_changed_active_or_pending_state_before_live_writes(self): + self.publisher.publish(self.one) + recovery = self.snapshot("recovery", "fresh signed old selection") + before = self.live() + with self.assertRaises(Conflict): + self.publisher.publish(recovery, recovery=(self.one.id, self.two.id)) + self.assertEqual(self.live(), before) self.publisher.publish(self.two) + before = self.live() + with self.assertRaises(Conflict): + self.publisher.publish(recovery, recovery=(self.one.id, None)) + self.assertEqual(self.live(), before) + + def test_recovery_can_resume_its_own_interrupted_generation(self): + self.publisher.publish(self.one) + recovery = self.snapshot("recovery", "fresh signed old selection") + def stop(phase): + if phase.endswith("repomd.xml"): + raise Interrupted("torn recovery pair") + with self.assertRaises(Interrupted): + Publisher(self.store, CONFIG, stop).publish(recovery, recovery=(self.one.id, None)) + self.assertEqual(self.publisher.load_state()[0]["pending"], recovery.id) + self.publisher.publish(recovery, recovery=(self.one.id, None)) + self.assertEqual(self.publisher.load_state()[0]["active"], recovery.id) def test_lost_write_response_retains_lock_and_blocks_early_recovery(self): self.publisher.publish(self.one) @@ -198,10 +229,10 @@ def test_corrupt_archived_metadata_fails_before_any_live_change(self): self.publisher.publish(self.one) self.publisher.publish(self.two) before = self.live() - key = self.publisher.archives + self.one.id + "/files/apt/dists/stable/InRelease" + key = self.publisher.archives + self.two.id + "/files/apt/dists/stable/InRelease" self.store.put(key, b"bad", byte_entry(b"bad"), previous=self.store.head(key)) with self.assertRaises(Conflict): - self.publisher.publish(self.publisher.archived(self.one.id), rollback=True) + self.publisher.publish(self.two) self.assertEqual(before, self.live()) def test_manifest_rejects_extra_file_changed_bytes_symlink_and_traversal(self): diff --git a/docs/reference/package-repositories.md b/docs/reference/package-repositories.md index 47c75b3f2a7ec..e5c9a8179a968 100644 --- a/docs/reference/package-repositories.md +++ b/docs/reference/package-repositories.md @@ -186,19 +186,18 @@ that is the intended operation. ## Recovery Use the manually dispatched `Package Repository Maintenance` workflow on `master` -for `status`, `rollback` or `recover-lock`. It uses the protected publishing -environment, its OIDC role and the same concurrency group as release publication. -It receives no signing secrets. Maintenance remains available when -`PACKAGE_REPOSITORIES_ENABLED` is disabled so an incident can be recovered while -new publication is stopped. The publishing environment and role must be -configured before maintenance can run. - -The workflow requires the exact archived snapshot digest for rollback. Lock -recovery additionally requires the exact control-object version and confirmation -that the old owner has stopped. The publisher enforces the same 65-minute safety -window for the workflow and CLI. A successful run records the resulting control -state in its summary; public endpoint and client verification still follow a -rollback. +for `status`, `rollback` or `recover-lock`. It shares the release workflow's +concurrency group. Status and lock recovery use only the protected publishing +environment and its OIDC role. Rollback downloads archives there, signs in the +protected signing environment, then publishes and verifies public HTTPS in +separate jobs. Signing never receives AWS credentials; publishing never receives +the private key. + +Maintenance remains available when `PACKAGE_REPOSITORIES_ENABLED` is disabled. +The protected environments and role must be configured before it can run. +Rollback requires an exact archived snapshot digest. Lock recovery also requires +the exact control-object version and confirmation that the old owner stopped. +A successful operation records the resulting control state in its summary. Inspect the state before acting: @@ -217,22 +216,35 @@ cargo vdev release repositories recover-lock \ --expected-version VERSION_ID --confirm-owner-stopped ``` -Resume the pending signed snapshot, or restore the last completed snapshot: +Resume the pending signed publication, or select an archived package set. When +abandoning an incomplete publication, the selection must be the last completed +snapshot. The maintenance workflow performs these phases with separate +credentials; the equivalent CLI sequence uses a fresh workspace: ```sh -cargo vdev release repositories rollback SNAPSHOT_SHA256 +cargo vdev release repositories --work target/repository-rollback rollback base SNAPSHOT_SHA256 +cargo vdev release repositories --work target/repository-rollback rollback prepare +cargo vdev release repositories --work target/repository-rollback rollback publish +cargo vdev release repositories --work target/repository-rollback rollback verify ``` -Rollback restores retained signed metadata and checks that its immutable objects -still exist. It does not downgrade installed packages or restore Datadog-only -client trust. Verify the public endpoints and native clients after rollback. -Never delete a control object to bypass the recovery checks. - -Apt can retain newer cached indexes when a restored Release has an older Date. -The original rollback client tests cleared those caches, so they do not prove -that existing clients adopt the restored snapshot. Before activation, validate a -recovery design that accounts for cached Release dates, such as publishing a -freshly dated and signed generation for the intended package set. Do not disable -apt's date or signature checks globally. This remains an open cutover requirement. -The behavior is visible in +Skip `prepare` when `base` restores a saved signed result. Rollback signs a new apt +Release date, newer than the selected, active and any pending roots. It preserves +the selected indexes and repository identity, reuses the selected yum signatures, +and retains all completed immutable history. Publication rejects a changed base +or pending state. Interrupted and completed retries restore the same signed +bundle without signing again or creating extra live-object versions. + +Replaying old apt signatures alone is insufficient: clients can keep their newer +cached indexes when the restored Release date is older, as shown in [apt's metadata verification](https://sources.debian.org/src/apt/2.2.4/apt-pkg/acquire-item.cc/). +Neither cache deletion nor disabling date/signature checks is part of recovery. +A clock more than five seconds behind a required new date stops preparation; +correct the clock or retry later. Repositories with `Valid-Until` or a changed +metadata layout require a separate recovery policy. + +Rollback changes which versions the indexes advertise. It does not downgrade +installed packages or restore Datadog-only trust. To restore a newer package +selection afterward, dispatch rollback with that newer archived snapshot; do not +replay its old publication. Run native client checks after public verification. +Never delete a control object to bypass recovery checks. diff --git a/vdev/src/commands/release/repositories.rs b/vdev/src/commands/release/repositories.rs index 908220ceab0ab..32424cf8bfff0 100644 --- a/vdev/src/commands/release/repositories.rs +++ b/vdev/src/commands/release/repositories.rs @@ -47,8 +47,11 @@ enum Commands { Verify, /// Print the publication state and its S3 version for operator inspection. Status, - /// Restore archived metadata; this does not downgrade installed packages. - Rollback { snapshot: String }, + /// Restore an archived package selection with fresh signed metadata. + Rollback { + #[command(subcommand)] + phase: Rollback, + }, /// Release an abandoned lock after the fixed credential safety window. RecoverLock { #[arg(long)] @@ -68,6 +71,18 @@ enum Commands { }, } +#[derive(clap::Subcommand, Debug)] +enum Rollback { + /// Download the selected archive and the current/pending roots, or a saved retry. + Base { snapshot: String }, + /// Sign fresh apt roots for the selected indexes; package bytes stay unchanged. + Prepare, + /// Publish the prepared rollback only if its expected state still matches. + Publish, + /// Verify public HTTPS objects against the prepared rollback. + Verify, +} + #[derive(clap::ValueEnum, Clone, Copy, Debug)] enum Channel { Stable, @@ -148,11 +163,16 @@ impl Commands { fn credentials(&self) -> &'static [&'static str] { match self { Self::Fetch { .. } => &["GH_TOKEN"], - Self::Prepare => &["PACKAGE_SIGNING_SUBKEY", "PACKAGE_SIGNING_PASSPHRASE"], + Self::Prepare + | Self::Rollback { + phase: Rollback::Prepare, + } => &["PACKAGE_SIGNING_SUBKEY", "PACKAGE_SIGNING_PASSPHRASE"], Self::Base | Self::Publish | Self::Status - | Self::Rollback { .. } + | Self::Rollback { + phase: Rollback::Base { .. } | Rollback::Publish, + } | Self::RecoverLock { .. } | Self::Test { s3: true, .. } => &[ "AWS_ACCESS_KEY_ID", @@ -195,8 +215,22 @@ impl Commands { Commands::Status => { command.arg("status"); } - Commands::Rollback { snapshot } => { - command.args(["rollback", &snapshot]); + Commands::Rollback { phase } => { + command.arg("rollback"); + match phase { + Rollback::Base { snapshot } => { + command.args(["base", &snapshot]); + } + Rollback::Prepare => { + command.arg("prepare"); + } + Rollback::Publish => { + command.arg("publish"); + } + Rollback::Verify => { + command.arg("verify"); + } + } } Commands::RecoverLock { expected_version, @@ -223,7 +257,7 @@ impl Commands { #[cfg(test)] mod tests { - use super::Commands; + use super::{Commands, Rollback}; #[test] fn signing_and_publishing_receive_disjoint_credentials() { @@ -235,6 +269,28 @@ mod tests { ); assert!(publishing.contains(&"AWS_SESSION_TOKEN")); assert!(signing.iter().all(|name| !publishing.contains(name))); + for phase in [ + Rollback::Base { + snapshot: "test".into(), + }, + Rollback::Publish, + ] { + assert_eq!(Commands::Rollback { phase }.credentials(), publishing); + } + assert_eq!( + Commands::Rollback { + phase: Rollback::Prepare + } + .credentials(), + signing + ); + assert!( + Commands::Rollback { + phase: Rollback::Verify + } + .credentials() + .is_empty() + ); assert!(Commands::Verify.credentials().is_empty()); assert!( Commands::Test { From 0f62aa7f12b0c7005ec9dd2da8c97905f81fcef6 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 20 Oct 2026 15:15:05 -0400 Subject: [PATCH 04/23] docs(packaging): note removal of Datadog key recommendation --- changelog.d/remove_datadog_key_recommendation.enhancement.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog.d/remove_datadog_key_recommendation.enhancement.md diff --git a/changelog.d/remove_datadog_key_recommendation.enhancement.md b/changelog.d/remove_datadog_key_recommendation.enhancement.md new file mode 100644 index 0000000000000..496ec44875082 --- /dev/null +++ b/changelog.d/remove_datadog_key_recommendation.enhancement.md @@ -0,0 +1,3 @@ +Debian packages no longer recommend installing the `datadog-signing-keys` package. + +authors: pront From 8ebff2ed8bb6a838a6c33295c165e0dbd64aa45e Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 20 Oct 2026 15:24:41 -0400 Subject: [PATCH 05/23] refactor(ci): keep package repository fixtures in the private sandbox --- .../package-repositories-maintenance.yml | 1 - .github/workflows/package-repositories.yml | 23 +- .github/workflows/test.yml | 4 +- distribution/repositories/.dockerignore | 2 - distribution/repositories/Dockerfile | 1 - distribution/repositories/engine.py | 14 - distribution/repositories/tests/client.sh | 18 -- .../repositories/tests/installer-client.sh | 237 --------------- .../repositories/tests/installer_fixture.py | 103 ------- .../tests/legacy-apt-identity.json | 9 - .../repositories/tests/package_fixture.py | 122 -------- .../repositories/tests/publication_fixture.py | 188 ------------ .../repositories/tests/rollback_fixture.py | 81 ------ .../repositories/tests/test_inputs.py | 106 ------- .../repositories/tests/test_rollback.py | 55 ---- .../repositories/tests/test_storage.py | 275 ------------------ docs/reference/package-repositories.md | 17 +- vdev/src/commands/release/repositories.rs | 29 +- 18 files changed, 30 insertions(+), 1255 deletions(-) delete mode 100644 distribution/repositories/tests/client.sh delete mode 100644 distribution/repositories/tests/installer-client.sh delete mode 100644 distribution/repositories/tests/installer_fixture.py delete mode 100644 distribution/repositories/tests/legacy-apt-identity.json delete mode 100644 distribution/repositories/tests/package_fixture.py delete mode 100644 distribution/repositories/tests/publication_fixture.py delete mode 100644 distribution/repositories/tests/rollback_fixture.py delete mode 100644 distribution/repositories/tests/test_inputs.py delete mode 100644 distribution/repositories/tests/test_rollback.py delete mode 100644 distribution/repositories/tests/test_storage.py diff --git a/.github/workflows/package-repositories-maintenance.yml b/.github/workflows/package-repositories-maintenance.yml index 8e5aeccea991f..ce2f726e83597 100644 --- a/.github/workflows/package-repositories-maintenance.yml +++ b/.github/workflows/package-repositories-maintenance.yml @@ -56,7 +56,6 @@ jobs: run: | cargo build --locked --package vdev target/debug/vdev release repositories build-image - target/debug/vdev release repositories test - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 with: role-to-assume: ${{ vars.PACKAGE_REPOSITORIES_ROLE_ARN }} diff --git a/.github/workflows/package-repositories.yml b/.github/workflows/package-repositories.yml index 2cad16562698c..36cc62f4c4a94 100644 --- a/.github/workflows/package-repositories.yml +++ b/.github/workflows/package-repositories.yml @@ -57,7 +57,6 @@ jobs: run: | cargo build --locked --package vdev target/debug/vdev release repositories build-image - target/debug/vdev release repositories test - name: Freeze and verify release inputs env: GH_TOKEN: ${{ github.token }} @@ -231,7 +230,23 @@ jobs: export CHANNEL TAG export APT_URL=https://apt.vector.dev/ export YUM_URL=https://yum.vector.dev/ - docker run --rm --env CHANNEL --env TAG --env APT_URL --env YUM_URL \ + docker run --rm --interactive --env CHANNEL --env TAG --env APT_URL --env YUM_URL \ --mount "type=bind,source=$PWD/distribution/repositories/setup.sh,target=/setup.sh,readonly" \ - --mount "type=bind,source=$PWD/distribution/repositories/tests/client.sh,target=/test.sh,readonly" \ - "$IMAGE" bash /test.sh + "$IMAGE" bash -seuo pipefail <<'VERIFY' + version=${TAG#v} + version=${version/-/\~} + VECTOR_DIST_CHANNEL="$CHANNEL" TESTING_APT_URL="$APT_URL" TESTING_YUM_URL="$YUM_URL" bash /setup.sh + if command -v apt-get >/dev/null; then + export DEBIAN_FRONTEND=noninteractive + apt-get install -y "vector=$version-1" + test "$(dpkg-query -W -f='${Version}' vector)" = "$version-1" + else + version=${version//-/.} + dnf install -y "vector-$version-1" + test "$(rpm -q --qf '%{VERSION}-%{RELEASE}' vector)" = "$version-1" + fi + getent passwd vector + getent group vector + vector --version + test -f /usr/share/vector/examples/vector.yaml + VERIFY diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ef7dadd519f0c..22c13c1a21c83 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,12 +79,12 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - # Offline regressions need neither signing material nor AWS access. + # CLI and image checks require neither signing material nor AWS access. - run: | cargo test --locked --package vdev commands::release::repositories:: cargo build --locked --package vdev target/debug/vdev release repositories build-image - target/debug/vdev release repositories test + docker run --rm vector-package-publisher:local --help test: needs: changes diff --git a/distribution/repositories/.dockerignore b/distribution/repositories/.dockerignore index 68a5289c7a3be..840bcab19c537 100644 --- a/distribution/repositories/.dockerignore +++ b/distribution/repositories/.dockerignore @@ -2,7 +2,5 @@ !Dockerfile !requirements.txt !*.py -!tests/ -!tests/** **/__pycache__ diff --git a/distribution/repositories/Dockerfile b/distribution/repositories/Dockerfile index cff4f3e635318..051d103e7349a 100644 --- a/distribution/repositories/Dockerfile +++ b/distribution/repositories/Dockerfile @@ -6,6 +6,5 @@ COPY requirements.txt /app/requirements.txt RUN python3 -m venv /opt/publisher && /opt/publisher/bin/pip install \ --no-cache-dir --require-hashes --only-binary=:all: -r /app/requirements.txt COPY *.py /app/ -COPY tests /app/tests WORKDIR /app ENTRYPOINT ["/opt/publisher/bin/python", "/app/engine.py"] diff --git a/distribution/repositories/engine.py b/distribution/repositories/engine.py index 0c28970ef7c12..c8ba04cb96bb3 100644 --- a/distribution/repositories/engine.py +++ b/distribution/repositories/engine.py @@ -6,7 +6,6 @@ import os from pathlib import Path import re -import sys from urllib.request import urlopen from bundles import checked_download, pack_bundle, unpack_bundle @@ -190,9 +189,6 @@ def main(): download.add_argument("--channel", choices=("stable", "beta"), required=True) for name in ("base", "prepare", "publish", "verify", "status"): commands.add_parser(name) - test_options = commands.add_parser("test").add_mutually_exclusive_group() - test_options.add_argument("--packages", action="store_true") - test_options.add_argument("--s3", action="store_true") rollback_commands = commands.add_parser("rollback").add_subparsers(dest="phase", required=True) rollback_commands.add_parser("base").add_argument("snapshot") for name in ("prepare", "publish", "verify"): @@ -224,16 +220,6 @@ def main(): else: _, snapshot = rollback.check_result(config, work) verify_snapshot(config, snapshot) - elif args.command == "test": - import unittest - suite = unittest.defaultTestLoader.discover("tests") - require(unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful(), "Publisher regressions failed") - if args.packages: - from tests.package_fixture import prepare_fixture - prepare_fixture(config, work) - elif args.s3: - from tests.publication_fixture import publish_fixture - publish_fixture(config, work) else: publisher = Publisher(store_for(config), config) if args.command == "status": diff --git a/distribution/repositories/tests/client.sh b/distribution/repositories/tests/client.sh deleted file mode 100644 index fa9c7c6c6f4f0..0000000000000 --- a/distribution/repositories/tests/client.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -version=${TAG#v} -version=${version/-/\~} -VECTOR_DIST_CHANNEL="$CHANNEL" TESTING_APT_URL="$APT_URL" TESTING_YUM_URL="$YUM_URL" bash /setup.sh -if command -v apt-get >/dev/null; then - export DEBIAN_FRONTEND=noninteractive - apt-get install -y "vector=$version-1" - test "$(dpkg-query -W -f='${Version}' vector)" = "$version-1" -else - version=${version//-/.} - dnf install -y "vector-$version-1" - test "$(rpm -q --qf '%{VERSION}-%{RELEASE}' vector)" = "$version-1" -fi -getent passwd vector -getent group vector -vector --version -test -f /usr/share/vector/examples/vector.yaml diff --git a/distribution/repositories/tests/installer-client.sh b/distribution/repositories/tests/installer-client.sh deleted file mode 100644 index 54511460e99f6..0000000000000 --- a/distribution/repositories/tests/installer-client.sh +++ /dev/null @@ -1,237 +0,0 @@ -#!/usr/bin/env bash -# Run only in a disposable test container with no cloud or signing credentials. -set -Eeuo pipefail -scenario=${1:?Expected early, late, or fresh} -[[ "$scenario" == early || "$scenario" == late || "$scenario" == fresh ]] -[[ -f /.dockerenv && -f /bundle.tar && -f /setup.sh ]] -work=$(mktemp -d) -server_pid= -cleanup() { - status=$? - if ((status)); then - tail -n 60 "$work/command.log" "$work/refresh.log" "$work/http.log" 2>/dev/null || true - if [[ -n "${source_file:-}" && -f "$source_file" ]]; then cat "$source_file"; fi - if command -v apt-cache >/dev/null; then apt-cache policy vector || true; fi - fi - [[ -z "$server_pid" ]] || kill "$server_pid" 2>/dev/null || true - rm -rf "$work" /run/installer-keys/* -} -trap cleanup EXIT -trap 'printf "Installer test failed at line %s\n" "$LINENO" >&2' ERR -mkdir -p "$work/bundle" "$work/http/keys" /run/installer-keys -tar -xf /bundle.tar -C "$work/bundle" -cp "$work/bundle/public/"* "$work/http/keys/" -select_view() { - ln -s "$work/bundle/views/$1" "$work/http/next" - mv -Tf "$work/http/next" "$work/http/current" -} -select_view legacy -openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj /CN=localhost \ - -addext 'subjectAltName=DNS:localhost,IP:127.0.0.1' \ - -keyout /run/installer-keys/tls.key -out /run/installer-keys/tls.crt > "$work/command.log" 2>&1 -if command -v apt-get >/dev/null; then - family=apt - install -m 644 /run/installer-keys/tls.crt /usr/local/share/ca-certificates/vector-installer-test.crt - update-ca-certificates > "$work/command.log" 2>&1 -else - family=rpm - install -m 644 /run/installer-keys/tls.crt /etc/pki/ca-trust/source/anchors/vector-installer-test.crt - update-ca-trust > "$work/command.log" 2>&1 -fi -python3 - "$work/http" > "$work/http.log" 2>&1 <<'PY' & -import http.server, os, socketserver, ssl, sys -class Handler(http.server.SimpleHTTPRequestHandler): - protocol_version = "HTTP/1.1" - def send_head(self): - # The archived views were copied together and may share a filesystem - # mtime. Always return the selected bytes, while apt retains its cached - # Release identity for comparison. CDN caching is a separate test. - if "If-Modified-Since" in self.headers: - del self.headers["If-Modified-Since"] - return super().send_head() -# AlmaLinux 8's system Python predates ThreadingHTTPServer and the directory -# parameter. Keep the fixture compatible without changing the client runtime. -class Server(socketserver.ThreadingMixIn, http.server.HTTPServer): - daemon_threads = True -os.chdir(sys.argv[1]) -server = Server(("127.0.0.1", 18766), Handler) -context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) -context.load_cert_chain("/run/installer-keys/tls.crt", "/run/installer-keys/tls.key") -server.socket = context.wrap_socket(server.socket, server_side=True) -server.serve_forever() -PY -server_pid=$! -for _ in $(seq 1 30); do curl -fsS https://localhost:18766/ >/dev/null 2>&1 && break; sleep .2; done -curl -fsS https://localhost:18766/ >/dev/null -export VECTOR_PUBLIC_KEY_URL=https://localhost:18766/keys/cose.asc -VECTOR_PUBLIC_KEY_FINGERPRINT=$(cat "$work/bundle/public/fingerprint") -export VECTOR_PUBLIC_KEY_FINGERPRINT -base=https://localhost:18766/current -pass() { printf 'PASS: %s %s: %s\n' "$family" "$scenario" "$*"; } -setup() { bash /setup.sh "$@"; } -expect_failure() { - local pattern=$1; shift - if "$@" > "$work/command.log" 2>&1; then printf 'Unexpected success: %s\n' "$*" >&2; exit 1; fi - grep -Eiq "$pattern" "$work/command.log" || { cat "$work/command.log"; exit 1; } -} -if [[ "$family" == apt ]]; then - source_file=/etc/apt/sources.list.d/vector.list - shared_key=/usr/share/keyrings/datadog-archive-keyring.gpg - owned_key=/usr/share/keyrings/vector-archive-keyring.gpg - timber_file=/etc/apt/sources.list.d/timber-vector.list - refresh() { apt-get -o Dir::Etc::sourcelist=sources.list.d/vector.list -o Dir::Etc::sourceparts=- -o APT::Get::List-Cleanup=0 -o APT::Update::Error-Mode=any update; } - install_version() { DEBIAN_FRONTEND=noninteractive apt-get -y --allow-downgrades install "vector=$1-1"; } - # Resolve normal distro dependencies separately; later refreshes retain the - # actual cached Vector Release information across the simulated cutover. - apt-get update > "$work/command.log" 2>&1 -else - source_file=/etc/yum.repos.d/vector.repo - shared_key=/etc/pki/rpm-gpg/LEGACY-INSTALLER-TEST - owned_key=/etc/pki/rpm-gpg/RPM-GPG-KEY-vector - timber_file=/etc/yum.repos.d/timber-vector.repo - refresh() { - dnf -y --disablerepo='*' --enablerepo=vector clean metadata - dnf -y --disablerepo='*' --enablerepo=vector --refresh makecache - } - install_version() { dnf -y install "vector-$1-1"; } -fi - -if [[ "$scenario" == fresh ]]; then - select_view upgraded - expect_failure 'requires an existing|No .* found' setup --migrate - expect_failure 'cannot be disabled' env VEC_RPM_REPO_GPGCHECK=0 bash /setup.sh - expect_failure 'HTTPS URL' env VECTOR_PUBLIC_KEY_URL=http://localhost/key bash /setup.sh - expect_failure 'pinned primary fingerprint' env VECTOR_PUBLIC_KEY_URL=https://localhost:18766/keys/legacy.asc bash /setup.sh - [[ ! -e "$source_file" && ! -e "$owned_key" ]] - pass 'invalid trust inputs fail before creating repository files' - - # Both legacy override spellings accept HTTPS URLs and retain strict checks. - VECTOR_DIST_CHANNEL=beta TESTING_APT_URL="$base/apt" TESTING_YUM_URL="$base/yum" setup > "$work/command.log" 2>&1 - grep -q beta "$source_file" - setup --migrate > "$work/command.log" 2>&1 - grep -q beta "$source_file" - pass 'fresh beta setup and repeated migration retain the channel' - rm "$source_file" "$source_file.pre-cose" - TESTING_APT_REPO_VERSION='stable vector-0' TESTING_YUM_VERSION_PATH=stable/vector-0 \ - TESTING_APT_URL="$base/apt" TESTING_YUM_URL="$base/yum" setup > "$work/command.log" 2>&1 - if [[ "$family" == apt ]]; then - [[ $(gpg --batch --with-colons --show-keys "$owned_key" | grep -c '^pub:') == 1 ]] - else - grep -qx 'gpgcheck=1' "$source_file" - grep -qx 'repo_gpgcheck=1' "$source_file" - if grep -q legacy "$source_file"; then exit 1; fi - fi - install_version 0.60.0 > "$work/command.log" 2>&1 - vector --version | grep -F 'vector 0.60.0 ' - getent passwd vector >/dev/null - getent group vector >/dev/null - pass 'fresh stable installation uses only COSE trust over verified HTTPS' - exit 0 -fi - -if [[ "$family" == apt ]]; then - install -m 644 "$work/bundle/public/legacy.gpg" "$shared_key" - printf '# existing Vector settings\ndeb [arch=%s signed-by=%s] %s/apt/ stable vector-0\n' \ - "$(dpkg --print-architecture)" "$shared_key" "$base" > "$source_file" -else - install -m 644 "$work/bundle/public/legacy.asc" "$shared_key" - rpm --import "$shared_key" - cat > "$source_file" < "$work/command.log" 2>&1 -install_version 0.59.0 > "$work/command.log" 2>&1 -vector --version | grep -F 'vector 0.59.0 ' -mkdir -p /etc/vector -printf '# preserve installer migration configuration\n' > /etc/vector/vector.yaml -account=$(id vector) -pass 'legacy installation succeeds with cached metadata and legacy-only trust' - -expect_failure 'pinned primary fingerprint' env VECTOR_PUBLIC_KEY_URL=https://localhost:18766/keys/legacy.asc bash /setup.sh --migrate -expect_failure 'fresh setup only' env VECTOR_DIST_CHANNEL=beta bash /setup.sh --migrate -if [[ "$family" == apt ]]; then - printf 'Types: deb\nURIs: %s/apt\nSuites: stable\nComponents: vector-0\n' "$base" > /etc/apt/sources.list.d/custom-vector.sources - expect_failure 'Additional Vector source' setup --migrate - rm /etc/apt/sources.list.d/custom-vector.sources -else - # shellcheck disable=SC2016 - printf '[other-vector]\nbaseurl=%s/yum/stable/vector-0/$basearch/\n' "$base" > /etc/yum.repos.d/custom-vector.repo - expect_failure 'Additional Vector source' setup --migrate - rm /etc/yum.repos.d/custom-vector.repo -fi -[[ "$source_hash" == "$(sha256sum "$source_file")" && ! -e "$owned_key" && ! -e "$source_file.pre-cose" ]] -pass 'wrong key, channel override and ambiguous custom sources leave existing files untouched' - -if [[ "$scenario" == late ]]; then - select_view cose - expect_failure 'NO_PUBKEY|public key|GPG|signature' refresh - pass 'a client that missed the announcement rejects the new key before migration' -else - # The opt-in legacy Cloudsmith file is disabled with a recoverable backup. - printf '# old source at https://repositories.timber.io/public/vector\n' > "$timber_file" - export CSM_MIGRATE=true -fi -setup --migrate > "$work/command.log" 2>&1 -[[ "$shared_hash" == "$(sha256sum "$shared_key")" ]] -[[ $(sha256sum "$source_file.pre-cose" | cut -d ' ' -f 1) == $(sha256sum "$work/original-source" | cut -d ' ' -f 1) ]] -first_source_hash=$(sha256sum "$source_file") -first_key_hash=$(sha256sum "$owned_key") -# Subsequent migration and metadata verification must no longer need the old -# remote key endpoint. The shared key on disk remains available to other apps. -rm "$work/http/keys/legacy.asc" -setup --migrate > "$work/command.log" 2>&1 -[[ "$first_source_hash" == "$(sha256sum "$source_file")" && "$first_key_hash" == "$(sha256sum "$owned_key")" ]] -if [[ "$family" == apt ]]; then - [[ $(gpg --batch --with-colons --show-keys "$owned_key" | grep -c '^pub:') == 2 ]] - grep -q "arch=$(dpkg --print-architecture)" "$source_file" -else - grep -qx "gpgkey=file://$owned_key" "$source_file" - [[ $(gpg --batch --with-colons --show-keys "$owned_key" | grep -c '^pub:') == 2 ]] - grep -qx priority=7 "$source_file" - grep -qx exclude=other-package "$source_file" - grep -qx gpgcheck=1 "$source_file" - grep -qx repo_gpgcheck=1 "$source_file" -fi -pass 'migration is byte-idempotent, retains legacy trust and options, and leaves shared keys unchanged' -if [[ "$scenario" == early ]]; then - [[ ! -e "$timber_file" && -f "$timber_file.pre-cose" ]] - refresh > "$work/command.log" 2>&1 - pass 'pre-cutover migration still authenticates the legacy repository' - select_view cose - refresh > "$work/command.log" 2>&1 - pass 'cutover succeeds with cached apt identity and no release-information bypass' -fi -select_view upgraded -refresh > "$work/refresh.log" 2>&1 -install_version 0.60.0 > "$work/command.log" 2>&1 -vector --version | grep -F 'vector 0.60.0 ' -grep -q 'preserve installer migration configuration' /etc/vector/vector.yaml -[[ "$account" == "$(id vector)" ]] -[[ "$shared_hash" == "$(sha256sum "$shared_key")" ]] -pass 'upgrade to 0.60.0 over verified HTTPS preserves configuration and account' - -# After cutover, legacy trust can be retired from the Vector source even if the -# old key endpoint has gone away. Other packages' shared key files stay intact. -install -m 644 "$work/original-source" "$source_file" -if [[ "$family" == apt ]]; then - sed -i "s|$shared_key|/usr/share/keyrings/vector-unavailable-test.gpg|" "$source_file" -fi -setup --migrate --cose-only > "$work/command.log" 2>&1 -[[ $(gpg --batch --with-colons --show-keys "$owned_key" | grep -c '^pub:') == 1 ]] -[[ "$shared_hash" == "$(sha256sum "$shared_key")" ]] -refresh > "$work/command.log" 2>&1 -pass 'post-cutover migration from original settings needs no legacy key source and preserves shared keys' diff --git a/distribution/repositories/tests/installer_fixture.py b/distribution/repositories/tests/installer_fixture.py deleted file mode 100644 index 3411cb0edc603..0000000000000 --- a/distribution/repositories/tests/installer_fixture.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Private installer fixtures, with a separate ephemeral legacy signing identity.""" - -import json -import os -from datetime import timedelta -from email.utils import format_datetime, parsedate_to_datetime -from pathlib import Path -import shutil -import tarfile - -from inputs import require -from repository import Signer, key_fingerprints, prepare -from storage import Snapshot, canonical, sha256_file - - -def installer_fixture(config, work, gpg, password): - gpg("--quick-generate-key", "Vector simulated legacy repository", "rsa4096", "cert", "2d", protected=True) - primary = key_fingerprints(gpg("--with-colons", "--list-keys", "Vector simulated legacy repository"), "pub", "sub")[0][1] - gpg("--quick-add-key", primary, "rsa4096", "sign", "2d", protected=True) - subkey = key_fingerprints(gpg("--with-colons", "--list-keys", primary), "pub", "sub")[1][1] - private = gpg("--armor", "--export-secret-subkeys", subkey + "!", protected=True).decode() - public = work / "installer/public" - public.mkdir(parents=True) - (public / "legacy.asc").write_bytes(gpg("--armor", "--export", primary)) - (public / "legacy.gpg").write_bytes(gpg("--export", primary)) - shutil.copyfile(config["public_key_path"], public / "cose.asc") - (public / "fingerprint").write_text(config["primary_fingerprint"] + "\n") - legacy_config = {**config, "primary_fingerprint": primary, "signing_fingerprint": subkey, - "public_key_path": str(public / "legacy.asc")} - legacy = work / "installer-legacy" - legacy.mkdir() - shutil.copyfile(work / "one/input.json", legacy / "input.json") - shutil.copytree(work / "one/input", legacy / "input", copy_function=os.link) - (legacy / "base").mkdir() - (legacy / "base.json").write_bytes(b"null\n") - os.environ.update(PACKAGE_SIGNING_SUBKEY=private, PACKAGE_SIGNING_PASSPHRASE=password) - prepare(legacy_config, legacy) - # Model the observed Datadog Release fields, without pretending that this - # fixture has Datadog's signing key or its non-Vector package payloads. - observed = json.loads((Path(__file__).parent / "legacy-apt-identity.json").read_bytes()) - signer = Signer(legacy_config, "/run/keys/legacy-metadata", private, password) - try: - for channel in ("stable", "beta"): - suite = legacy / "result/files/apt/dists" / channel - release = suite / "Release" - lines = [line for line in release.read_text().splitlines() - if not line.startswith(("Origin:", "Label:", "Suite:", "Components:", "Acquire-By-Hash:", "Date:"))] - first = (work / "one/result/files/apt/dists" / channel / "Release").read_text() - date = next(line[6:] for line in first.splitlines() if line.startswith("Date: ")) - # Apt rejects Release date regressions while retaining its cached - # indexes. This simulated legacy publication must precede cutover. - lines.insert(0, "Date: " + format_datetime(parsedate_to_datetime(date) - timedelta(hours=1))) - lines.insert(0, ("Suite: " + observed["Suite"]).rstrip()) - lines.insert(1, "Components: " + observed["Components"]) - release.write_text("\n".join(lines) + "\n") - signer.sign(release, clear=True, destination=suite / "InRelease") - signer.sign(release, destination=suite / "Release.gpg") - finally: - signer.close() - private = None - original = Snapshot.decode((legacy / "result/manifest.json").read_bytes()) - entries = {p.relative_to(legacy / "result/files").as_posix(): {"size": p.stat().st_size, "sha256": sha256_file(p)} - for p in (legacy / "result/files").rglob("*") if p.is_file()} - (legacy / "result/manifest.json").write_bytes(Snapshot(entries, input_id=original.input_id).data) - - views = work / "installer/views" - for view, sources in ( - ("legacy", [legacy / "result/files"]), - ("cose", [work / "one/result/files"]), - ("upgraded", [work / "one/result/files", work / "two/result/files"]), - ): - for source in sources: - for path in source.rglob("*"): - if path.is_file(): - target = views / view / path.relative_to(source) - target.parent.mkdir(parents=True, exist_ok=True) - # Copies permit client-side tamper tests without modifying - # another view through a shared inode. - shutil.copyfile(path, target) - for channel in ("stable", "beta"): - release = (views / "cose/apt/dists" / channel / "Release").read_text() - require("Origin:" not in release and "Label:" not in release and "Suite:\n" in release, - "The new apt repository would require release-information approval") - output = work / "installer-bundles" - output.mkdir() - for deb_arch, rpm_arch in (("amd64", "x86_64"), ("arm64", "aarch64")): - with tarfile.open(output / (deb_arch + ".tar"), "w") as archive: - for path in sorted((work / "installer").rglob("*")): - if not path.is_file(): - continue - if path.name.endswith(".deb") and not path.name.endswith("_" + deb_arch + ".deb"): - continue - if path.name.endswith(".rpm") and not path.name.endswith("." + rpm_arch + ".rpm"): - continue - archive.add(path, arcname=path.relative_to(work / "installer").as_posix()) - (work / "installer-report.json").write_bytes(canonical({ - "primary_fingerprint": config["primary_fingerprint"], - "legacy_fingerprint": primary, - "checks": ["independent ephemeral legacy signing key", "legacy apt identity fields captured from live metadata", - "unchanged apt Origin, Label, Suite and Codename", "legacy metadata predates COSE publications", - "three signed repository views for native HTTPS tests"], - })) - print("PASS: installer fixtures preserve the observed apt identity with independent legacy and COSE keys", flush=True) diff --git a/distribution/repositories/tests/legacy-apt-identity.json b/distribution/repositories/tests/legacy-apt-identity.json deleted file mode 100644 index 6125005a85fe4..0000000000000 --- a/distribution/repositories/tests/legacy-apt-identity.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "source": "https://apt.vector.dev/dists/stable/Release", - "observed": "2026-10-20", - "Origin": null, - "Label": null, - "Suite": "", - "Codename": "stable", - "Components": "main 6 7 observability-pipelines-worker-1 vector-0 observability-pipelines-worker-2 agentless-scanner datadog-security-cli datadog-bits-cli" -} diff --git a/distribution/repositories/tests/package_fixture.py b/distribution/repositories/tests/package_fixture.py deleted file mode 100644 index 91b62c1077174..0000000000000 --- a/distribution/repositories/tests/package_fixture.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Opt-in real-package fixture. Its ephemeral private keys never leave tmpfs.""" - -import json -import os -from pathlib import Path -import secrets -import shutil - -from inputs import require -from repository import Signer, key_fingerprints, prepare, run -from storage import PublicationError, Snapshot, canonical - - -def prepare_fixture(configuration, work): - require(configuration["account"] == "092620891642" and configuration["bucket"] == "pront-ci-sandbox-092620891642-us-east-1", - "Real-package fixture generation is restricted to the private COSE sandbox") - config = dict(configuration) - home = Path("/run/keys/generate") - home.mkdir(mode=0o700) - password = secrets.token_urlsafe(48) - env = {**os.environ, "GNUPGHOME": str(home)} - def gpg(*args, protected=False): - command = ["gpg", "--batch", "--no-tty", "--rfc4880", "--cert-digest-algo", "SHA256"] - if protected: - command += ["--pinentry-mode", "loopback", "--passphrase-fd", "0"] - return run([*command, *args], env=env, data=(password + "\n").encode() if protected else None) - try: - gpg("--quick-generate-key", "Vector ephemeral repository test", "rsa4096", "cert", "2d", protected=True) - primary = key_fingerprints(gpg("--with-colons", "--list-keys"), "pub", "sub")[0][1] - gpg("--quick-add-key", primary, "rsa4096", "sign", "2d", protected=True) - subkey = key_fingerprints(gpg("--with-colons", "--list-keys"), "pub", "sub")[1][1] - key = gpg("--armor", "--export-secret-subkeys", subkey + "!", protected=True).decode() - private_primary = gpg("--armor", "--export-secret-keys", primary, protected=True).decode() - public = gpg("--armor", "--export", primary) - directory = work / "config" - directory.mkdir() - (directory / "public.asc").write_bytes(public) - (directory / "public.gpg").write_bytes(gpg("--export", primary)) - config.update(primary_fingerprint=primary, signing_fingerprint=subkey, public_key="public.asc", - public_key_path=str(directory / "public.asc"), require_existing_repository=False) - rejected = False - try: - Signer(config, "/run/keys/reject-primary", private_primary, password) - except PublicationError: - rejected = True - finally: - run(["gpgconf", "--kill", "all"], env={**os.environ, "GNUPGHOME": "/run/keys/reject-primary"}) - shutil.rmtree("/run/keys/reject-primary") - require(rejected, "The signer accepted a primary secret key") - private_primary = None - print("PASS: production signer rejects a usable primary secret key", flush=True) - one, two = work / "one", work / "two" - (one / "base").mkdir() - (one / "base.json").write_bytes(b"null\n") - os.environ.update(PACKAGE_SIGNING_SUBKEY=key, PACKAGE_SIGNING_PASSPHRASE=password) - first = prepare(config, one) - print("PASS: encrypted subkey prepared the first six-package release", flush=True) - (two / "base").mkdir() - (two / "base.json").write_bytes((one / "result/manifest.json").read_bytes()) - for source in (one / "result/files").rglob("*"): - if source.is_file() and not source.name.endswith((".deb", ".rpm")): - target = two / "base" / source.relative_to(one / "result/files") - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(source, target) - os.environ.update(PACKAGE_SIGNING_SUBKEY=key, PACKAGE_SIGNING_PASSPHRASE=password) - second = prepare(config, two) - snapshot = Snapshot.decode((two / "result/manifest.json").read_bytes()) - require(snapshot.base == first, "Incremental generation lost its base") - for path in (two / "result/files/apt/dists/stable/vector-0").glob("binary-*/Packages"): - if path.parent.name not in {"binary-all", "binary-i386"}: - require(path.read_text().count("Package: vector\n") == 2, "Incremental apt history lost a release") - for name in Snapshot.decode((one / "result/manifest.json").read_bytes()).entries: - if name.endswith((".deb", ".rpm")): - require(not (two / "result/files" / name).exists(), "Second generation copied historical payloads") - # A final release may first be selected for beta. Both channels share the - # apt pool, and the source-digest cache must reuse RPM signatures. - beta = work / "beta-check" - beta.mkdir() - source = json.loads((two / "input.json").read_bytes()) - source["channel"] = "beta" - (beta / "input.json").write_bytes(canonical(source)) - shutil.copytree(two / "input", beta / "input", copy_function=os.link) - (beta / "base.json").write_bytes(snapshot.data) - (beta / "base").mkdir() - for path in (two / "result/files").rglob("*"): - if path.is_file() and not path.name.endswith((".deb", ".rpm")): - target = beta / "base" / path.relative_to(two / "result/files") - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(path, target) - (beta / "reuse").mkdir() - for name, asset in source["assets"].items(): - if asset.get("format") == "rpm": - shutil.copyfile(two / f"result/files/yum/stable/vector-0/{asset['arch']}" / name, beta / "reuse" / name) - os.environ.update(PACKAGE_SIGNING_SUBKEY=key, PACKAGE_SIGNING_PASSPHRASE=password) - prepare(config, beta) - promoted = Snapshot.decode((beta / "result/manifest.json").read_bytes()) - for name, asset in source["assets"].items(): - if asset.get("format") == "rpm": - relative = f"vector-0/{asset['arch']}/{name}" - require(promoted.entries["yum/beta/" + relative] == snapshot.entries["yum/stable/" + relative], - "Selecting another channel changed RPM signatures") - shutil.rmtree(beta) - from tests.rollback_fixture import prepare_fixture as prepare_rollback_fixture - prepare_rollback_fixture(config, work, key, password) - from tests.installer_fixture import installer_fixture - installer_fixture(config, work, gpg, password) - config.pop("public_key_path") - config.pop("role", None) - (directory / "staging.json").write_bytes(canonical(config)) - (work / "fixture-report.json").write_bytes(canonical({"first": first, "second": second, - "primary_fingerprint": primary, "signing_fingerprint": subkey, - "checks": ["primary secret rejected", "encrypted subkey signing", "SHA-256 RPM signatures", - "apt/dnf metadata signatures", "all six architecture packages", "incremental retained history", - "historical payloads not copied", "channel selection reuses shared Debian bytes and RPM signatures", - "fresh rollback and recovery roots", "pending root date covered", "corrupt source rejected before key import"]})) - print("PASS: incremental metadata retains both versions without copying old packages", flush=True) - print("PASS: explicit beta selection reuses the existing package bytes and RPM signatures", flush=True) - finally: - os.environ.pop("PACKAGE_SIGNING_SUBKEY", None) - os.environ.pop("PACKAGE_SIGNING_PASSPHRASE", None) - run(["gpgconf", "--kill", "all"], env=env) - shutil.rmtree(home) diff --git a/distribution/repositories/tests/publication_fixture.py b/distribution/repositories/tests/publication_fixture.py deleted file mode 100644 index a8e122989eddf..0000000000000 --- a/distribution/repositories/tests/publication_fixture.py +++ /dev/null @@ -1,188 +0,0 @@ -"""Opt-in private S3 integration test; exports verified views for native clients.""" - -import base64 -import os -from pathlib import Path -import shutil -import tarfile -from unittest.mock import patch - -import engine -import rollback -from inputs import require -from storage import Conflict, Interrupted, Publisher, Snapshot, canonical, mutable - - -def publish_fixture(config, work): - require(config["account"] == "092620891642" and config["bucket"] == "pront-ci-sandbox-092620891642-us-east-1", - "S3 fixture is restricted to the private COSE sandbox") - require(config["live_prefix"].startswith("repository/publications/vdev-") - and config["control_key"].startswith("repository/control/vdev-"), "Invalid integration namespace") - store = engine.store_for(config) - publisher = Publisher(store, config) - state, _ = publisher.load_state() - require(state["active"] is None and not state["held"], "Fixture namespace is not empty; choose a new run ID") - one, two = work / "one", work / "two" - first = Snapshot.local(one / "result/files") - second = Snapshot.local(two / "result/files") - shared = work / "verified-objects" - shared.mkdir() - views = work / "views" - reports = [] - - def capture(name): - view = views / name - view.mkdir(parents=True) - # Objects not uploaded yet are absent from the initial view. Every object - # included in a client bundle is fetched from S3 and checked independently. - for relative in second.entries: - key = publisher.live + relative - head = store.head(key) - if head is None: - continue - digest = base64.b64decode(head["ChecksumSHA256"]).hex() - content = shared / digest - if not content.exists(): - engine.checked_download(store, key, content, - {"sha256": digest, "size": head["ContentLength"]}) - target = view / relative - target.parent.mkdir(parents=True, exist_ok=True) - os.link(content, target) - - def live_versions(snapshot): - return {name: store.head(publisher.live + name)["VersionId"] for name in snapshot.entries} - - engine.publish(config, one) - capture("initial") - reports.append("first six-package release published and downloaded from S3") - before = live_versions(first) - retry = work / "retry" - retry.mkdir() - shutil.copyfile(one / "input.json", retry / "input.json") - shutil.copytree(one / "input", retry / "input", copy_function=os.link) - engine.base(config, retry) - require(Snapshot.local(retry / "result/files").id == first.id, "Retry changed the signed snapshot") - engine.publish(config, retry) - require(live_versions(first) == before, "Retry replaced a live object") - reports.append("S3 bundle restored without a private key; retry created no live object versions") - - def interrupt_at(phase, view): - def factory(store, configuration): - def hook(actual): - if actual == phase: - capture(view) - raise Interrupted("intentional fixture interruption") - return Publisher(store, configuration, hook) - with patch.object(engine, "Publisher", factory): - try: - engine.publish(config, two) - except Interrupted: - return - raise AssertionError("Expected publication interruption was not reached") - - interrupt_at("payloads", "payloads") - interrupt_at("written:yum/stable/vector-0/x86_64/repodata/repomd.xml", "torn") - engine.publish(config, two) - capture("current") - reports.append("incremental publication resumed after payload and RPM root/signature interruptions") - # This historical view reproduces apt's rejection of an older Release date. - shutil.copytree(views / "initial", views / "stale-rollback", copy_function=os.link) - - def exercise_rollback(configuration, fixture, target, label, interrupt=False): - request = work / label - request.mkdir() - rollback.base(configuration, request, target, store) - require((request / "input.json").read_bytes() == (fixture / "input.json").read_bytes(), - "S3 preparation differs from the independently signed recovery request") - require(not (request / "result").exists(), "Unexpected rollback cache hit") - shutil.copytree(fixture / "result", request / "result", copy_function=os.link) - signed = Snapshot.local(request / "result/files") - if interrupt: - def factory(store, configuration): - def hook(phase): - if phase == "written:apt/dists/stable/InRelease": - raise Interrupted("intentional rollback interruption") - return Publisher(store, configuration, hook) - with patch.object(rollback, "Publisher", factory): - try: - rollback.publish(configuration, request, store) - except Interrupted: - pass - else: - raise AssertionError("Expected rollback interruption was not reached") - resumed = work / (label + "-resume") - resumed.mkdir() - rollback.base(configuration, resumed, target, store) - require(Snapshot.local(resumed / "result/files").id == signed.id, "Resume changed the signed rollback") - rollback.publish(configuration, resumed, store) - else: - rollback.publish(configuration, request, store) - live = Publisher(store, configuration).live - before = {name: store.head(live + name)["VersionId"] for name in signed.entries} - retry = work / (label + "-retry") - retry.mkdir() - rollback.base(configuration, retry, target, store) - require(Snapshot.local(retry / "result/files").id == signed.id, "Completed retry changed the signed rollback") - rollback.publish(configuration, retry, store) - require(before == {name: store.head(live + name)["VersionId"] for name in signed.entries}, - "Rollback retry replaced a live object") - return signed - - restored = exercise_rollback(config, work / "rollback", first.id, "restore", interrupt=True) - capture("rollback") - recovered = exercise_rollback(config, work / "recovered", second.id, "forward") - capture("recovered") - reports.append("fresh signed rollback and forward recovery preserve immutable history and advance apt dates") - reports.append("interrupted and completed rollback retries restore exact signed bytes without a key or extra live versions") - - pending_config = dict(config) - for field in ("live_prefix", "archive_prefix", "bundle_prefix"): - pending_config[field] += "pending-test/" - pending_config["control_key"] = config["control_key"].removesuffix(".json") + "-pending.json" - engine.publish(pending_config, one) - def pending_factory(store, configuration): - def hook(phase): - if phase == "pending": - raise Interrupted("stopped before any new payload or index upload") - return Publisher(store, configuration, hook) - with patch.object(engine, "Publisher", pending_factory): - try: - engine.publish(pending_config, two) - except Interrupted: - pass - else: - raise AssertionError("Expected early publication interruption was not reached") - exercise_rollback(pending_config, work / "rollback-pending", first.id, "abandon-pending") - reports.append("pending publication abandoned safely even when its immutable RPM indexes were never uploaded") - before = live_versions(recovered) - try: - publisher.publish(first) - except Conflict: - pass - else: - raise AssertionError("An old preparation replaced newer repository metadata") - require(live_versions(recovered) == before, "Stale input modified live data") - reports.append("stale prepared generation rejected before any live object changed") - public = work / "public" - public.mkdir() - shutil.copyfile(Path(config["public_key_path"]), public / "cose.asc") - shutil.copyfile(Path(config["public_key_path"]).with_suffix(".gpg"), public / "cose.gpg") - output = work / "client-bundles" - output.mkdir() - for deb_arch, rpm_arch in (("amd64", "x86_64"), ("arm64", "aarch64")): - with tarfile.open(output / (deb_arch + ".tar"), "w") as archive: - for path in sorted(public.iterdir()): - archive.add(path, arcname="public/" + path.name) - for path in sorted(views.rglob("*")): - if not path.is_file(): - continue - if path.name.endswith(".deb") and not path.name.endswith("_" + deb_arch + ".deb"): - continue - if path.name.endswith(".rpm") and not path.name.endswith("." + rpm_arch + ".rpm"): - continue - archive.add(path, arcname="views/" + path.relative_to(views).as_posix()) - report = {"first": first.id, "second": second.id, "rollback": restored.id, "recovered": recovered.id, "checks": reports, - "state": publisher.load_state()[0]} - (work / "publication-report.json").write_bytes(canonical(report)) - for item in reports: - print("PASS: " + item, flush=True) diff --git a/distribution/repositories/tests/rollback_fixture.py b/distribution/repositories/tests/rollback_fixture.py deleted file mode 100644 index 73dafaaaf3d25..0000000000000 --- a/distribution/repositories/tests/rollback_fixture.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Pre-sign recovery cases without exposing a private key to S3 publication.""" - -import os -from pathlib import Path -import shutil -from unittest.mock import patch - -from inputs import require -from repository import repomd_locations -import rollback -from storage import PublicationError, Snapshot, mutable - - -def seed(work, target, active, pending=None): - work.mkdir() - data = rollback.plan_bytes(target.id, active.id, pending.id if pending else None) - (work / "input.json").write_bytes(data) - for snapshot in {item.id: item for item in (target, active, pending) if item}.values(): - directory = work / "sources" / snapshot.id - root = directory / "files" - root.mkdir(parents=True) - (directory / "manifest.json").write_bytes(snapshot.data) - names = {name for name in snapshot.entries if mutable(name)} - if snapshot.id in {target.id, active.id}: - for package_root in snapshot.source.glob("yum/*/vector-0/*"): - names.update((package_root / name).relative_to(snapshot.source).as_posix() - for name in repomd_locations(package_root)) - for name in names: - path = root / name - path.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(snapshot.source / name, path) - - -def prepare_fixture(config, work, key, password): - first = Snapshot.local(work / "one/result/files") - second = Snapshot.local(work / "two/result/files") - back = work / "rollback" - seed(back, first, second) - os.environ.update(PACKAGE_SIGNING_SUBKEY=key, PACKAGE_SIGNING_PASSPHRASE=password) - rollback.prepare(config, back) - restored = Snapshot.local(back / "result/files") - forward = work / "recovered" - seed(forward, second, restored) - os.environ.update(PACKAGE_SIGNING_SUBKEY=key, PACKAGE_SIGNING_PASSPHRASE=password) - rollback.prepare(config, forward) - interrupted = work / "rollback-pending" - seed(interrupted, first, first, second) - os.environ.update(PACKAGE_SIGNING_SUBKEY=key, PACKAGE_SIGNING_PASSPHRASE=password) - rollback.prepare(config, interrupted) - for name, entry in second.entries.items(): - if not mutable(name): - require(restored.entries[name] == entry, "Rollback removed immutable history") - require(not any(path.name.endswith((".deb", ".rpm")) for path in back.rglob("*")), - "Rollback copied package payloads") - - # A corrupt downloaded source must fail before importing the private key. - broken = work / "rollback-corrupt" - seed(broken, first, second) - path = broken / "sources" / first.id / "files/apt/dists/stable/Release" - path.write_bytes(path.read_bytes() + b"Corruption\n") - with patch.object(rollback, "Signer") as signer: - try: - rollback.prepare(config, broken) - except PublicationError: - pass - else: - raise AssertionError("Corrupt rollback source was accepted") - signer.assert_not_called() - shutil.rmtree(broken) - - # A valid signed bundle cannot be relabeled as another selection. - original = (back / "input.json").read_bytes() - (back / "input.json").write_bytes(rollback.plan_bytes(second.id, second.id, None)) - try: - rollback.check_result(config, back) - except PublicationError: - pass - else: - raise AssertionError("Rollback accepted a signed bundle for another selection") - (back / "input.json").write_bytes(original) - print("PASS: fresh signed rollback preserves package bytes, validates sources before key import, and binds the selected archive", flush=True) diff --git a/distribution/repositories/tests/test_inputs.py b/distribution/repositories/tests/test_inputs.py deleted file mode 100644 index e8ad967ecf5ed..0000000000000 --- a/distribution/repositories/tests/test_inputs.py +++ /dev/null @@ -1,106 +0,0 @@ -import copy -import io -import json -import sys -import tarfile -import tempfile -import unittest -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from engine import unpack_bundle -from inputs import expected_packages, parse_checksums, select_assets, validate_run, versions -from repository import apt_indexes, merge_packages -from storage import PublicationError - - -class InputTests(unittest.TestCase): - def test_stable_and_prerelease_native_versions(self): - self.assertEqual(versions("v0.61.0", "stable"), ("0.61.0", "0.61.0-1", "0.61.0-1")) - self.assertEqual(versions("v0.61.0-rc.1-test", "beta"), - ("0.61.0-rc.1-test", "0.61.0~rc.1-test-1", "0.61.0~rc.1.test-1")) - for tag in ("v0.61.0-rc.1", "v0.61.0+build", "v0.61.0;sh", "main", "v1.0.0", "v0.061.0"): - with self.subTest(tag=tag), self.assertRaises(PublicationError): - versions(tag, "stable") - - def test_release_admission_rejects_fork_wrong_workflow_ref_commit_and_failure(self): - config = {"repository_id": 146327667, "workflow": ".github/workflows/release.yml"} - run = {"repository": {"id": 146327667}, "head_repository": {"id": 146327667}, - "path": config["workflow"], "event": "push", "head_branch": "v0.60.0", - "head_sha": "a" * 40, "status": "completed", "conclusion": "success"} - validate_run(config, run, "v0.60.0", "a" * 40) - for key, value in (("head_repository", {"id": 7}), ("repository", {"id": 7}), - ("path", ".github/workflows/nightly.yml"), ("event", "pull_request"), - ("head_branch", "master"), ("head_sha", "b" * 40), ("conclusion", "failure")): - with self.subTest(field=key), self.assertRaises(PublicationError): - validate_run(config, {**run, key: value}, "v0.60.0", "a" * 40) - - def test_asset_selection_rejects_missing_duplicate_unexpected_and_unhashed(self): - names = list(expected_packages("v0.60.0", "stable")) + ["vector-0.60.0-SHA256SUMS"] - release = {"tag_name": "v0.60.0", "draft": False, "prerelease": False, "published_at": "today", - "assets": [{"id": i + 1, "name": name, "size": 10, "digest": "sha256:" + "a" * 64} - for i, name in enumerate(names)]} - self.assertEqual(len(select_assets(release, "v0.60.0", "stable")), 7) - cases = [] - missing = copy.deepcopy(release); missing["assets"].pop(); cases.append(missing) - duplicate = copy.deepcopy(release); duplicate["assets"].append(duplicate["assets"][0]); cases.append(duplicate) - unknown = copy.deepcopy(release); unknown["assets"][0]["name"] = "vector-unexpected.rpm"; cases.append(unknown) - unhashed = copy.deepcopy(release); unhashed["assets"][0]["digest"] = None; cases.append(unhashed) - for case in cases: - with self.assertRaises(PublicationError): - select_assets(case, "v0.60.0", "stable") - - def test_checksums_reject_duplicate_and_traversal(self): - line = "a" * 64 + " vector.deb\n" - self.assertEqual(parse_checksums(line.encode()), {"vector.deb": "a" * 64}) - for data in (line + line, "a" * 64 + " ../vector.deb\n", "invalid"): - with self.assertRaises(PublicationError): - parse_checksums(data.encode()) - - def test_incremental_apt_merge_keeps_historical_native_versions(self): - old = b"Package: vector\nVersion: 0.10.0\nArchitecture: amd64\nFilename: pool/v/ve/vector_0.10.0-1_amd64.deb\n\n" - new = b"Package: vector\nVersion: 0.60.0-1\nArchitecture: amd64\nFilename: pool/v/ve/vector_0.60.0-1_amd64.deb\n\n" - self.assertEqual(merge_packages(old, new), old + new) - self.assertEqual(merge_packages(old, old), old) - with self.assertRaises(PublicationError): - merge_packages(old, old.replace(b"pool/v/ve", b"pool/replaced")) - - def test_apt_identity_matches_observed_legacy_repository(self): - # Identity fields are cached by apt and changes require explicit user - # approval. Compare with observed input rather than the desired output. - observed = json.loads((Path(__file__).parent / "legacy-apt-identity.json").read_bytes()) - class UnsignedFixture: - def sign(self, source, **kwargs): - pass - def verify(self, *args): - pass - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - apt_indexes(root, "stable", {}, root / "previous", UnsignedFixture()) - release = {key: value.strip() for line in - (root / "apt/dists/stable/Release").read_text().splitlines() - if ":" in line and not line.startswith(" ") - for key, value in [line.split(":", 1)]} - for field in ("Origin", "Label", "Suite", "Codename"): - self.assertEqual(release.get(field), observed[field], field) - - def test_signed_bundle_rejects_links_traversal_and_duplicates(self): - for names, linked in ((["files/apt/../escape"], False), (["files/apt/pool/x"], True), - (["input.json", "input.json"], False)): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - archive = root / "bundle.tar.gz" - with tarfile.open(archive, "w:gz") as target: - for name in names: - member = tarfile.TarInfo(name) - member.size = 1 - if linked: - member.type = tarfile.SYMTYPE - member.linkname = "/etc/passwd" - target.addfile(member, io.BytesIO(b"x")) - with self.assertRaises(PublicationError): - unpack_bundle(archive, root / "output") - - -if __name__ == "__main__": - unittest.main() diff --git a/distribution/repositories/tests/test_rollback.py b/distribution/repositories/tests/test_rollback.py deleted file mode 100644 index 5a16338bb99de..0000000000000 --- a/distribution/repositories/tests/test_rollback.py +++ /dev/null @@ -1,55 +0,0 @@ -from datetime import datetime, timedelta, timezone -from email.utils import format_datetime -from pathlib import Path -import sys -import unittest - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from repository import dated_release, next_release_date, release_date -from rollback import decode_plan, plan_bytes -from storage import PublicationError - - -class RollbackDateTests(unittest.TestCase): - def setUp(self): - self.now = datetime(2026, 10, 20, 12, tzinfo=timezone.utc) - - def root(self, date): - return ("Suite:\nCodename: stable\nDate: " + format_datetime(date) - + "\nSHA256:\n abc 3 vector-0/binary-amd64/Packages\n").encode() - - def test_new_date_supersedes_cached_and_interrupted_roots_without_changing_identity(self): - old = self.root(self.now - timedelta(days=30)) - cached = self.root(self.now) - pending = self.root(self.now + timedelta(seconds=1)) - refreshed = dated_release(old, next_release_date([old, cached, pending], self.now)) - self.assertGreater(release_date(refreshed), release_date(pending)) - self.assertEqual(dated_release(refreshed, "DATE"), dated_release(old, "DATE")) - - def test_normal_release_immediately_after_rollback_also_advances(self): - rollback = self.root(self.now + timedelta(seconds=1)) - later = self.root(self.now + timedelta(seconds=2)) - self.assertEqual(next_release_date([rollback], self.now), format_datetime(release_date(later))) - - def test_future_clock_drift_is_rejected_instead_of_disabling_apt_checks(self): - with self.assertRaises(PublicationError): - next_release_date([self.root(self.now + timedelta(minutes=1))], self.now) - - def test_ambiguous_date_or_expiry_policy_is_rejected(self): - root = self.root(self.now) - for extra in (b"Date: Tue, 20 Oct 2026 12:00:01 +0000\n", b"Valid-Until: Wed, 21 Oct 2026 12:00:00 +0000\n"): - with self.assertRaises(PublicationError): - dated_release(root + extra, next_release_date([], self.now)) - - def test_pending_publication_can_only_be_abandoned_to_last_completed_selection(self): - active, pending, other = "a" * 64, "b" * 64, "c" * 64 - plan, _ = decode_plan(plan_bytes(active, active, pending)) - self.assertEqual(plan["pending"], pending) - with self.assertRaises(PublicationError): - plan_bytes(other, active, pending) - with self.assertRaises(PublicationError): - plan_bytes("../escape", active, None) - - -if __name__ == "__main__": - unittest.main() diff --git a/distribution/repositories/tests/test_storage.py b/distribution/repositories/tests/test_storage.py deleted file mode 100644 index 46fc7f9521209..0000000000000 --- a/distribution/repositories/tests/test_storage.py +++ /dev/null @@ -1,275 +0,0 @@ -import base64 -from concurrent.futures import ThreadPoolExecutor -from datetime import datetime, timedelta, timezone -import hashlib -import json -from pathlib import Path -import sys -import tempfile -import threading -import unittest -from unittest.mock import Mock - -from botocore.exceptions import ClientError - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] )) -from storage import (Busy, Conflict, Interrupted, PublicationError, Publisher, - S3Store, Snapshot, byte_entry, canonical) - - -CONFIG = {"live_prefix": "repository/publications/test/live/", - "archive_prefix": "repository/publications/test/snapshots/", - "control_key": "repository/control/test.json"} - - -class MemoryStore: - def __init__(self): - self.values = {} - self.owner = "test-fixed-session" - self.server_time = datetime(2026, 10, 19, tzinfo=timezone.utc) - self.mutex = threading.Lock() - self.serial = 0 - self.lose_response = None - self.barrier = None - - def head(self, key): - with self.mutex: - item = self.values.get(key) - return dict(item[1]) if item else None - - def read(self, key): - with self.mutex: - data, head = self.values[key] - result = data, dict(head) - if self.barrier and key.startswith("repository/control/"): - self.barrier.wait(timeout=5) - return result - - def put(self, key, body, entry, *, previous=None, immutable=False): - data = body if isinstance(body, bytes) else body.read() - assert byte_entry(data) == entry - with self.mutex: - old = self.values.get(key) - if (old and previous is None) or (previous and ( - not old or old[1]["ETag"] != previous["ETag"] - )): - raise ClientError({"Error": {"Code": "PreconditionFailed"}, - "ResponseMetadata": {"HTTPStatusCode": 412}}, "PutObject") - self.serial += 1 - head = {"ETag": '"' + hashlib.md5(data, usedforsecurity=False).hexdigest() + '"', - "VersionId": str(self.serial), "ContentLength": len(data), - "ChecksumSHA256": base64.b64encode(hashlib.sha256(data).digest()).decode()} - self.values[key] = data, head - if key == self.lose_response: - self.lose_response = None - raise OSError("simulated lost response after successful write") - return dict(head) - - -class PublisherTests(unittest.TestCase): - def setUp(self): - self.temp = tempfile.TemporaryDirectory() - self.addCleanup(self.temp.cleanup) - self.root = Path(self.temp.name) - self.store = MemoryStore() - self.one = self.snapshot("one", "one") - self.two = self.snapshot("two", "two") - self.publisher = Publisher(self.store, CONFIG) - - def snapshot(self, directory, generation, payload=b"same signed package"): - directory = self.root / directory - root = directory / "files" - files = { - "apt/pool/vector.deb": payload, - "apt/dists/stable/InRelease": generation.encode(), - "apt/dists/stable/vector-0/binary-amd64/Packages": generation.encode(), - "yum/stable/vector-0/x86_64/repodata/repomd.xml": generation.encode(), - "yum/stable/vector-0/x86_64/repodata/repomd.xml.asc": (generation + " signature").encode(), - } - for name, data in files.items(): - path = root / name - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(data) - base = self.one.id if hasattr(self, "one") else None - snapshot = Snapshot({k: byte_entry(v) for k, v in files.items()}, base=base) - (directory / "manifest.json").write_bytes(snapshot.data) - return Snapshot.local(root) - - def live(self): - return {key: value for key, value in self.store.values.items() if "/live/" in key} - - def test_retry_does_not_create_new_live_versions(self): - self.publisher.publish(self.one) - before = self.live() - self.publisher.publish(self.one) - self.assertEqual(self.live(), before) - - def test_incremental_snapshot_reuses_inherited_payload_without_a_local_copy(self): - self.publisher.publish(self.one) - (self.two.source / "apt/pool/vector.deb").unlink() - snapshot = Snapshot.local(self.two.source) - self.publisher.publish(snapshot) - self.assertEqual(self.publisher.load_state()[0]["active"], snapshot.id) - - def test_stale_preparation_and_history_removal_fail_before_live_writes(self): - self.publisher.publish(self.one) - stale = self.snapshot("stale", "stale") - self.publisher.publish(self.two) - before = self.live() - with self.assertRaises(Conflict): - self.publisher.publish(stale) - entries = {k: v for k, v in self.two.entries.items() if not k.endswith(".deb")} - removal = Snapshot(entries, self.two.source, base=self.two.id) - with self.assertRaises(Conflict): - self.publisher.publish(removal) - self.assertEqual(before, self.live()) - - def test_missing_inherited_payload_fails_before_roots_change(self): - self.publisher.publish(self.one) - del self.store.values[self.publisher.live + "apt/pool/vector.deb"] - (self.two.source / "apt/pool/vector.deb").unlink() - before = self.live() - with self.assertRaises(Conflict): - self.publisher.publish(self.two) - self.assertEqual(before, self.live()) - - def test_conflict_preflight_leaves_metadata_unchanged(self): - self.publisher.publish(self.one) - before = self.live() - conflict = self.snapshot("conflict", "two", b"different package at same URL") - with self.assertRaises(Conflict): - self.publisher.publish(conflict) - self.assertEqual(before, self.live()) - self.assertFalse(self.publisher.load_state()[0]["held"]) - - def test_racing_lock_acquisitions_have_exactly_one_winner(self): - self.publisher.acquire() - self.publisher.release() - self.store.barrier = threading.Barrier(2) - - def acquire(_): - try: - Publisher(self.store, CONFIG).acquire() - return "winner" - except Busy: - return "busy" - - with ThreadPoolExecutor(max_workers=2) as executor: - result = list(executor.map(acquire, range(2))) - self.assertCountEqual(result, ["winner", "busy"]) - - def test_interrupted_publish_can_only_resume_or_roll_back(self): - self.publisher.publish(self.one) - - def stop(phase): - if phase.endswith("repomd.xml"): - raise Interrupted("torn pair") - - with self.assertRaises(Interrupted): - Publisher(self.store, CONFIG, stop).publish(self.two) - state, _ = self.publisher.load_state() - self.assertEqual(state["active"], self.one.id) - self.assertEqual(state["pending"], self.two.id) - with self.assertRaises(Conflict): - self.publisher.publish(self.snapshot("three", "three")) - recovery = self.snapshot("recovery", "fresh signed old selection") - self.publisher.publish(recovery, recovery=(self.one.id, self.two.id)) - self.assertEqual(self.publisher.load_state()[0]["active"], recovery.id) - self.assertIsNone(self.publisher.load_state()[0]["pending"]) - before = self.live() - self.publisher.publish(recovery, recovery=(self.one.id, self.two.id)) - self.assertEqual(self.live(), before) - with self.assertRaises(Conflict): - self.publisher.publish(self.two) - - def test_recovery_rejects_changed_active_or_pending_state_before_live_writes(self): - self.publisher.publish(self.one) - recovery = self.snapshot("recovery", "fresh signed old selection") - before = self.live() - with self.assertRaises(Conflict): - self.publisher.publish(recovery, recovery=(self.one.id, self.two.id)) - self.assertEqual(self.live(), before) - self.publisher.publish(self.two) - before = self.live() - with self.assertRaises(Conflict): - self.publisher.publish(recovery, recovery=(self.one.id, None)) - self.assertEqual(self.live(), before) - - def test_recovery_can_resume_its_own_interrupted_generation(self): - self.publisher.publish(self.one) - recovery = self.snapshot("recovery", "fresh signed old selection") - def stop(phase): - if phase.endswith("repomd.xml"): - raise Interrupted("torn recovery pair") - with self.assertRaises(Interrupted): - Publisher(self.store, CONFIG, stop).publish(recovery, recovery=(self.one.id, None)) - self.assertEqual(self.publisher.load_state()[0]["pending"], recovery.id) - self.publisher.publish(recovery, recovery=(self.one.id, None)) - self.assertEqual(self.publisher.load_state()[0]["active"], recovery.id) - - def test_lost_write_response_retains_lock_and_blocks_early_recovery(self): - self.publisher.publish(self.one) - self.store.lose_response = self.publisher.live + "apt/dists/stable/InRelease" - with self.assertRaises(OSError): - self.publisher.publish(self.two) - _, head = self.publisher.load_state() - with self.assertRaises(Busy): - Publisher(self.store, CONFIG).publish(self.two) - with self.assertRaises(Busy): - Publisher(self.store, CONFIG).recover_lock(head["VersionId"], True) - self.store.server_time += timedelta(minutes=66) - with self.assertRaises(PublicationError): - self.publisher.recover_lock(head["VersionId"], False) - self.publisher.recover_lock(head["VersionId"], True) - Publisher(self.store, CONFIG).publish(self.two) - with self.assertRaises(Conflict): - self.publisher.recover_lock(head["VersionId"], True) - - def test_corrupt_archived_metadata_fails_before_any_live_change(self): - self.publisher.publish(self.one) - self.publisher.publish(self.two) - before = self.live() - key = self.publisher.archives + self.two.id + "/files/apt/dists/stable/InRelease" - self.store.put(key, b"bad", byte_entry(b"bad"), previous=self.store.head(key)) - with self.assertRaises(Conflict): - self.publisher.publish(self.two) - self.assertEqual(before, self.live()) - - def test_manifest_rejects_extra_file_changed_bytes_symlink_and_traversal(self): - extra = self.one.source / "apt/pool/extra.deb" - extra.write_bytes(b"unexpected") - with self.assertRaises(PublicationError): - Snapshot.local(self.one.source) - extra.unlink() - package = self.one.source / "apt/pool/vector.deb" - package.write_bytes(b"changed") - with self.assertRaises(PublicationError): - Snapshot.local(self.one.source) - for name in ("../escape", "/apt/root", "apt/a/../root", "apt//root"): - with self.assertRaises(PublicationError): - Snapshot({name: byte_entry(b"test")}) - linked = self.two.source / "apt/pool/linked.deb" - linked.symlink_to(self.two.source / "apt/pool/vector.deb") - with self.assertRaises(PublicationError): - Snapshot.local(self.two.source) - - def test_head_403_requires_authorized_listing_to_prove_absence(self): - store = S3Store.__new__(S3Store) - store.bucket = "test" - store.client = Mock() - store.response_time = Mock() - denial = ClientError({"Error": {"Code": "AccessDenied"}, - "ResponseMetadata": {"HTTPStatusCode": 403}}, "HeadObject") - store.client.head_object.side_effect = denial - store.client.list_objects_v2.return_value = {"Contents": []} - self.assertIsNone(store.head("repository/control/test.json")) - store.client.list_objects_v2.return_value = {"Contents": [{"Key": "repository/control/test.json"}]} - with self.assertRaises(ClientError): - store.head("repository/control/test.json") - store.client.list_objects_v2.side_effect = denial - with self.assertRaises(ClientError): - store.head("repository/control/test.json") - - -if __name__ == "__main__": - unittest.main() diff --git a/docs/reference/package-repositories.md b/docs/reference/package-repositories.md index e5c9a8179a968..27f711f8ecb8d 100644 --- a/docs/reference/package-repositories.md +++ b/docs/reference/package-repositories.md @@ -9,9 +9,9 @@ This workflow is disabled unless the repository variable `PACKAGE_REPOSITORIES_ENABLED` is `true`. It requires a completed historical backfill. Enabling it is part of the repository migration cutover, after the new public key, installation instructions and client migration have been published. -The normal Test Suite runs the offline publisher regressions when this tooling -changes. Those checks require no signing secrets or AWS credentials and remain -available while publication is disabled. +The normal Test Suite checks the vdev CLI and builds the publisher image when +this tooling changes. Those checks require no signing secrets or AWS credentials. +Publisher regressions and migration rehearsals live in the private CI sandbox. ## Inputs and job boundaries @@ -133,8 +133,8 @@ channel on a repeated invocation. `CSM_MIGRATE=true` disables the known The publisher keeps the existing apt identity: no `Origin` or `Label`, an empty `Suite`, and the channel as `Codename`. This permits the signing-key and hosting transition without a separate release-information approval. The observed -legacy fields are recorded in `tests/legacy-apt-identity.json`; a regression -check compares the publisher's output with that observation. +legacy fields and the regression check comparing them with publisher output +are kept in the private migration test suite. Before announcement, deploy the reviewed script at `vector/setup.sh` and the public key at `vector/keys/vector-packages.asc` @@ -146,9 +146,9 @@ new script and update those installation pages together. The protected production client job runs this setup script before installing a release. Activation therefore requires the public key endpoint to be available. -The real-package fixture also creates separate ephemeral legacy and COSE signing -identities and installer bundles for disposable HTTPS client tests. Private keys -stay in tmpfs; client bundles contain only public keys and signed packages. +Private sandbox tests use separate ephemeral legacy and COSE signing identities +for disposable HTTPS clients. Private keys stay in tmpfs; client bundles contain +only public keys and signed packages. ## Maintainer commands @@ -156,7 +156,6 @@ Build the tools once from reviewed source: ```sh cargo vdev release repositories build-image -cargo vdev release repositories test ``` The default workspace is `target/package-repositories`. Use a different `--work` diff --git a/vdev/src/commands/release/repositories.rs b/vdev/src/commands/release/repositories.rs index 32424cf8bfff0..7e62270252670 100644 --- a/vdev/src/commands/release/repositories.rs +++ b/vdev/src/commands/release/repositories.rs @@ -60,15 +60,6 @@ enum Commands { #[arg(long)] confirm_owner_stopped: bool, }, - /// Run the publisher's offline protocol and metadata regressions. - Test { - /// Prepare two releases in the workspace with an ephemeral test key. - #[arg(long, conflicts_with = "s3")] - packages: bool, - /// Exercise prepared fixtures against the isolated private S3 sandbox. - #[arg(long)] - s3: bool, - }, } #[derive(clap::Subcommand, Debug)] @@ -173,8 +164,7 @@ impl Commands { | Self::Rollback { phase: Rollback::Base { .. } | Rollback::Publish, } - | Self::RecoverLock { .. } - | Self::Test { s3: true, .. } => &[ + | Self::RecoverLock { .. } => &[ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", @@ -241,15 +231,6 @@ impl Commands { command.arg("--confirm-owner-stopped"); } } - Commands::Test { packages, s3 } => { - command.arg("test"); - if packages { - command.arg("--packages"); - } - if s3 { - command.arg("--s3"); - } - } Commands::BuildImage => unreachable!(), } } @@ -292,13 +273,5 @@ mod tests { .is_empty() ); assert!(Commands::Verify.credentials().is_empty()); - assert!( - Commands::Test { - packages: true, - s3: false - } - .credentials() - .is_empty() - ); } } From d501fb6965ce2892a0bd38382eefba5291d585ad Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 20 Oct 2026 15:28:11 -0400 Subject: [PATCH 06/23] docs(packaging): use the migration announcement for release notes --- changelog.d/remove_datadog_key_recommendation.enhancement.md | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 changelog.d/remove_datadog_key_recommendation.enhancement.md diff --git a/changelog.d/remove_datadog_key_recommendation.enhancement.md b/changelog.d/remove_datadog_key_recommendation.enhancement.md deleted file mode 100644 index 496ec44875082..0000000000000 --- a/changelog.d/remove_datadog_key_recommendation.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -Debian packages no longer recommend installing the `datadog-signing-keys` package. - -authors: pront From 4a1efdc627798629235eeecba6369312ee035424 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 20 Oct 2026 16:33:30 -0400 Subject: [PATCH 07/23] refactor(releasing): implement package repository publishing in Rust --- .github/workflows/test.yml | 2 +- Cargo.lock | 127 ++- LICENSE-3rdparty.csv | 2 + distribution/repositories/.dockerignore | 6 - distribution/repositories/Dockerfile | 21 +- distribution/repositories/bundles.py | 52 - distribution/repositories/engine.py | 234 ---- distribution/repositories/inputs.py | 169 --- distribution/repositories/repository.py | 432 -------- distribution/repositories/requirements.txt | 8 - distribution/repositories/rollback.py | 236 ----- distribution/repositories/storage.py | 400 ------- docs/reference/package-repositories.md | 9 +- vdev/Cargo.toml | 22 + vdev/src/commands/release/repositories.rs | 24 +- .../release/repositories/publisher/bundles.rs | 95 ++ .../release/repositories/publisher/engine.rs | 291 +++++ .../release/repositories/publisher/inputs.rs | 319 ++++++ .../release/repositories/publisher/mod.rs | 76 ++ .../release/repositories/publisher/model.rs | 375 +++++++ .../repositories/publisher/repository.rs | 999 ++++++++++++++++++ .../repositories/publisher/rollback.rs | 335 ++++++ .../release/repositories/publisher/storage.rs | 684 ++++++++++++ 23 files changed, 3366 insertions(+), 1552 deletions(-) delete mode 100644 distribution/repositories/.dockerignore delete mode 100644 distribution/repositories/bundles.py delete mode 100644 distribution/repositories/engine.py delete mode 100644 distribution/repositories/inputs.py delete mode 100644 distribution/repositories/repository.py delete mode 100644 distribution/repositories/requirements.txt delete mode 100644 distribution/repositories/rollback.py delete mode 100644 distribution/repositories/storage.py create mode 100644 vdev/src/commands/release/repositories/publisher/bundles.rs create mode 100644 vdev/src/commands/release/repositories/publisher/engine.rs create mode 100644 vdev/src/commands/release/repositories/publisher/inputs.rs create mode 100644 vdev/src/commands/release/repositories/publisher/mod.rs create mode 100644 vdev/src/commands/release/repositories/publisher/model.rs create mode 100644 vdev/src/commands/release/repositories/publisher/repository.rs create mode 100644 vdev/src/commands/release/repositories/publisher/rollback.rs create mode 100644 vdev/src/commands/release/repositories/publisher/storage.rs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 22c13c1a21c83..f959cee482e0f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -72,7 +72,7 @@ jobs: check-package-repositories: name: Check package repository tooling runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 45 if: ${{ needs.changes.outputs.package_repositories == 'true' || needs.changes.outputs.test-yml == 'true' }} needs: changes steps: diff --git a/Cargo.lock b/Cargo.lock index 61392f3c4cbfa..3606092547f7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1279,6 +1279,29 @@ dependencies = [ "zeroize", ] +[[package]] +name = "aws-lc-rs" +version = "1.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + [[package]] name = "aws-runtime" version = "1.9.2" @@ -1416,6 +1439,33 @@ dependencies = [ "tracing 0.1.44", ] +[[package]] +name = "aws-sdk-iam" +version = "1.123.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabfa8ac26d6684cd0a021cf775bafb972dfa0dcc8ab151354de5e806d4f7d8a" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "http 1.5.0", + "regex-lite", + "tracing 0.1.44", +] + [[package]] name = "aws-sdk-kinesis" version = "1.119.0" @@ -1795,10 +1845,19 @@ dependencies = [ "h2 0.3.26", "h2 0.4.19", "http 0.2.12", + "http 1.5.0", "http-body 0.4.6", "hyper 0.14.32", + "hyper 1.11.1", + "hyper-rustls", + "hyper-util", "pin-project-lite", + "rustls 0.23.46", + "rustls-native-certs 0.8.1", + "rustls-pki-types", "tokio", + "tokio-rustls 0.26.2", + "tower 0.5.3", "tracing 0.1.44", ] @@ -2255,7 +2314,7 @@ dependencies = [ "bitflags 2.13.2", "cexpr", "clang-sys", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2 1.0.107", "quote 1.0.47", "regex", @@ -4191,6 +4250,12 @@ dependencies = [ "shared_child", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -4634,6 +4699,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -4780,6 +4855,12 @@ dependencies = [ "num", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -9249,7 +9330,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.10.5", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -9269,7 +9350,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck 0.5.0", - "itertools 0.10.5", + "itertools 0.14.0", "log", "multimap", "petgraph 0.8.3", @@ -9303,7 +9384,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2 1.0.107", "quote 1.0.47", "syn 2.0.119", @@ -9316,7 +9397,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2 1.0.107", "quote 1.0.47", "syn 2.0.119", @@ -10626,6 +10707,7 @@ version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -10732,6 +10814,7 @@ version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -12079,6 +12162,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tcp-stream" version = "0.34.2" @@ -13429,6 +13523,13 @@ name = "vdev" version = "0.3.32" dependencies = [ "anyhow", + "aws-config", + "aws-sdk-iam", + "aws-sdk-s3", + "aws-sdk-sts", + "aws-smithy-runtime-api", + "aws-smithy-types", + "base64 0.23.1", "cfg-if", "chrono", "clap", @@ -13436,6 +13537,7 @@ dependencies = [ "clap_complete", "convert_case 0.12.0", "directories", + "flate2", "git2", "glob", "handlebars", @@ -13450,6 +13552,8 @@ dependencies = [ "proc-macro2 1.0.107", "quote 1.0.47", "regex", + "reqwest 0.11.26", + "roxmltree", "semver", "serde", "serde_json", @@ -13457,9 +13561,12 @@ dependencies = [ "sha2 0.11.0", "strum 0.28.0", "syn 3.0.5", + "tar", "tempfile", + "tokio", "toml", "toml_edit 0.23.9", + "uuid", ] [[package]] @@ -15219,6 +15326,16 @@ dependencies = [ "tap", ] +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix 1.1.4", +] + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 0aa81c28a3037..ada7572e512ab 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -71,6 +71,8 @@ atoi,https://github.com/pacman82/atoi-rs,MIT,Markus Klein atomic-waker,https://github.com/smol-rs/atomic-waker,Apache-2.0 OR MIT,"Stjepan Glavina , Contributors to futures-rs" aws-config,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-credential-types,https://github.com/smithy-lang/smithy-rs,Apache-2.0,AWS Rust SDK Team +aws-lc-rs,https://github.com/aws/aws-lc-rs,ISC AND (Apache-2.0 OR ISC),AWS-LibCrypto +aws-lc-sys,https://github.com/aws/aws-lc-rs,ISC AND (Apache-2.0 OR ISC) AND Apache-2.0 AND MIT AND BSD-3-Clause AND (Apache-2.0 OR ISC OR MIT) AND (Apache-2.0 OR ISC OR MIT-0),AWS-LC aws-runtime,https://github.com/smithy-lang/smithy-rs,Apache-2.0,AWS Rust SDK Team aws-sdk-cloudwatch,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-cloudwatchlogs,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " diff --git a/distribution/repositories/.dockerignore b/distribution/repositories/.dockerignore deleted file mode 100644 index 840bcab19c537..0000000000000 --- a/distribution/repositories/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -* -!Dockerfile -!requirements.txt -!*.py - -**/__pycache__ diff --git a/distribution/repositories/Dockerfile b/distribution/repositories/Dockerfile index 051d103e7349a..012915d13ff72 100644 --- a/distribution/repositories/Dockerfile +++ b/distribution/repositories/Dockerfile @@ -1,10 +1,19 @@ +FROM rust:1.96.1-bookworm@sha256:a339861ae23e9abb272cea45dfafde21760d2ce6577a70f8a926153677902663 AS build +RUN apt-get update && apt-get install -y --no-install-recommends cmake libssl-dev pkg-config \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /src +COPY . . +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + cargo build --locked --package vdev --features package-repositories \ + && strip target/debug/vdev + FROM ubuntu:24.04@sha256:68ee22004a9109b44973313b02ff7b1de250f1de0f5ecb628d73f579aaff33ac RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - apt-utils ca-certificates createrepo-c dpkg-dev gnupg python3 python3-venv rpm \ + apt-utils ca-certificates createrepo-c dpkg-dev gnupg rpm libssl3t64 \ && rm -rf /var/lib/apt/lists/* -COPY requirements.txt /app/requirements.txt -RUN python3 -m venv /opt/publisher && /opt/publisher/bin/pip install \ - --no-cache-dir --require-hashes --only-binary=:all: -r /app/requirements.txt -COPY *.py /app/ +COPY --from=build /src/target/debug/vdev /usr/local/bin/vdev +# vdev normally locates its checkout before dispatching a command. +COPY --from=build /src/Cargo.toml /app/Cargo.toml WORKDIR /app -ENTRYPOINT ["/opt/publisher/bin/python", "/app/engine.py"] +ENTRYPOINT ["vdev", "release", "repositories"] diff --git a/distribution/repositories/bundles.py b/distribution/repositories/bundles.py deleted file mode 100644 index 75bef64170521..0000000000000 --- a/distribution/repositories/bundles.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Checksum-verified transport of signed publication workspaces.""" - -import gzip -import shutil -import tarfile - -from inputs import require -from storage import Snapshot, matches, safe_relative - - -def checked_download(store, key, path, entry=None): - path.parent.mkdir(parents=True, exist_ok=True) - head = store.head(key) - require(head is not None and head["ContentLength"] < 5 * 1024**3, "Missing or oversized S3 input") - actual = store.download(key, path) - require(matches(head, actual) and (entry is None or actual == entry), "Downloaded S3 object failed checksum verification") - return actual - - -def unpack_bundle(path, destination): - destination.mkdir() - total = 0 - names = set() - with tarfile.open(path, "r:gz") as archive: - for member in archive: - require(member.isfile() and member.name not in names, "Bundle has a duplicate or nonregular member") - names.add(member.name) - require(member.name in {"input.json", "manifest.json"} or member.name.startswith("files/"), "Unexpected bundle member") - if member.name.startswith("files/"): - safe_relative(member.name[6:]) - total += member.size - require(total < 5 * 1024**3, "Bundle exceeds the extraction budget") - target = destination / member.name - target.parent.mkdir(parents=True, exist_ok=True) - with archive.extractfile(member) as source, target.open("xb") as output: - shutil.copyfileobj(source, output) - require({"input.json", "manifest.json"}.issubset(names), "Incomplete signed bundle") - return Snapshot.local(destination / "files") - - -def pack_bundle(source, target): - with target.open("xb") as raw, gzip.GzipFile(fileobj=raw, mode="wb", mtime=0, filename="") as compressed: - with tarfile.open(fileobj=compressed, mode="w|") as archive: - for path in sorted(source.rglob("*")): - require(not path.is_symlink(), "Bundle contains a symlink") - if path.is_file(): - info = archive.gettarinfo(str(path), arcname=path.relative_to(source).as_posix()) - info.uid = info.gid = info.mtime = 0 - info.uname = info.gname = "" - info.mode = 0o644 - with path.open("rb") as data: - archive.addfile(info, data) diff --git a/distribution/repositories/engine.py b/distribution/repositories/engine.py deleted file mode 100644 index c8ba04cb96bb3..0000000000000 --- a/distribution/repositories/engine.py +++ /dev/null @@ -1,234 +0,0 @@ -"""Container implementation for the typed `vdev release repositories` commands.""" - -import argparse -import hashlib -import json -import os -from pathlib import Path -import re -from urllib.request import urlopen - -from bundles import checked_download, pack_bundle, unpack_bundle -import rollback -from inputs import fetch, require, verify_input -from repository import PublicKey, lint_package, prepare, repomd_locations, run, verify_metadata -from storage import (Busy, Conflict, Publisher, S3Store, Snapshot, byte_entry, - mutable, sha256_file) - - -def configuration(path): - config = json.loads(path.read_text()) - require(config["schema"] == 1, "Unsupported publisher configuration") - require(config["repository"] == "vectordotdev/vector" and config["repository_id"] == 146327667, "Wrong source repository") - require(re.fullmatch(r"[0-9]{12}", config["account"]), "Invalid AWS account") - require(re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", config["bucket"]), "Invalid bucket") - paths = [config[field] for field in ("live_prefix", "archive_prefix", "bundle_prefix", "control_key")] - for name in paths: - require(re.fullmatch(r"[a-zA-Z0-9_./-]+", name) and not name.startswith("/") - and all(part not in ("", ".", "..") for part in name.rstrip("/").split("/")), "Invalid S3 path") - require(all(config[field].endswith("/") for field in ("live_prefix", "archive_prefix", "bundle_prefix")), "Prefixes must end in / ") - require(all(not a.startswith(b) and not b.startswith(a) for i, a in enumerate(paths) for b in paths[i+1:]), - "Live, archive, bundle and control paths must be disjoint") - if config["account"] == "486234852809": - require(config["bucket"] == "dd-cose-releases" and config["live_prefix"] == "vector/repositories/", - "Unexpected production repository destination") - require(all(not value.startswith("vector/") for value in paths[1:]), "Internal state must be outside the public vector/ prefix") - for name in ("primary_fingerprint", "signing_fingerprint"): - require(re.fullmatch(r"[0-9A-F]{40}", config[name]), "Invalid pinned key fingerprint") - require(Path(config["public_key"]).name == config["public_key"], "Public key must be beside configuration") - config["public_key_path"] = str(path.parent / config["public_key"]) - config["role"] = os.environ.get("PACKAGE_REPOSITORIES_ROLE_ARN", "") - return config - - -def store_for(config): - require(re.fullmatch(r"arn:aws:iam::" + config["account"] + r":role/[A-Za-z0-9_+=,.@-]+", config["role"]), - "Set the configured publishing role ARN explicitly") - return S3Store(config) - - -def bundle_key(config, snapshot): - require(snapshot.input_id is not None, "Snapshot has no input identity") - return config["bundle_prefix"] + "bundles/" + snapshot.input_id + "/" + (snapshot.base or "initial") + ".tar.gz" - - -def rpm_cache_key(config, asset): - return config["bundle_prefix"] + "rpms/" + asset["sha256"] + ".rpm" - - -def base(config, work): - manifest, input_id = verify_input(work) - store = store_for(config) - publisher = Publisher(store, config) - state, _ = publisher.load_state() - require(not state["held"], "Publisher is locked; inspect status before retrying") - active = publisher.archived(state["active"]) if state["active"] else None - require(active is not None or not config.get("require_existing_repository", True), - "Historical repository backfill is required before release publication") - if state["pending"]: - pending = publisher.archived(state["pending"]) - require(pending.input_id == input_id, "Resume or roll back the pending publication first") - candidate = pending - elif active and active.input_id == input_id: - candidate = active - else: - candidate = Snapshot({"apt/dists/stable/InRelease": byte_entry(b"")}, - base=active.id if active else None, input_id=input_id) - key = bundle_key(config, candidate) - if store.head(key): - archive = work / "cached.tar.gz" - checked_download(store, key, archive) - snapshot = unpack_bundle(archive, work / "result") - require(snapshot.input_id == input_id and snapshot.base == candidate.base, "Cached bundle belongs to another input or base") - require((work / "result/input.json").read_bytes() == (work / "input.json").read_bytes(), "Cached source manifest differs") - archive.unlink() - print("Restored the saved signed publication; signing is skipped") - return - require(not state["pending"] and not (active and active.input_id == input_id), "Required retry bundle is missing") - (work / "base.json").write_bytes(active.data if active else b"null\n") - destination = work / "base" - destination.mkdir() - if active: - for name, entry in active.entries.items(): - if mutable(name): - key = publisher.archives + active.id + "/files/" + name - checked_download(store, key, destination / name, entry) - for root in destination.glob("yum/*/vector-0/*"): - for relative in repomd_locations(root): - name = (root / relative).relative_to(destination).as_posix() - require(name in active.entries, "RPM metadata is missing from the active manifest") - checked_download(store, publisher.live + name, destination / name, active.entries[name]) - reuse = work / "reuse" - reuse.mkdir() - for name, asset in manifest["assets"].items(): - if asset.get("format") == "rpm" and store.head(rpm_cache_key(config, asset)): - checked_download(store, rpm_cache_key(config, asset), reuse / name) - print("Downloaded current metadata and any previously signed RPMs") - - -def check_result(config, work): - manifest, input_id = verify_input(work) - snapshot = Snapshot.local(work / "result/files") - require(snapshot.input_id == input_id and (work / "result/input.json").read_bytes() == (work / "input.json").read_bytes(), - "Signed output belongs to a different immutable input") - public = PublicKey(config, "/run/keys/check-result") - try: - verify_metadata(snapshot, snapshot.source, public) - rpmdb = public.home / "rpmdb" - run(["rpm", "--dbpath", rpmdb, "--import", public.key]) - for name, asset in manifest["assets"].items(): - if asset.get("format") == "deb": - require(snapshot.entries[f"apt/pool/v/ve/{name}"] == {"size": asset["size"], "sha256": asset["sha256"]}, - "Debian package changed after input verification") - elif asset.get("format") == "rpm": - filename = f"yum/{manifest['channel']}/vector-0/{asset['arch']}/{name}" - lint_package(snapshot.source / filename, asset) - require("digests signatures OK" in run(["rpm", "--dbpath", rpmdb, "--checksig", snapshot.source / filename]).decode(), - "RPM signature verification failed") - finally: - public.close() - return manifest, snapshot - - -def publish(config, work): - manifest, snapshot = check_result(config, work) - store = store_for(config) - publisher = Publisher(store, config) - require(publisher.load_state()[0]["active"] is not None or not config.get("require_existing_repository", True), - "Historical repository backfill is required before release publication") - # Freeze signed bytes before any live write. Conditional single PUTs are atomic; - # an ambiguous response is resolved by reading the same immutable key on retry. - for name, asset in manifest["assets"].items(): - if asset.get("format") == "rpm": - relative = f"yum/{manifest['channel']}/vector-0/{asset['arch']}/{name}" - with (snapshot.source / relative).open("rb") as data: - publisher.ensure_immutable(rpm_cache_key(config, asset), data, snapshot.entries[relative]) - archive = work / "prepared.tar.gz" - if archive.exists(): - archive.unlink() - pack_bundle(work / "result", archive) - require(archive.stat().st_size < 5 * 1024**3, "Prepared bundle exceeds the single-PUT limit") - with archive.open("rb") as data: - publisher.ensure_immutable(bundle_key(config, snapshot), data, - {"size": archive.stat().st_size, "sha256": sha256_file(archive)}) - print("Published snapshot " + publisher.publish(snapshot)) - - -def verify_snapshot(config, snapshot): - names = {name for name in snapshot.entries if mutable(name)} - names.update(path.relative_to(snapshot.source).as_posix() for path in snapshot.source.rglob("*") if path.is_file()) - for name in sorted(names): - repository, relative = name.split("/", 1) - url = config[repository + "_url"] - require(url.startswith("https://") and url.endswith("/"), "Verification requires HTTPS repository roots") - expected = snapshot.entries[name] - digest = hashlib.sha256() - size = 0 - with urlopen(url + relative, timeout=90) as response: - require(response.geturl().startswith("https://"), "Repository redirected away from HTTPS") - while data := response.read(1024 * 1024): - size += len(data) - require(size <= expected["size"], "Endpoint returned an oversized object") - digest.update(data) - require({"sha256": digest.hexdigest(), "size": size} == expected, "Endpoint differs from published snapshot: " + name) - print(f"Verified {len(names)} endpoint objects for snapshot {snapshot.id}") - - -def verify(config, work): - _, snapshot = check_result(config, work) - verify_snapshot(config, snapshot) - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--config", type=Path, required=True) - commands = parser.add_subparsers(dest="command", required=True) - download = commands.add_parser("fetch") - download.add_argument("--run", type=int, required=True) - download.add_argument("--tag", required=True) - download.add_argument("--channel", choices=("stable", "beta"), required=True) - for name in ("base", "prepare", "publish", "verify", "status"): - commands.add_parser(name) - rollback_commands = commands.add_parser("rollback").add_subparsers(dest="phase", required=True) - rollback_commands.add_parser("base").add_argument("snapshot") - for name in ("prepare", "publish", "verify"): - rollback_commands.add_parser(name) - recovery = commands.add_parser("recover-lock") - recovery.add_argument("--expected-version", required=True) - recovery.add_argument("--confirm-owner-stopped", action="store_true") - args = parser.parse_args() - config = configuration(args.config) - work = Path("/work") - if args.command == "fetch": - fetch(config, work, args.run, args.tag, args.channel) - print("Verified six repository packages and their published checksums") - elif args.command == "base": - base(config, work) - elif args.command == "prepare": - print("Prepared snapshot " + prepare(config, work)) - elif args.command == "publish": - publish(config, work) - elif args.command == "verify": - verify(config, work) - elif args.command == "rollback": - if args.phase == "base": - rollback.base(config, work, args.snapshot, store_for(config)) - elif args.phase == "prepare": - print("Prepared rollback snapshot " + rollback.prepare(config, work)) - elif args.phase == "publish": - rollback.publish(config, work, store_for(config)) - else: - _, snapshot = rollback.check_result(config, work) - verify_snapshot(config, snapshot) - else: - publisher = Publisher(store_for(config), config) - if args.command == "status": - state, head = publisher.load_state() - print(json.dumps({"state": state, "version": head["VersionId"] if head else None}, indent=2)) - else: - publisher.recover_lock(args.expected_version, args.confirm_owner_stopped) - print("Lock released; resume the pending snapshot or roll back to the last completed snapshot") - - -if __name__ == "__main__": - main() diff --git a/distribution/repositories/inputs.py b/distribution/repositories/inputs.py deleted file mode 100644 index b94f2ac7f7cf6..0000000000000 --- a/distribution/repositories/inputs.py +++ /dev/null @@ -1,169 +0,0 @@ -"""Unprivileged admission and download of already-built GitHub release assets.""" - -import hashlib -import json -import os -from pathlib import Path -import re -from urllib.parse import quote -from urllib.request import HTTPRedirectHandler, Request, build_opener, urlopen - -from storage import PublicationError, canonical, sha256_file - -DEB_ARCHES = ("amd64", "arm64", "armhf") -RPM_ARCHES = ("x86_64", "aarch64", "armv7hl") -TAG = re.compile(r"v0\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)(?:-([0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*))?") - - -def require(condition, message): - if not condition: - raise PublicationError(message) - - -def versions(tag, channel): - match = TAG.fullmatch(tag) - require(match is not None, "Expected a Vector 0.x semantic release tag without build metadata") - require(channel in {"stable", "beta"}, "Invalid repository channel") - require(channel != "stable" or match[1] is None, "Stable cannot publish a prerelease") - raw = tag[1:] - native = raw.replace("-", "~", 1) - return raw, native + "-1", native.replace("-", ".") + "-1" - - -def expected_packages(tag, channel): - _, deb, rpm = versions(tag, channel) - return {**{f"vector_{deb}_{arch}.deb": {"format": "deb", "arch": arch, "version": deb} - for arch in DEB_ARCHES}, - **{f"vector-{rpm}.{arch}.rpm": {"format": "rpm", "arch": arch, "version": rpm} - for arch in RPM_ARCHES}} - - -def validate_run(config, run, tag, commit): - require(run.get("repository", {}).get("id") == config["repository_id"], "Wrong source repository") - require(run.get("head_repository", {}).get("id") == config["repository_id"], "Fork runs cannot publish") - require(run.get("path") == config["workflow"], "Wrong source workflow") - require(run.get("event") == "push" and run.get("head_branch") == tag, "Source must be a tag push") - require(run.get("status") == "completed" and run.get("conclusion") == "success", "Source run did not succeed") - require(run.get("head_sha") == commit and re.fullmatch(r"[0-9a-f]{40}", commit), "Run and tag commit differ") - - -def select_assets(release, tag, channel): - raw, deb, _ = versions(tag, channel) - require(release.get("tag_name") == tag and not release.get("draft") and release.get("published_at"), - "Release must be published and match the tag") - require(channel != "stable" or not release.get("prerelease"), "Stable release is marked prerelease") - expected = expected_packages(tag, channel) - checksums = f"vector-{raw}-SHA256SUMS" - wanted = set(expected) | {checksums} - allowed_packages = set(expected) | {f"vector_{deb}_armel.deb"} - selected = {} - seen = set() - for asset in release.get("assets", []): - name = asset["name"] - require(name not in seen, "Duplicate release asset name") - seen.add(name) - require(not name.endswith((".deb", ".rpm")) or name in allowed_packages, "Unexpected Linux package: " + name) - if name not in wanted: - continue - digest = asset.get("digest", "") - require(isinstance(digest, str) and re.fullmatch(r"sha256:[0-9a-f]{64}", digest), "Asset lacks a GitHub SHA-256 digest") - require(type(asset.get("size")) is int and 0 < asset["size"] < 1024**3, "Invalid asset size") - require(type(asset.get("id")) is int and asset["id"] > 0, "Invalid asset ID") - selected[name] = {"id": asset["id"], "size": asset["size"], "sha256": digest[7:], **expected.get(name, {})} - require(set(selected) == wanted, "Release is missing required packages or SHA256SUMS") - return selected - - -def parse_checksums(data): - result = {} - for line in data.decode("utf-8").splitlines(): - match = re.fullmatch(r"([0-9a-f]{64}) [ *]([^/\\\s]+)", line) - require(match is not None, "Malformed SHA256SUMS entry") - require(match[2] not in result, "Duplicate SHA256SUMS entry") - result[match[2]] = match[1] - return result - - -def github(endpoint): - class NoRedirect(HTTPRedirectHandler): - def redirect_request(self, request, response, code, message, headers, url): - raise PublicationError("GitHub API redirects are not allowed") - - request = Request("https://api.github.com/" + endpoint, - headers={"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", - **({"Authorization": "Bearer " + os.environ["GH_TOKEN"]} if os.environ.get("GH_TOKEN") else {})}) - with build_opener(NoRedirect).open(request, timeout=90) as response: - require(response.geturl().startswith("https://api.github.com/"), "Unexpected API redirect") - return json.load(response) - - -def fetch(config, work, run_id, tag, channel): - expected_packages(tag, channel) - repo = config["repository"] - require(repo == "vectordotdev/vector" and config["repository_id"] == 146327667, - "Release inputs must come from vectordotdev/vector") - run = github(f"repos/{repo}/actions/runs/{run_id}") - ref = github(f"repos/{repo}/git/ref/tags/{quote(tag, safe='')}")["object"] - for _ in range(4): - if ref["type"] == "commit": - break - require(ref["type"] == "tag", "Release ref does not identify a commit") - ref = github(f"repos/{repo}/git/tags/{ref['sha']}")["object"] - require(ref["type"] == "commit", "Too many annotated tag indirections") - validate_run(config, run, tag, ref["sha"]) - endpoint = f"repos/{repo}/releases/tags/{quote(tag, safe='')}" - release = github(endpoint) - assets = select_assets(release, tag, channel) - manifest = {"schema": 1, "repository": repo, "repository_id": config["repository_id"], - "run": run_id, "commit": ref["sha"], "tag": tag, "channel": channel, - "release": release["id"], "assets": assets} - destination = work / "input" - destination.mkdir(exist_ok=True) - require({p.name for p in destination.iterdir()} <= set(assets), "Unexpected files in the input directory") - if (work / "input.json").exists(): - require((work / "input.json").read_bytes() == canonical(manifest), "Previously frozen inputs have changed") - for name, asset in assets.items(): - if (destination / name).exists(): - require((destination / name).is_file() and not (destination / name).is_symlink() - and (destination / name).stat().st_size == asset["size"] - and sha256_file(destination / name) == asset["sha256"], "Previously downloaded asset differs") - continue - # Public downloads deliberately carry no GitHub credential through redirects. - url = f"https://github.com/{repo}/releases/download/{quote(tag, safe='')}/{quote(name, safe='')}" - temporary = work / "download.partial" - require(not temporary.is_symlink(), "Invalid temporary download file") - with urlopen(url, timeout=90) as source, temporary.open("wb") as target: - size = 0 - while data := source.read(1024 * 1024): - size += len(data) - require(size <= asset["size"], "Asset exceeded its declared size") - target.write(data) - require(size == asset["size"] and sha256_file(temporary) == asset["sha256"], "Asset digest/size mismatch") - temporary.replace(destination / name) - after = github(endpoint) - require(after["id"] == release["id"] and select_assets(after, tag, channel) == assets, - "Release assets changed during download") - (work / "input.json").write_bytes(canonical(manifest)) - verify_input(work) - return manifest - - -def verify_input(work): - data = (work / "input.json").read_bytes() - manifest = json.loads(data) - require(canonical(manifest) == data and manifest["schema"] == 1, "Invalid input manifest") - require(manifest["repository"] == "vectordotdev/vector" and manifest["repository_id"] == 146327667, "Wrong input repository") - expected = expected_packages(manifest["tag"], manifest["channel"]) - checksum_name = f"vector-{manifest['tag'][1:]}-SHA256SUMS" - require(set(manifest["assets"]) == set(expected) | {checksum_name}, "Input package set changed") - require({p.name for p in (work / "input").iterdir()} == set(manifest["assets"]), "Unexpected input files") - for name, asset in manifest["assets"].items(): - path = work / "input" / name - require(path.is_file() and not path.is_symlink(), "Input must be a regular file") - require(path.stat().st_size == asset["size"] and sha256_file(path) == asset["sha256"], "Input asset changed: " + name) - for field, value in expected.get(name, {}).items(): - require(asset.get(field) == value, "Input native identity changed") - sums = parse_checksums((work / "input" / checksum_name).read_bytes()) - for name in expected: - require(sums.get(name) == manifest["assets"][name]["sha256"], "SHA256SUMS disagrees with GitHub digest") - return manifest, hashlib.sha256(data).hexdigest() diff --git a/distribution/repositories/repository.py b/distribution/repositories/repository.py deleted file mode 100644 index c3f6d4cb6ee69..0000000000000 --- a/distribution/repositories/repository.py +++ /dev/null @@ -1,432 +0,0 @@ -"""Native package validation and incremental apt/yum metadata construction.""" - -import gzip -import hashlib -import os -from pathlib import Path -import shutil -import subprocess -import tarfile -import tempfile -import time -from datetime import datetime, timedelta, timezone -from email.utils import format_datetime, parsedate_to_datetime -import xml.etree.ElementTree as ET - -from inputs import DEB_ARCHES, RPM_ARCHES, require, verify_input -from storage import Snapshot, byte_entry, mutable, safe_relative, sha256_file - - -def run(args, *, cwd=None, data=None, env=None): - result = subprocess.run([str(arg) for arg in args], cwd=cwd, input=data, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, check=False) - require(result.returncode == 0, f"{args[0]} failed: " + result.stderr.decode(errors="replace")[-4000:]) - return result.stdout - - -def fields(paragraph): - result = {} - for line in paragraph.splitlines(): - if line.startswith((" ", "\t")): - continue - key, separator, value = line.partition(":") - require(separator and key not in result, "Malformed or duplicate package metadata field") - result[key] = value.strip() - return result - - -def merge_packages(previous, incoming): - packages = {} - for content in (previous, incoming): - for paragraph in content.decode().strip().split("\n\n"): - if not paragraph: - continue - item = fields(paragraph) - require(item.get("Package") == "vector", "Non-Vector Debian package") - identity = (item["Package"], item["Version"], item["Architecture"]) - old = packages.get(identity) - require(old is None or fields(old) == item, "Conflicting Debian package identity") - packages[identity] = paragraph - return ("\n\n".join(packages[key] for key in sorted(packages)) + ("\n\n" if packages else "")).encode() - - -def lint_package(path, identity): - if identity["format"] == "deb": - for field, expected in (("Package", "vector"), ("Version", identity["version"]), ("Architecture", identity["arch"])): - require(run(["dpkg-deb", "-f", path, field]).decode().strip() == expected, "Debian native identity mismatch") - process = subprocess.Popen(["dpkg-deb", "--fsys-tarfile", str(path)], stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL) - try: - size = 0 - with tarfile.open(fileobj=process.stdout, mode="r|") as archive: - for member in archive: - size += member.size - require(size < 8 * 1024**3, "Unreasonably large package payload") - require(member.issym() or member.islnk() or not member.mode & 0o002, - "World-writable Debian package entry: " + member.name) - require(process.wait() == 0, "Cannot inspect Debian payload") - finally: - process.stdout.close() - if process.poll() is None: - process.kill() - process.wait() - else: - actual = run(["rpm", "-qp", "--qf", "%{NAME} %{VERSION}-%{RELEASE} %{ARCH}", path]).decode() - require(actual == f"vector {identity['version']} {identity['arch']}", "RPM native identity mismatch") - listing = run(["rpm", "-qp", "--qf", "[%{FILEMODES:perms} %{FILENAMES}\\n]", path]).decode() - for line in listing.splitlines(): - require(len(line) >= 10, "Invalid RPM file listing") - require(line[0] == "l" or line[8] != "w", "World-writable RPM package entry: " + line[11:]) - - -def key_fingerprints(listing, primary_type, subkey_type): - result = [] - pending = None - for line in listing.decode().splitlines(): - parts = line.split(":") - if parts[0] in (primary_type, subkey_type): - pending = parts - elif parts[0] == "fpr" and pending is not None: - result.append((pending, parts[9])) - pending = None - return result - - -class PublicKey: - def __init__(self, config, directory): - self.config = config - self.home = Path(directory) - self.home.mkdir(mode=0o700, parents=True) - self.env = {**os.environ, "GNUPGHOME": str(self.home)} - self.key = Path(config["public_key_path"]) - listing = self.gpg("--with-colons", "--show-keys", str(self.key)) - keys = key_fingerprints(listing, "pub", "sub") - require([fpr for parts, fpr in keys if parts[0] == "pub"] == [config["primary_fingerprint"]], "Wrong public primary key") - require(config["signing_fingerprint"] in [fpr for parts, fpr in keys if parts[0] == "sub"], "Wrong public signing subkey") - self.gpg("--import", str(self.key)) - - def gpg(self, *args, data=None): - return run(["gpg", "--batch", "--no-tty", *args], data=data, env=self.env) - - def verify(self, signature, data=None): - args = ["--status-fd", "1", "--verify", str(signature)] - if data is not None: - args.append(str(data)) - status = self.gpg(*args).decode() - require(any(line.startswith("[GNUPG:] VALIDSIG " + self.config["signing_fingerprint"] + " ") - for line in status.splitlines()), "Signature did not use the pinned signing subkey") - - def close(self): - run(["gpgconf", "--kill", "all"], env=self.env) - shutil.rmtree(self.home) - - -class Signer(PublicKey): - def __init__(self, config, directory, private_key, passphrase): - super().__init__(config, directory) - require(private_key and passphrase and "\n" not in passphrase and "\r" not in passphrase, - "An encrypted signing subkey and a single-line passphrase are required") - self.password = self.home / "passphrase" - self.password.write_text(passphrase + "\n") - self.password.chmod(0o600) - self.gpg("--import", data=private_key.encode()) - keys = key_fingerprints(self.gpg("--with-colons", "--list-secret-keys"), "sec", "ssb") - require(len(keys) == 2, "Expected exactly one primary stub and one signing subkey") - primary, subkey = keys - require(primary[1] == config["primary_fingerprint"] and primary[0][0] == "sec" - and len(primary[0]) > 14 and primary[0][14] == "#", "Usable primary secret key is forbidden in CI") - require(subkey[1] == config["signing_fingerprint"] and subkey[0][0] == "ssb" - and len(subkey[0]) > 14 and subkey[0][14] == "+" and "s" in subkey[0][11], - "Wrong or unavailable signing subkey") - for parts, _ in keys: - require(parts[1] not in ("r", "e", "d") and parts[6] and int(parts[6]) > time.time(), "Signing identity is expired or revoked") - challenge = self.home / "challenge" - challenge.write_text("Vector repository signing capability check\n") - run(["gpgconf", "--kill", "gpg-agent"], env=self.env) - wrong = subprocess.run(["gpg", "--batch", "--no-tty", "--pinentry-mode", "loopback", - "--passphrase-fd", "0", "--local-user", config["signing_fingerprint"] + "!", - "--output", str(self.home / "wrong-passphrase.asc"), "--detach-sign", str(challenge)], - input=os.urandom(32).hex().encode() + b"\n", env=self.env, - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False) - require(wrong.returncode != 0, "Signing subkey must be passphrase-protected") - self.sign(challenge) - self.verify(challenge.with_suffix(".asc"), challenge) - self.rpmdb = self.home / "rpmdb" - run(["rpm", "--dbpath", self.rpmdb, "--import", self.key]) - - def sign(self, source, *, clear=False, destination=None): - destination = destination or Path(str(source) + ".asc") - self.gpg("--yes", "--pinentry-mode", "loopback", "--passphrase-file", str(self.password), - "--local-user", self.config["signing_fingerprint"] + "!", "--digest-algo", "SHA256", - "--armor", "--output", str(destination), "--clearsign" if clear else "--detach-sign", str(source)) - - def rpm(self, package): - run(["rpmsign", "--define", "_gpg_name " + self.config["signing_fingerprint"] + "!", - "--define", "_gpg_path " + str(self.home), "--define", "__gpg /usr/bin/gpg", - "--define", "_gpg_digest_algo sha256", "--define", - "_gpg_sign_cmd_extra_args --pinentry-mode loopback --passphrase-file " + str(self.password), - "--addsign", package], env=self.env) - output = run(["rpm", "--dbpath", self.rpmdb, "--checksig", package]).decode() - require("digests signatures OK" in output, "Signed RPM failed native verification") - - -def repomd_locations(root): - document = ET.fromstring((root / "repodata/repomd.xml").read_bytes()) - result = [] - for entry in document.findall("{*}data"): - location = entry.find("{*}location") - require(location is not None and set(location.attrib) == {"href"}, "Unexpected RPM metadata base URL") - name = location.attrib["href"] - safe_relative("yum/metadata/" + name) - require(name.startswith("repodata/") and len(Path(name).parts) == 2, "Invalid RPM metadata location") - result.append(name) - require(result, "Empty RPM root metadata") - return result - - -def check_apt_index(data, snapshot): - for paragraph in data.decode().strip().split("\n\n"): - if not paragraph: - continue - item = fields(paragraph) - require(item.get("Package") == "vector", "Foreign package in apt history") - name = "apt/" + item["Filename"] - safe_relative(name) - require(snapshot.entries.get(name) == {"sha256": item["SHA256"], "size": int(item["Size"])}, - "Debian history does not match the snapshot manifest") - - -def verify_metadata(snapshot, root, public): - for name in snapshot.entries: - if name.endswith("/InRelease"): - suite = (root / name).parent - public.verify(suite / "InRelease") - public.verify(suite / "Release.gpg", suite / "Release") - release = (suite / "Release").read_bytes() - require(public.gpg("--decrypt", str(suite / "InRelease")) == release, "Inline and detached apt roots disagree") - algorithm = None - checked = set() - for line in release.decode().splitlines(): - if not line.startswith(" "): - algorithm = "sha256" if line == "SHA256:" else None - elif algorithm: - digest, size, relative = line.split() - safe_relative("apt/index/" + relative) - entry = {"sha256": digest, "size": int(size)} - filename = (suite / relative).relative_to(root).as_posix() - require(snapshot.entries.get(filename) == entry, "Apt index differs from its signed root") - if relative.endswith("/Packages"): - require(byte_entry((suite / relative).read_bytes()) == entry, "Apt index checksum mismatch") - check_apt_index((suite / relative).read_bytes(), snapshot) - checked.add(filename) - require(checked, "Apt root has no SHA-256 package indexes") - elif name.endswith("/repomd.xml"): - package_root = (root / name).parent.parent - public.verify(Path(str(root / name) + ".asc"), root / name) - document = ET.fromstring((root / name).read_bytes()) - for item in document.findall("{*}data"): - location = item.find("{*}location") - require(location is not None and set(location.attrib) == {"href"}, "RPM metadata must use relative locations") - relative = location.attrib["href"] - safe_relative("yum/metadata/" + relative) - path = package_root / relative - entry = snapshot.entries.get(path.relative_to(root).as_posix()) - checksum = item.find("{*}checksum") - require(checksum is not None and checksum.attrib.get("type") in {"sha256", "sha512"}, "RPM metadata needs SHA-2 checksums") - data = path.read_bytes() - require(entry == byte_entry(data) and hashlib.new(checksum.attrib["type"], data).hexdigest() == checksum.text, - "RPM metadata differs from its signed root") - if item.attrib.get("type") == "primary": - require(path.suffix == ".gz", "Expected gzip RPM primary metadata") - primary = ET.fromstring(gzip.decompress(data)) - for package in primary.findall("{*}package"): - require(package.findtext("{*}name") == "vector", "Foreign package in RPM history") - url = package.find("{*}location") - checksum = package.find("{*}checksum") - size = package.find("{*}size") - require(url is not None and set(url.attrib) == {"href"}, "RPM package must use a relative location") - safe_relative("yum/package/" + url.attrib["href"]) - filename = (package_root / url.attrib["href"]).relative_to(root).as_posix() - require(checksum is not None and checksum.attrib.get("type") == "sha256" and size is not None, - "RPM package needs a SHA-256 checksum and size") - require(snapshot.entries.get(filename) == {"sha256": checksum.text, "size": int(size.attrib["package"])}, - "RPM package differs from its signed index") - - -def release_date(data): - metadata = fields(data.decode()) - require("Date" in metadata and "Valid-Until" not in metadata, - "Expected a dated repository without Valid-Until; expiry needs a separate policy") - value = parsedate_to_datetime(metadata["Date"]) - require(value.tzinfo is not None, "Repository date must have a timezone") - return value.astimezone(timezone.utc) - - -def next_release_date(previous, now=None): - now = (now or datetime.now(timezone.utc)).replace(microsecond=0) - value = max([now, *(release_date(data) + timedelta(seconds=1) for data in previous)]) - # Apt normally tolerates ten seconds into the future. Keep a margin and - # fail on clock skew rather than publishing a root clients cannot accept. - require(value <= now + timedelta(seconds=5), "Repository date is too far ahead; check the clock and retry later") - return format_datetime(value) - - -def dated_release(data, date): - release_date(data) # Reject ambiguous/unsupported roots before changing one field. - return b"".join(("Date: " + date + "\n").encode() if line.startswith(b"Date:") else line - for line in data.splitlines(keepends=True)) - - -def apt_indexes(root, channel, incoming, previous, signer): - suite = root / "apt/dists" / channel - architectures = (*DEB_ARCHES, "x86_64", "all", "i386") - for arch in architectures: - relative = f"apt/dists/{channel}/vector-0/binary-{arch}/Packages" - old = (previous / relative).read_bytes() if (previous / relative).exists() else b"" - added = incoming.get("amd64" if arch == "x86_64" else arch, b"") - data = merge_packages(old, added) - index = root / relative - index.parent.mkdir(parents=True, exist_ok=True) - index.write_bytes(data) - index.with_suffix(".gz").write_bytes(gzip.compress(data, mtime=0)) - for path in (index, index.with_suffix(".gz")): - for algorithm in ("sha256", "sha512"): - content = path.read_bytes() - hashed = path.parent / "by-hash" / algorithm.upper() / hashlib.new(algorithm, content).hexdigest() - hashed.parent.mkdir(parents=True, exist_ok=True) - hashed.write_bytes(content) - # Preserve the existing public repository identity during the key/hosting - # migration. Adding Origin/Label or changing the empty Suite would require - # a separate apt release-information approval on clients with cached indexes. - prior_release = previous / f"apt/dists/{channel}/Release" - date = next_release_date([prior_release.read_bytes()] if prior_release.exists() else []) - lines = ["Suite:", f"Codename: {channel}", "Date: " + date, - "Architectures: " + " ".join(architectures), "Components: vector-0", "Acquire-By-Hash: yes"] - for algorithm in ("sha256", "sha512"): - lines.append(algorithm.upper() + ":") - for arch in architectures: - for filename in ("Packages", "Packages.gz"): - relative = f"vector-0/binary-{arch}/{filename}" - data = (suite / relative).read_bytes() - lines.append(f" {hashlib.new(algorithm, data).hexdigest()} {len(data)} {relative}") - release = suite / "Release" - release.write_text("\n".join(lines) + "\n") - signer.sign(release, clear=True, destination=suite / "InRelease") - signer.sign(release, destination=suite / "Release.gpg") - signer.verify(suite / "InRelease") - signer.verify(suite / "Release.gpg", release) - - -def prepare(config, work): - private_key = os.environ.pop("PACKAGE_SIGNING_SUBKEY", "") - passphrase = os.environ.pop("PACKAGE_SIGNING_PASSPHRASE", "") - manifest, input_id = verify_input(work) - require(not (work / "result").exists(), "Prepared output already exists; reuse it or select a new work directory") - base_data = (work / "base.json").read_bytes() - previous = Snapshot.decode(base_data) if base_data != b"null\n" else None - base_files = work / "base" - # Check every downloaded base file before a private key is imported. - for path in base_files.rglob("*"): - require(not path.is_symlink(), "Base metadata cannot contain symlinks") - if path.is_file(): - name = path.relative_to(base_files).as_posix() - require(previous is not None and previous.entries.get(name) == byte_entry(path.read_bytes()), "Base metadata changed") - for name, identity in manifest["assets"].items(): - if "format" in identity: - lint_package(work / "input" / name, identity) - public = PublicKey(config, "/run/keys/verify") - try: - if previous: - verify_metadata(previous, base_files, public) - finally: - public.close() - signer = None - try: - signer = Signer(config, "/run/keys/sign", private_key, passphrase) - private_key = passphrase = None - result = work / "result" - root = result / "files" - root.mkdir(parents=True) - entries = dict(previous.entries) if previous else {} - if previous: - for source in base_files.rglob("*"): - if source.is_file(): - path = root / source.relative_to(base_files) - path.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(source, path) - incoming = {} - channel = manifest["channel"] - for name, identity in manifest["assets"].items(): - if "format" not in identity: - continue - relative = (f"apt/pool/v/ve/{name}" if identity["format"] == "deb" - else f"yum/{channel}/vector-0/{identity['arch']}/{name}") - if relative in entries: - require(identity["format"] == "deb" and entries[relative] == - {"size": identity["size"], "sha256": identity["sha256"]}, - "Package is already present; reuse its saved signed publication") - destination = root / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(work / "input" / name, destination) - if identity["format"] == "rpm": - cached = work / "reuse" / name - if cached.exists(): - require(cached.is_file() and not cached.is_symlink(), "Invalid saved signed RPM") - shutil.copyfile(cached, destination) - lint_package(destination, identity) - require("digests signatures OK" in run(["rpm", "--dbpath", signer.rpmdb, "--checksig", destination]).decode(), - "Saved RPM does not have a valid COSE signature") - else: - signer.rpm(destination) - else: - incoming[identity["arch"]] = run(["apt-ftparchive", "packages", relative[4:]], cwd=root / "apt") - for selected in ("stable", "beta"): - if previous is None or selected == channel: - apt_indexes(root, selected, incoming if selected == channel else {}, base_files, signer) - for arch in RPM_ARCHES: - relative = f"yum/{selected}/vector-0/{arch}" - target = root / relative - target.mkdir(parents=True, exist_ok=True) - # Previously copied mutable roots are not input to createrepo. - shutil.rmtree(target / "repodata", ignore_errors=True) - run(["createrepo_c", "--no-database", "--checksum", "sha256", "--compress-type", "gz", target]) - old = base_files / relative - if (old / "repodata/repomd.xml").exists(): - with tempfile.TemporaryDirectory() as temporary: - run(["mergerepo_c", "--all", "--omit-baseurl", "--no-database", "--compress-type", "gz", - "--repo", old.as_uri(), "--repo", target.as_uri(), "--outputdir", temporary]) - shutil.rmtree(target / "repodata") - shutil.copytree(Path(temporary) / "repodata", target / "repodata") - repomd_locations(target) - signer.sign(target / "repodata/repomd.xml") - signer.verify(target / "repodata/repomd.xml.asc", target / "repodata/repomd.xml") - for path in root.rglob("*"): - if path.is_file(): - name = path.relative_to(root).as_posix() - entry = {"size": path.stat().st_size, "sha256": sha256_file(path)} - require(mutable(name) or name not in entries or entries[name] == entry, "Immutable repository object changed") - entries[name] = entry - snapshot = Snapshot(entries, root, base=previous.id if previous else None, input_id=input_id) - # Metadata inherited from an untouched channel may refer to files held only - # in the base workspace. Validate a metadata-only view without copying RPMs. - with tempfile.TemporaryDirectory() as temporary: - view = Path(temporary) - for source in (base_files, root): - for path in source.rglob("*"): - if path.is_file() and not path.name.endswith((".deb", ".rpm")): - destination = view / path.relative_to(source) - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(path, destination) - verify_metadata(snapshot, view, signer) - (result / "manifest.json").write_bytes(snapshot.data) - (result / "input.json").write_bytes((work / "input.json").read_bytes()) - Snapshot.local(root) - return snapshot.id - finally: - if signer is not None: - signer.close() - else: - # A failed key check can leave an initialized agent; stop it before cleanup. - if Path("/run/keys/sign").exists(): - run(["gpgconf", "--kill", "all"], env={**os.environ, "GNUPGHOME": "/run/keys/sign"}) - shutil.rmtree("/run/keys/sign") diff --git a/distribution/repositories/requirements.txt b/distribution/repositories/requirements.txt deleted file mode 100644 index d27f4c469d3e6..0000000000000 --- a/distribution/repositories/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Pinned AWS SDK and transitive dependencies; wheels from PyPI. -boto3==1.43.114 --hash=sha256:d9cac2eb921ce674970cef1c9ad750f85ee3a846aedcf188d18368fb9eb6da23 -botocore==1.43.115 --hash=sha256:9ceda640c868bc15f5399129e8cfdef8cbb4b350e8a99547c9b5f477cfb9619a -jmespath==1.1.0 --hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64 -python-dateutil==2.9.0.post0 --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 -s3transfer==0.19.3 --hash=sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553 -six==1.17.0 --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 -urllib3==2.8.0 --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 diff --git a/distribution/repositories/rollback.py b/distribution/repositories/rollback.py deleted file mode 100644 index 2ca2b23109144..0000000000000 --- a/distribution/repositories/rollback.py +++ /dev/null @@ -1,236 +0,0 @@ -"""Restore an archived package selection as a new, freshly signed generation.""" - -from datetime import datetime, timedelta, timezone -import hashlib -import json -import os -from pathlib import Path -import re -import shutil - -from bundles import checked_download, pack_bundle, unpack_bundle -from inputs import require -from repository import (PublicKey, Signer, dated_release, next_release_date, - release_date, repomd_locations, verify_metadata) -from storage import Publisher, Snapshot, byte_entry, canonical, mutable, sha256_file - - -def plan_bytes(target, active, pending): - data = canonical({"schema": 1, "operation": "rollback", "target": target, - "active": active, "pending": pending}) - decode_plan(data) - return data - - -def decode_plan(data): - plan = json.loads(data) - require(set(plan) == {"schema", "operation", "target", "active", "pending"} - and plan["schema"] == 1 and plan["operation"] == "rollback" - and canonical(plan) == data, "Invalid rollback request") - for name in ("target", "active", "pending"): - value = plan[name] - require((name == "pending" and value is None) or - (isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value)), "Invalid rollback snapshot identity") - require(plan["pending"] is None or plan["target"] == plan["active"], - "Abandoning a pending publication must restore the last completed package selection") - return plan, hashlib.sha256(data).hexdigest() - - -def request_key(config, input_id): - return config["bundle_prefix"] + "rollback-requests/" + input_id + ".json" - - -def bundle_key(config, input_id): - return config["bundle_prefix"] + "rollbacks/" + input_id + ".tar.gz" - - -def source_ids(plan): - return sorted({plan[key] for key in ("target", "active", "pending") if plan[key]}) - - -def download_sources(publisher, work, plan): - for identity in source_ids(plan): - snapshot = publisher.archived(identity) - directory = work / "sources" / identity - root = directory / "files" - root.mkdir(parents=True) - (directory / "manifest.json").write_bytes(snapshot.data) - for name, entry in snapshot.entries.items(): - if mutable(name): - checked_download(publisher.store, publisher.archives + identity + "/files/" + name, root / name, entry) - # A pending generation may have stopped before uploading immutable RPM - # indexes. Its signed root dates still constrain the replacement. - if identity in {plan["active"], plan["target"]}: - for package_root in root.glob("yum/*/vector-0/*"): - for relative in repomd_locations(package_root): - name = (package_root / relative).relative_to(root).as_posix() - require(name in snapshot.entries, "RPM metadata is missing from the archived manifest") - checked_download(publisher.store, publisher.live + name, root / name, snapshot.entries[name]) - - -def base(config, work, target, store): - publisher = Publisher(store, config) - state, _ = publisher.load_state() - require(not state["held"] and state["active"], "Rollback requires an unlocked, initialized repository") - data = None - retry = False - candidate = publisher.archived(state["pending"] or state["active"]) - if candidate.input_id and store.head(request_key(config, candidate.input_id)): - saved, _ = store.read(request_key(config, candidate.input_id)) - plan, identity = decode_plan(saved) - require(identity == candidate.input_id, "Saved rollback request failed checksum verification") - if plan["target"] == target: - data = saved - retry = True - data = data or plan_bytes(target, state["active"], state["pending"]) - plan, input_id = decode_plan(data) - (work / "input.json").write_bytes(data) - download_sources(publisher, work, plan) - key = bundle_key(config, input_id) - if store.head(key): - archive = work / "cached.tar.gz" - checked_download(store, key, archive) - snapshot = unpack_bundle(archive, work / "result") - require(snapshot.input_id == input_id and snapshot.base == plan["active"] - and (work / "result/input.json").read_bytes() == data, "Saved rollback belongs to another request") - if retry: - require(snapshot.id == candidate.id, "Saved rollback differs from the pending/completed generation") - archive.unlink() - print("Restored the saved signed rollback; signing is skipped") - else: - require(candidate.input_id != input_id, "Required rollback retry bundle is missing") - print("Downloaded archived selections and all potentially exposed root dates") - - -def check_sources(config, work): - plan, input_id = decode_plan((work / "input.json").read_bytes()) - sources = {} - public = PublicKey(config, "/run/keys/rollback-sources") - try: - for identity in source_ids(plan): - directory = work / "sources" / identity - snapshot = Snapshot.decode((directory / "manifest.json").read_bytes(), identity) - root = directory / "files" - require(root.is_dir() and not root.is_symlink(), "Invalid rollback source directory") - present = set() - for path in root.rglob("*"): - require(not path.is_symlink(), "Rollback source contains a symlink") - if path.is_dir(): - continue - name = path.relative_to(root).as_posix() - require(path.is_file() and snapshot.entries.get(name) == - {"sha256": sha256_file(path), "size": path.stat().st_size}, "Rollback source differs from its manifest") - require(not name.endswith((".deb", ".rpm")), "Rollback does not require local package payloads") - present.add(name) - require({name for name in snapshot.entries if mutable(name)} <= present, "Rollback source is missing mutable metadata") - snapshot.source = root - sources[identity] = snapshot - if identity in {plan["active"], plan["target"]}: - verify_metadata(snapshot, root, public) - else: - for name in present: - if name.endswith("/Release"): - suite = (root / name).parent - public.verify(suite / "InRelease") - public.verify(suite / "Release.gpg", suite / "Release") - require(public.gpg("--decrypt", str(suite / "InRelease")) == (root / name).read_bytes(), - "Pending apt roots disagree") - elif name.endswith("/repomd.xml"): - public.verify(Path(str(root / name) + ".asc"), root / name) - selected = sources[plan["target"]] - mutable_names = {name for name in selected.entries if mutable(name)} - for snapshot in sources.values(): - require({name for name in snapshot.entries if mutable(name)} == mutable_names, - "Rollback across repository layout changes needs a separate migration") - # Completed generations preserve the immutable history. Never resurrect - # bytes absent from that history or allow two identities at the same URL. - active = sources[plan["active"]] - require(all(active.entries.get(name) == entry for name, entry in selected.entries.items() if not mutable(name)), - "Selected snapshot is not contained in the active immutable history") - finally: - public.close() - return plan, input_id, sources - - -def expected_entries(plan, sources): - entries = dict(sources[plan["target"]].entries) - entries.update({name: entry for name, entry in sources[plan["active"]].entries.items() if not mutable(name)}) - return entries - - -def prepare(config, work): - private_key = os.environ.pop("PACKAGE_SIGNING_SUBKEY", "") - passphrase = os.environ.pop("PACKAGE_SIGNING_PASSPHRASE", "") - plan, input_id, sources = check_sources(config, work) - selected = sources[plan["target"]] - root = work / "result/files" - shutil.copytree(selected.source, root) - entries = expected_entries(plan, sources) - signer = Signer(config, "/run/keys/rollback-signing", private_key, passphrase) - try: - for name in selected.entries: - if name.endswith("/Release"): - release = root / name - date = next_release_date([(snapshot.source / name).read_bytes() for snapshot in sources.values()]) - release.write_bytes(dated_release(release.read_bytes(), date)) - signer.sign(release, clear=True, destination=release.parent / "InRelease") - signer.sign(release, destination=release.parent / "Release.gpg") - for path in root.rglob("*"): - if path.is_file(): - entries[path.relative_to(root).as_posix()] = byte_entry(path.read_bytes()) - snapshot = Snapshot(entries, root, base=plan["active"], input_id=input_id) - (work / "result/manifest.json").write_bytes(snapshot.data) - (work / "result/input.json").write_bytes((work / "input.json").read_bytes()) - finally: - signer.close() - check_result(config, work) - return snapshot.id - - -def check_result(config, work): - plan, input_id, sources = check_sources(config, work) - snapshot = Snapshot.local(work / "result/files") - require(snapshot.input_id == input_id and snapshot.base == plan["active"] - and (work / "result/input.json").read_bytes() == (work / "input.json").read_bytes(), - "Signed rollback belongs to a different request") - selected = sources[plan["target"]] - expected = expected_entries(plan, sources) - require(snapshot.entries.keys() == expected.keys(), "Rollback changed the repository file set") - refreshed = {name for name in expected if name.endswith(("/Release", "/InRelease", "/Release.gpg"))} - require(all(snapshot.entries[name] == entry for name, entry in expected.items() if name not in refreshed), - "Rollback changed package bytes or the selected indexes") - for name in expected: - if name.endswith("/Release"): - data = (snapshot.source / name).read_bytes() - date = release_date(data) - require(all(date > release_date((source.source / name).read_bytes()) for source in sources.values()), - "Rollback date is not newer than every potentially exposed root") - require(date <= datetime.now(timezone.utc) + timedelta(seconds=5), "Rollback root is dated too far into the future") - require(dated_release(data, "DATE") == dated_release((selected.source / name).read_bytes(), "DATE"), - "Rollback changed apt identity or index checksums") - public = PublicKey(config, "/run/keys/rollback-result") - try: - verify_metadata(snapshot, snapshot.source, public) - finally: - public.close() - return plan, snapshot - - -def publish(config, work, store): - plan, snapshot = check_result(config, work) - publisher = Publisher(store, config) - for identity in source_ids(plan): - require(publisher.archived(identity).data == (work / "sources" / identity / "manifest.json").read_bytes(), - "Rollback source is not the saved archive") - archive = work / "prepared.tar.gz" - archive.unlink(missing_ok=True) - pack_bundle(work / "result", archive) - require(archive.stat().st_size < 5 * 1024**3, "Rollback bundle exceeds the single-PUT limit") - with archive.open("rb") as stream: - publisher.ensure_immutable(bundle_key(config, snapshot.input_id), stream, - {"sha256": sha256_file(archive), "size": archive.stat().st_size}) - data = (work / "input.json").read_bytes() - publisher.ensure_immutable(request_key(config, snapshot.input_id), data, byte_entry(data)) - identity = publisher.publish(snapshot, recovery=(plan["active"], plan["pending"])) - print("Published fresh rollback snapshot " + identity) - return identity diff --git a/distribution/repositories/storage.py b/distribution/repositories/storage.py deleted file mode 100644 index d06f500ff9446..0000000000000 --- a/distribution/repositories/storage.py +++ /dev/null @@ -1,400 +0,0 @@ -"""Conditional S3 publication protocol, invoked through vdev release repositories.""" - -import base64 -from concurrent.futures import ThreadPoolExecutor -from datetime import datetime, timedelta, timezone -from email.utils import parsedate_to_datetime -import hashlib -import json -import os -from pathlib import Path, PurePosixPath -import re -import sys -import uuid - -import boto3 -from botocore.config import Config -from botocore.exceptions import ClientError - - -class PublicationError(RuntimeError): - pass - - -class Busy(PublicationError): - pass - - -class Conflict(PublicationError): - pass - - -class Interrupted(PublicationError): - """A deliberate test interruption after all preceding S3 requests finished.""" - - -MUTABLE_NAMES = {"Packages", "Packages.gz", "Release", "Release.gpg", "InRelease", - "repomd.xml", "repomd.xml.asc"} -ROOT_ORDER = {"Release": 0, "Release.gpg": 1, "InRelease": 2, - "repomd.xml": 3, "repomd.xml.asc": 4} - - -def canonical(value): - return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode() - - -def sha256_file(path): - digest = hashlib.sha256() - with path.open("rb") as stream: - for block in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(block) - return digest.hexdigest() - - -def safe_relative(name): - if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9_./+~\-]+", name): - raise PublicationError("Invalid manifest path") - path = PurePosixPath(name) - if path.is_absolute() or str(path) != name or ".." in path.parts: - raise PublicationError("Noncanonical manifest path") - if len(path.parts) < 3 or path.parts[0] not in {"apt", "yum"}: - raise PublicationError("Manifest path is outside the repository roots") - return path - - -def mutable(name): - return safe_relative(name).name in MUTABLE_NAMES - - -class Snapshot: - def __init__(self, entries, source=None, *, base=None, input_id=None): - if not isinstance(entries, dict) or not entries: - raise PublicationError("Empty or invalid snapshot manifest") - for name, entry in entries.items(): - safe_relative(name) - if not isinstance(entry, dict) or set(entry) != {"sha256", "size"}: - raise PublicationError("Invalid manifest entry") - if not re.fullmatch(r"[0-9a-f]{64}", entry["sha256"]): - raise PublicationError("Invalid manifest checksum") - if type(entry["size"]) is not int or not 0 <= entry["size"] <= 5 * 1024**3: - raise PublicationError("Only single-PUT objects up to 5 GiB are supported") - if not any(mutable(name) for name in entries): - raise PublicationError("Snapshot has no repository metadata") - self.entries = entries - self.source = source - for value in (base, input_id): - if value is not None and not re.fullmatch(r"[0-9a-f]{64}", value): - raise PublicationError("Invalid snapshot identity") - self.base = base - self.input_id = input_id - self.data = canonical({"schema": 2, "base": base, "input": input_id, "files": entries}) - self.id = hashlib.sha256(self.data).hexdigest() - - @classmethod - def decode(cls, data, expected_id=None): - value = json.loads(data) - if set(value) != {"schema", "base", "input", "files"} or value["schema"] != 2: - raise PublicationError("Unsupported snapshot schema") - snapshot = cls(value["files"], base=value["base"], input_id=value["input"]) - if data != snapshot.data or (expected_id is not None and snapshot.id != expected_id): - raise PublicationError("Snapshot manifest is not canonical or has the wrong digest") - return snapshot - - @classmethod - def local(cls, source): - source = Path(source) - if source.is_symlink() or not source.is_dir(): - raise PublicationError("Snapshot must be a regular directory") - snapshot = cls.decode((source.parent / "manifest.json").read_bytes()) - entries = {} - for path in sorted(source.rglob("*")): - if path.is_symlink(): - raise PublicationError("Snapshot symlinks are not allowed") - if path.is_dir(): - continue - if not path.is_file(): - raise PublicationError("Snapshot contains a nonregular file") - name = path.relative_to(source).as_posix() - safe_relative(name) - entries[name] = {"sha256": sha256_file(path), "size": path.stat().st_size} - for name, entry in entries.items(): - if snapshot.entries.get(name) != entry: - raise PublicationError("Snapshot content differs from recorded manifest: " + name) - for name in snapshot.entries: - if name not in entries and (mutable(name) or snapshot.base is None): - raise PublicationError("Snapshot is missing a required local file: " + name) - snapshot.source = source - return snapshot - - -class S3Store: - def __init__(self, config): - # Capture one fixed STS session. Never refresh credentials while owning a lock. - required = ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN") - if not all(os.environ.get(name) for name in required): - raise PublicationError("A fixed temporary AWS session is required") - session = boto3.Session( - aws_access_key_id=os.environ[required[0]], - aws_secret_access_key=os.environ[required[1]], - aws_session_token=os.environ[required[2]], - region_name=config["region"], - ) - # Disable automatic retries: an ambiguous mutable PUT must retain its lock. - sdk = Config(retries={"total_max_attempts": 1}, max_pool_connections=16, - connect_timeout=10, read_timeout=90, - s3={"us_east_1_regional_endpoint": "regional"}) - identity = session.client("sts", config=sdk).get_caller_identity() - role = config["role"].rsplit("/", 1)[1] - expected = f'arn:aws:sts::{config["account"]}:assumed-role/{role}/' - if identity["Account"] != config["account"] or not identity["Arn"].startswith(expected): - raise PublicationError("Wrong AWS account or role") - actual_role = session.client("iam", config=sdk).get_role(RoleName=role)["Role"] - if actual_role["MaxSessionDuration"] > 3600: - raise PublicationError("Role exceeds the one-hour recovery credential bound") - self.client = session.client("s3", config=sdk.merge(Config(signature_version="s3v4"))) - self.bucket = config["bucket"] - self.live_prefix = config["live_prefix"] - self.owner = identity["Arn"] - self.server_time = parsedate_to_datetime(identity["ResponseMetadata"]["HTTPHeaders"]["date"]) - - def response_time(self, response): - self.server_time = parsedate_to_datetime(response["ResponseMetadata"]["HTTPHeaders"]["date"]) - - def head(self, key): - try: - result = self.client.head_object(Bucket=self.bucket, Key=key, ChecksumMode="ENABLED") - except ClientError as exc: - self.response_time(exc.response) - status = exc.response["ResponseMetadata"]["HTTPStatusCode"] - if status == 404: - return None - if status == 403: - # Prefix-scoped ListBucket may hide missing keys behind HEAD 403. - # An authorized exact-prefix listing distinguishes absence from denial. - listed = self.client.list_objects_v2(Bucket=self.bucket, Prefix=key, MaxKeys=1) - self.response_time(listed) - if not any(item["Key"] == key for item in listed.get("Contents", [])): - return None - raise - self.response_time(result) - return result - - def read(self, key): - result = self.client.get_object(Bucket=self.bucket, Key=key, ChecksumMode="ENABLED") - self.response_time(result) - with result["Body"] as stream: - data = stream.read() - return data, result - - def put(self, key, body, entry, *, previous=None, immutable=False): - options = {"IfNoneMatch": "*"} if previous is None else {"IfMatch": previous["ETag"]} - result = self.client.put_object( - Bucket=self.bucket, Key=key, Body=body, ContentLength=entry["size"], - ChecksumSHA256=base64.b64encode(bytes.fromhex(entry["sha256"])).decode(), - ServerSideEncryption="AES256", ContentType=content_type(key), - CacheControl=(("public" if key.startswith(self.live_prefix) else "private") - + (",max-age=31536000,immutable" if immutable - else ",max-age=0,no-cache,must-revalidate")), **options, - ) - self.response_time(result) - return result - - def download(self, key, destination): - result = self.client.get_object(Bucket=self.bucket, Key=key, ChecksumMode="ENABLED") - self.response_time(result) - digest = hashlib.sha256() - with result["Body"] as source, destination.open("wb") as target: - for block in iter(lambda: source.read(1024 * 1024), b""): - digest.update(block) - target.write(block) - return {"sha256": digest.hexdigest(), "size": destination.stat().st_size} - - -def content_type(key): - if key.endswith(".deb"): - return "application/vnd.debian.binary-package" - if key.endswith(".rpm"): - return "application/x-rpm" - if key.endswith(".json"): - return "application/json" - if key.endswith(".gz"): - return "application/gzip" - if key.endswith(".xml"): - return "application/xml" - return "application/octet-stream" - - -def matches(head, entry): - return bool(head and head.get("ChecksumSHA256") == base64.b64encode( - bytes.fromhex(entry["sha256"])).decode() and head["ContentLength"] == entry["size"]) - - -def byte_entry(data): - return {"sha256": hashlib.sha256(data).hexdigest(), "size": len(data)} - - -class Publisher: - def __init__(self, store, config, hook=None): - self.store = store - self.live = config["live_prefix"] - self.archives = config["archive_prefix"] - self.lock_key = config["control_key"] - self.hook = hook or (lambda phase: None) - self.lock_head = None - self.state = None - - def load_state(self): - head = self.store.head(self.lock_key) - if head is None: - return {"schema": 1, "held": False, "active": None, "pending": None}, None - data, current = self.store.read(self.lock_key) - state = json.loads(data) - if state.get("schema") != 1 or type(state.get("held")) is not bool: - raise PublicationError("Invalid publication control object") - return state, current - - def write_state(self): - self.state["nonce"] = uuid.uuid4().hex # Prevent an ABA match on lock state. - data = canonical(self.state) - self.lock_head = self.store.put(self.lock_key, data, byte_entry(data), previous=self.lock_head) - - def acquire(self): - self.state, self.lock_head = self.load_state() - if self.state["held"]: - raise Busy("Publisher busy; locks are never stolen on timeout") - self.state.update(held=True, owner=self.store.owner, - recover_after=(self.store.server_time + timedelta(minutes=65)).isoformat()) - try: - self.write_state() - except ClientError as exc: - if exc.response["ResponseMetadata"]["HTTPStatusCode"] in (409, 412): - raise Busy("Another publisher acquired the lock") from None - raise - self.hook("locked") - - def release(self): - self.state["held"] = False - self.write_state() - - def recover_lock(self, expected_version, owner_stopped): - self.state, self.lock_head = self.load_state() - if not owner_stopped: - raise PublicationError("Confirm that the previous workflow/process has stopped") - if not self.state["held"] or self.lock_head["VersionId"] != expected_version: - raise Conflict("Lock version changed or is already released") - if self.store.server_time < datetime.fromisoformat(self.state["recover_after"]): - raise Busy("Recovery waits 65 minutes so the previous fixed STS session expires") - # Conditional replacement cannot unlock a different owner's newer lock. - self.release() - - def ensure_immutable(self, key, body, entry): - old = self.store.head(key) - if old is not None: - if not matches(old, entry): - raise Conflict("Immutable object conflict: " + key) - return - self.store.put(key, body, entry, immutable=True) - - def archive(self, snapshot): - prefix = self.archives + snapshot.id + "/" - for name, entry in sorted(snapshot.entries.items()): - if mutable(name): - with (snapshot.source / name).open("rb") as stream: - self.ensure_immutable(prefix + "files/" + name, stream, entry) - self.ensure_immutable(prefix + "manifest.json", snapshot.data, byte_entry(snapshot.data)) - - def archived(self, snapshot_id): - if not re.fullmatch(r"[0-9a-f]{64}", snapshot_id): - raise PublicationError("Invalid snapshot ID") - data, _ = self.store.read(self.archives + snapshot_id + "/manifest.json") - return Snapshot.decode(data, snapshot_id) - - def preflight(self, snapshot): - active = self.state["active"] - if active != snapshot.id: - if active != snapshot.base: - raise Conflict("Repository advanced after preparation; regenerate against the current base") - if active: - previous = self.archived(active) - for name, entry in previous.entries.items(): - if not mutable(name) and snapshot.entries.get(name) != entry: - raise Conflict("Publication would remove or replace historical data: " + name) - names = sorted(snapshot.entries) - with ThreadPoolExecutor(max_workers=16) as workers: - heads = list(workers.map(self.store.head, (self.live + name for name in names))) - for name, old in zip(names, heads): - entry = snapshot.entries[name] - if not mutable(name): - if old is not None and not matches(old, entry): - raise Conflict("Immutable object conflict: " + name) - if old is None and (snapshot.source is None or not (snapshot.source / name).is_file()): - raise Conflict("Inherited payload/index no longer exists: " + name) - - def publish(self, snapshot, *, recovery=None): - self.acquire() - try: - active, pending = self.state["active"], self.state["pending"] - if recovery is not None: - # Recovery may abandon a pending generation only from the exact - # state whose signed metadata was checked during preparation. - expected_active, expected_pending = recovery - allowed = {(expected_active, expected_pending), - (expected_active, snapshot.id), (snapshot.id, None)} - if snapshot.base != expected_active or (active, pending) not in allowed: - raise Conflict("Repository changed after rollback preparation; inspect and prepare again") - elif pending and snapshot.id != pending: - raise Conflict("Recover the pending snapshot or roll back to the last completed one") - self.preflight(snapshot) - # Nothing above changes client-visible metadata. - self.archive(snapshot) - metadata = {} - for name, entry in snapshot.entries.items(): - if mutable(name): - data, _ = self.store.read(self.archives + snapshot.id + "/files/" + name) - if byte_entry(data) != entry: - raise Conflict("Archived metadata failed digest verification: " + name) - metadata[name] = data - self.state["pending"] = snapshot.id - self.write_state() - self.hook("pending") - for name, entry in sorted(snapshot.entries.items()): - if not mutable(name): - # Preflight verified inherited immutable objects under the lock. - if not (snapshot.source / name).exists(): - continue - if matches(self.store.head(self.live + name), entry): - continue - with (snapshot.source / name).open("rb") as stream: - self.ensure_immutable(self.live + name, stream, entry) - self.hook("payloads") - indexes = sorted(name for name in snapshot.entries - if PurePosixPath(name).name in {"Packages", "Packages.gz"}) - roots = sorted((name for name in snapshot.entries if mutable(name) and name not in indexes), - key=lambda name: (ROOT_ORDER[PurePosixPath(name).name], name)) - for name in indexes + roots: - entry = snapshot.entries[name] - key = self.live + name - previous = self.store.head(key) - if matches(previous, entry): - continue - self.store.put(key, metadata[name], entry, previous=previous) - self.hook("written:" + name) - names = sorted(snapshot.entries) - with ThreadPoolExecutor(max_workers=16) as workers: - heads = list(workers.map(self.store.head, (self.live + name for name in names))) - for name, head in zip(names, heads): - if not matches(head, snapshot.entries[name]): - raise PublicationError("Published object failed final verification: " + name) - self.state.update(active=snapshot.id, pending=None) - self.write_state() - except (Conflict, Interrupted): - # These are deliberate, synchronous stops with no outstanding AWS request. - self.release() - raise - except BaseException: - # A lost response can hide a completed PUT. Never unlock automatically. - print("Publication stopped; the lock remains held for operator recovery", file=sys.stderr) - raise - else: - self.release() - return snapshot.id diff --git a/docs/reference/package-repositories.md b/docs/reference/package-repositories.md index 27f711f8ecb8d..b64e8e3d9c8b9 100644 --- a/docs/reference/package-repositories.md +++ b/docs/reference/package-repositories.md @@ -2,8 +2,13 @@ The `Package Repositories` workflow consumes existing GitHub release assets and publishes apt and yum repositories. `cargo vdev release repositories` provides the -maintainer commands. The container implementation is under -`distribution/repositories`; its image is built within each workflow run. +maintainer commands and the Rust publisher implementation in +`vdev/src/commands/release/repositories/publisher`. The tools image is built within +each workflow run from a pinned Rust builder and Ubuntu runtime. It contains +`vdev`, GPG and native Debian/RPM metadata tools; Python and pip are not required. +The `package-repositories` Cargo feature keeps AWS SDK dependencies out of ordinary +vdev builds. `distribution/repositories` holds the image definition, production +configuration, public key and standalone shell installer. This workflow is disabled unless the repository variable `PACKAGE_REPOSITORIES_ENABLED` is `true`. It requires a completed historical diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index 444e8073fcc66..16548b8201937 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -19,7 +19,29 @@ include = [ [lints] workspace = true +[features] +# The ordinary developer CLI only launches the tools container. Compile the AWS +# clients and publisher implementation inside that container. +package-repositories = [ + "dep:aws-config", "dep:aws-sdk-s3", "dep:aws-sdk-sts", "dep:aws-sdk-iam", + "dep:aws-smithy-runtime-api", "dep:aws-smithy-types", "dep:base64", "dep:flate2", "dep:reqwest", "dep:roxmltree", "dep:tar", + "dep:tokio", "dep:uuid", +] + [dependencies] +aws-config = { version = "=1.12.0", default-features = false, features = ["behavior-version-latest", "rt-tokio", "default-https-client"], optional = true } +aws-sdk-s3 = { version = "=1.146.0", default-features = false, features = ["behavior-version-latest", "rt-tokio", "default-https-client"], optional = true } +aws-sdk-sts = { version = "=1.114.0", default-features = false, features = ["behavior-version-latest", "rt-tokio", "default-https-client"], optional = true } +aws-sdk-iam = { version = "1.0", default-features = false, features = ["behavior-version-latest", "rt-tokio", "default-https-client"], optional = true } +aws-smithy-runtime-api = { version = "1.16", features = ["client"], optional = true } +aws-smithy-types = { version = "1", optional = true } +base64 = { workspace = true, optional = true } +flate2 = { workspace = true, optional = true } +reqwest = { workspace = true, features = ["blocking"], optional = true } +roxmltree = { version = "0.21", optional = true } +tar = { version = "0.4", optional = true } +tokio = { workspace = true, features = ["rt-multi-thread"], optional = true } +uuid = { version = "1", features = ["v4"], optional = true } anyhow.workspace = true chrono.workspace = true clap.workspace = true diff --git a/vdev/src/commands/release/repositories.rs b/vdev/src/commands/release/repositories.rs index 7e62270252670..a51238de831d1 100644 --- a/vdev/src/commands/release/repositories.rs +++ b/vdev/src/commands/release/repositories.rs @@ -1,3 +1,6 @@ +#[cfg(feature = "package-repositories")] +mod publisher; + use std::{path::PathBuf, process::Command}; use anyhow::{Context as _, Result, ensure}; @@ -10,6 +13,9 @@ pub(super) struct Cli { /// Container built with `build-image` from the reviewed publisher source. #[arg(long, default_value = "vector-package-publisher:local", global = true)] image: String, + /// Run the publisher inside the prepared tools image. + #[arg(long, hide = true, global = true)] + in_container: bool, /// Configuration and public key directory; never contains a private key. #[arg( long, @@ -82,10 +88,23 @@ enum Channel { impl Cli { pub fn exec(self) -> Result<()> { + if self.in_container { + #[cfg(feature = "package-repositories")] + return publisher::exec(&self.config, &self.work, self.command); + #[cfg(not(feature = "package-repositories"))] + anyhow::bail!("The publisher requires the package-repositories Cargo feature"); + } app::set_repo_dir()?; if matches!(self.command, Commands::BuildImage) { return Command::new("docker") - .args(["build", "--tag", &self.image, "distribution/repositories"]) + .args([ + "build", + "--tag", + &self.image, + "--file", + "distribution/repositories/Dockerfile", + ".", + ]) .check_run(); } std::fs::create_dir_all(&self.work)?; @@ -136,6 +155,9 @@ impl Cli { command.args(["--env", name]); } command.arg(&self.image).args([ + "--in-container", + "--work", + "/work", "--config", &format!( "/config/{}", diff --git a/vdev/src/commands/release/repositories/publisher/bundles.rs b/vdev/src/commands/release/repositories/publisher/bundles.rs new file mode 100644 index 0000000000000..eb8ad5a7e5819 --- /dev/null +++ b/vdev/src/commands/release/repositories/publisher/bundles.rs @@ -0,0 +1,95 @@ +//! Bounded, checksum-verified transport of signed workspaces. +use super::{ + model::{Entry, LIMIT, Snapshot, files, safe_relative}, + storage::Store, +}; +use anyhow::{Result, ensure}; +use flate2::{Compression, GzBuilder}; +use std::{collections::BTreeSet, fs, io::Write, path::Path}; +pub(super) fn checked_download( + store: &dyn Store, + key: &str, + path: &Path, + entry: Option<&Entry>, +) -> Result<()> { + let head = store + .head(key)? + .ok_or_else(|| anyhow::anyhow!("Missing S3 input: {key}"))?; + ensure!(head.entry.size < LIMIT, "Oversized S3 input"); + fs::create_dir_all( + path.parent() + .ok_or_else(|| anyhow::anyhow!("Missing parent"))?, + )?; + let actual = store.download(key, path, head.entry.size)?; + ensure!( + head.entry == actual && entry.is_none_or(|e| *e == actual), + "Downloaded S3 checksum mismatch" + ); + Ok(()) +} +pub(super) fn unpack_bundle(path: &Path, destination: &Path) -> Result { + fs::create_dir(destination)?; + let mut archive = tar::Archive::new(super::model::gzip(fs::File::open(path)?, LIMIT)); + let mut names = BTreeSet::new(); + let mut total: u64 = 0; + for member in archive.entries()? { + let mut member = member?; + let name = member + .path()? + .to_str() + .ok_or_else(|| anyhow::anyhow!("Non-UTF8 archive name"))? + .to_owned(); + ensure!( + member.header().entry_type().is_file() && names.insert(name.clone()), + "Duplicate or nonregular bundle member" + ); + if let Some(relative) = name.strip_prefix("files/") { + safe_relative(relative)?; + } else { + ensure!( + matches!(name.as_str(), "input.json" | "manifest.json"), + "Unexpected bundle member" + ); + } + total = total + .checked_add(member.size()) + .ok_or_else(|| anyhow::anyhow!("Archive size overflow"))?; + ensure!(total < LIMIT, "Bundle extraction budget exceeded"); + let target = destination.join(name); + fs::create_dir_all(target.parent().unwrap())?; + let mut output = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(target)?; + std::io::copy(&mut member, &mut output)?; + output.flush()?; + } + ensure!( + names.contains("input.json") && names.contains("manifest.json"), + "Incomplete signed bundle" + ); + Snapshot::local(&destination.join("files")) +} +pub(super) fn pack_bundle(source: &Path, target: &Path) -> Result<()> { + let output = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(target)?; + let compressed = GzBuilder::new() + .mtime(0) + .write(output, Compression::default()); + let mut archive = tar::Builder::new(compressed); + for (name, path) in files(source)? { + let mut header = tar::Header::new_gnu(); + header.set_size(fs::metadata(&path)?.len()); + header.set_uid(0); + header.set_gid(0); + header.set_mtime(0); + header.set_mode(0o644); + header.set_entry_type(tar::EntryType::Regular); + header.set_cksum(); + archive.append_data(&mut header, name, fs::File::open(path)?)?; + } + archive.into_inner()?.finish()?; + Ok(()) +} diff --git a/vdev/src/commands/release/repositories/publisher/engine.rs b/vdev/src/commands/release/repositories/publisher/engine.rs new file mode 100644 index 0000000000000..924c1a10c9cad --- /dev/null +++ b/vdev/src/commands/release/repositories/publisher/engine.rs @@ -0,0 +1,291 @@ +//! Publication phases shared by the protected CI jobs. +use super::{ + bundles::{checked_download, pack_bundle, unpack_bundle}, + inputs::{client, verify_input}, + model::{Config, Entry, LIMIT, Snapshot, files, mutable, string, write}, + repository::{PublicKey, lint_package, repomd_locations, verify_metadata, verify_rpm}, + storage::{Publisher, S3Store, Store}, +}; +use anyhow::{Context, Result, ensure}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::{collections::BTreeSet, fs, io::Read, path::Path}; + +fn bundle_key(config: &Config, input: &str, base: Option<&str>) -> String { + format!( + "{}bundles/{input}/{}.tar.gz", + config.bundle_prefix, + base.unwrap_or("initial") + ) +} +fn rpm_cache_key(config: &Config, asset: &Value) -> Result { + Ok(format!( + "{}rpms/{}.rpm", + config.bundle_prefix, + string(asset, "sha256")? + )) +} +pub(super) fn download_metadata( + publisher: &Publisher<'_>, + snapshot: &Snapshot, + destination: &Path, + indexes: bool, +) -> Result<()> { + fs::create_dir_all(destination)?; + for (name, entry) in &snapshot.entries { + if mutable(name) { + checked_download( + publisher.store, + &format!( + "{}{}/files/{name}", + publisher.config.archive_prefix, snapshot.id + ), + &destination.join(name), + Some(entry), + )?; + } + } + if indexes { + for name in snapshot + .entries + .keys() + .filter(|n| n.ends_with("/repomd.xml")) + { + let path = destination.join(name); + let root = path + .parent() + .and_then(Path::parent) + .context("Invalid RPM root")?; + for relative in repomd_locations(root)? { + let path = root.join(relative); + let name = path + .strip_prefix(destination)? + .to_str() + .context("Invalid index path")?; + let entry = snapshot + .entries + .get(name) + .context("RPM metadata absent from manifest")?; + checked_download( + publisher.store, + &format!("{}{name}", publisher.config.live_prefix), + &path, + Some(entry), + )?; + } + } + } + Ok(()) +} +pub(super) fn base(config: &Config, work: &Path) -> Result<()> { + let (manifest, input_id) = verify_input(work)?; + let store = S3Store::new(config)?; + let publisher = Publisher::new(&store, config); + let (state, _) = publisher.load_state()?; + ensure!(!state.held, "Publisher locked; inspect status"); + let active = state + .active + .as_deref() + .map(|id| publisher.archived(id)) + .transpose()?; + ensure!( + active.is_some() || !config.require_existing_repository, + "Historical backfill required before publication" + ); + let candidate = if let Some(pending) = &state.pending { + let snapshot = publisher.archived(pending)?; + ensure!( + snapshot.input.as_deref() == Some(&input_id), + "Resume or roll back pending publication" + ); + Some(snapshot) + } else { + active + .as_ref() + .filter(|s| s.input.as_deref() == Some(&input_id)) + .cloned() + }; + let base = candidate.as_ref().map_or_else( + || active.as_ref().map(|s| s.id.as_str()), + |s| s.base.as_deref(), + ); + let key = bundle_key(config, &input_id, base); + if store.head(&key)?.is_some() { + let archive = work.join("cached.tar.gz"); + checked_download(&store, &key, &archive, None)?; + let snapshot = unpack_bundle(&archive, &work.join("result"))?; + ensure!( + snapshot.input.as_deref() == Some(&input_id) + && snapshot.base.as_deref() == base + && fs::read(work.join("result/input.json"))? == fs::read(work.join("input.json"))?, + "Cached publication belongs to another input/base" + ); + fs::remove_file(archive)?; + println!("Restored saved signed publication; signing skipped"); + return Ok(()); + } + ensure!( + candidate.is_none(), + "Required signed retry bundle is missing" + ); + write( + &work.join("base.json"), + active + .as_ref() + .map_or(b"null\n".as_slice(), |s| s.data.as_slice()), + )?; + let destination = work.join("base"); + fs::create_dir(&destination)?; + if let Some(active) = active { + download_metadata(&publisher, &active, &destination, true)?; + } + let reuse = work.join("reuse"); + fs::create_dir(&reuse)?; + for (name, asset) in manifest["assets"].as_object().context("Missing assets")? { + if asset["format"] == "rpm" { + let key = rpm_cache_key(config, asset)?; + if store.head(&key)?.is_some() { + checked_download(&store, &key, &reuse.join(name), None)?; + } + } + } + println!("Downloaded current metadata and saved RPM signatures"); + Ok(()) +} +pub(super) fn check_result(config: &Config, work: &Path) -> Result<(Value, Snapshot)> { + let (manifest, input_id) = verify_input(work)?; + let snapshot = Snapshot::local(&work.join("result/files"))?; + ensure!( + snapshot.input.as_deref() == Some(&input_id) + && fs::read(work.join("result/input.json"))? == fs::read(work.join("input.json"))?, + "Signed output belongs to another input" + ); + let public = PublicKey::new(config, "check-result")?; + verify_metadata(&snapshot, &snapshot.source, &public)?; + let rpmdb = public.rpmdb()?; + for (name, asset) in manifest["assets"].as_object().context("Missing assets")? { + if asset["format"] == "deb" { + ensure!( + snapshot.entries.get(&format!("apt/pool/v/ve/{name}")) + == Some(&super::model::entry_value(asset)?), + "Debian package changed" + ); + } else if asset["format"] == "rpm" { + let path = snapshot.source.join(format!( + "yum/{}/vector-0/{}/{name}", + string(&manifest, "channel")?, + string(asset, "arch")? + )); + lint_package(&path, asset)?; + verify_rpm(&rpmdb, &path)?; + } + } + Ok((manifest, snapshot)) +} +pub(super) fn save_bundle(publisher: &Publisher<'_>, work: &Path, key: &str) -> Result<()> { + let archive = work.join("prepared.tar.gz"); + if archive.exists() { + fs::remove_file(&archive)?; + } + pack_bundle(&work.join("result"), &archive)?; + let entry = Entry::file(&archive)?; + ensure!( + entry.size < LIMIT, + "Prepared bundle exceeds single-PUT limit" + ); + publisher.ensure_file(key, &archive, &entry) +} +pub(super) fn publish(config: &Config, work: &Path) -> Result<()> { + let (manifest, snapshot) = check_result(config, work)?; + let store = S3Store::new(config)?; + let mut publisher = Publisher::new(&store, config); + ensure!( + publisher.load_state()?.0.active.is_some() || !config.require_existing_repository, + "Historical backfill required" + ); + for (name, asset) in manifest["assets"].as_object().context("Missing assets")? { + if asset["format"] == "rpm" { + let relative = format!( + "yum/{}/vector-0/{}/{name}", + string(&manifest, "channel")?, + string(asset, "arch")? + ); + publisher.ensure_file( + &rpm_cache_key(config, asset)?, + &snapshot.source.join(&relative), + &snapshot.entries[&relative], + )?; + } + } + save_bundle( + &publisher, + work, + &bundle_key( + config, + snapshot + .input + .as_deref() + .context("Missing input identity")?, + snapshot.base.as_deref(), + ), + )?; + publisher.publish(&snapshot, None)?; + println!("Published snapshot {}", snapshot.id); + Ok(()) +} +pub(super) fn verify_snapshot(config: &Config, snapshot: &Snapshot) -> Result<()> { + let mut names: BTreeSet<_> = snapshot + .entries + .keys() + .filter(|n| mutable(n)) + .cloned() + .collect(); + names.extend(files(&snapshot.source)?.into_keys()); + let client = client(true)?; + for name in &names { + let (repository, relative) = name.split_once('/').context("Invalid repository path")?; + let root = if repository == "apt" { + &config.apt_url + } else { + &config.yum_url + }; + ensure!( + root.starts_with("https://") && root.ends_with('/'), + "Verification requires HTTPS repository roots" + ); + let expected = &snapshot.entries[name]; + let response = client + .get(format!("{root}{relative}")) + .send()? + .error_for_status()?; + ensure!( + response.url().scheme() == "https", + "Repository redirected away from HTTPS" + ); + let mut source = response.take(expected.size + 1); + let mut hash = Sha256::new(); + let mut size = 0; + let mut buffer = vec![0; 1024 * 1024]; + loop { + let count = source.read(&mut buffer)?; + if count == 0 { + break; + } + size += count as u64; + hash.update(&buffer[..count]); + } + ensure!( + Entry { + sha256: hex::encode(hash.finalize()), + size + } == *expected, + "Endpoint differs from snapshot: {name}" + ); + } + println!( + "Verified {} endpoint objects for {}", + names.len(), + snapshot.id + ); + Ok(()) +} diff --git a/vdev/src/commands/release/repositories/publisher/inputs.rs b/vdev/src/commands/release/repositories/publisher/inputs.rs new file mode 100644 index 0000000000000..2986bd81d105a --- /dev/null +++ b/vdev/src/commands/release/repositories/publisher/inputs.rs @@ -0,0 +1,319 @@ +//! Admission of already-built, immutable GitHub release assets. +use super::model::{Config, Entry, canonical, digest, entry_value, files, is_digest, string}; +use anyhow::{Context, Result, ensure}; +use reqwest::{blocking::Client, redirect::Policy}; +use serde_json::{Value, json}; +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + io::{Read, Write}, + path::Path, + time::Duration, +}; + +pub(super) const DEB_ARCHES: [&str; 3] = ["amd64", "arm64", "armhf"]; +pub(super) const RPM_ARCHES: [&str; 3] = ["x86_64", "aarch64", "armv7hl"]; +pub(super) fn versions(tag: &str, channel: &str) -> Result<(String, String, String)> { + ensure!( + regex::Regex::new( + r"^v0\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)(?:-([0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*))?$" + )? + .is_match(tag), + "Expected a Vector 0.x release tag" + ); + ensure!( + matches!(channel, "stable" | "beta") && (channel != "stable" || !tag.contains('-')), + "Invalid channel or stable prerelease" + ); + let raw = tag + .strip_prefix('v') + .context("Missing tag prefix")? + .to_owned(); + let native = raw.replacen('-', "~", 1); + Ok(( + raw, + format!("{native}-1"), + format!("{}-1", native.replace('-', ".")), + )) +} +pub(super) fn expected_packages(tag: &str, channel: &str) -> Result> { + let (_, deb, rpm) = versions(tag, channel)?; + Ok(DEB_ARCHES + .into_iter() + .map(|arch| { + ( + format!("vector_{deb}_{arch}.deb"), + json!({"format":"deb","arch":arch,"version":deb}), + ) + }) + .chain(RPM_ARCHES.into_iter().map(|arch| { + ( + format!("vector-{rpm}.{arch}.rpm"), + json!({"format":"rpm","arch":arch,"version":rpm}), + ) + })) + .collect()) +} +pub(super) fn validate_run(config: &Config, run: &Value, tag: &str, commit: &str) -> Result<()> { + ensure!( + run["repository"]["id"] == config.repository_id + && run["head_repository"]["id"] == config.repository_id, + "Wrong source repository or fork run" + ); + ensure!( + run["path"] == config.workflow && run["event"] == "push" && run["head_branch"] == tag, + "Source must be the release workflow tag push" + ); + ensure!( + run["status"] == "completed" && run["conclusion"] == "success", + "Source run did not succeed" + ); + ensure!( + run["head_sha"] == commit + && commit.len() == 40 + && commit + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)), + "Run and tag commit differ" + ); + Ok(()) +} +pub(super) fn select_assets( + release: &Value, + tag: &str, + channel: &str, +) -> Result> { + let (raw, deb, _) = versions(tag, channel)?; + ensure!( + release["tag_name"] == tag + && release["draft"] == false + && release["published_at"] + .as_str() + .is_some_and(|s| !s.is_empty()), + "Release must be published and match tag" + ); + ensure!( + channel != "stable" || release["prerelease"] == false, + "Stable release marked prerelease" + ); + let expected = expected_packages(tag, channel)?; + let sums = format!("vector-{raw}-SHA256SUMS"); + let mut seen = BTreeSet::new(); + let mut selected = BTreeMap::new(); + for asset in release["assets"] + .as_array() + .context("Missing release assets")? + { + let name = string(asset, "name")?; + ensure!(seen.insert(name), "Duplicate release asset"); + ensure!( + !name.ends_with(".deb") && !name.ends_with(".rpm") + || expected.contains_key(name) + || name == format!("vector_{deb}_armel.deb"), + "Unexpected Linux package: {name}" + ); + if !expected.contains_key(name) && name != sums { + continue; + } + let hash = string(asset, "digest")? + .strip_prefix("sha256:") + .context("Asset lacks SHA-256 digest")?; + ensure!(is_digest(hash), "Invalid asset digest"); + let size = asset["size"].as_u64().context("Invalid asset size")?; + ensure!( + size > 0 && size < 1024 * 1024 * 1024 && asset["id"].as_u64().is_some_and(|id| id > 0), + "Invalid asset size or ID" + ); + let mut item = expected.get(name).cloned().unwrap_or_else(|| json!({})); + item["id"] = asset["id"].clone(); + item["size"] = json!(size); + item["sha256"] = json!(hash); + selected.insert(name.to_owned(), item); + } + ensure!( + selected.len() == expected.len() + 1, + "Missing required packages or SHA256SUMS" + ); + Ok(selected) +} +pub(super) fn parse_checksums(bytes: &[u8]) -> Result> { + let expression = regex::Regex::new(r"^([0-9a-f]{64}) [ *]([^/\\ \t\r\n\x0b\x0c]+)$")?; + let mut sums = BTreeMap::new(); + for line in std::str::from_utf8(bytes)?.lines() { + let captures = expression + .captures(line) + .context("Malformed SHA256SUMS entry")?; + ensure!( + sums.insert(captures[2].to_owned(), captures[1].to_owned()) + .is_none(), + "Duplicate SHA256SUMS entry" + ); + } + Ok(sums) +} +pub(super) fn client(redirects: bool) -> Result { + Ok(Client::builder() + .timeout(Duration::from_secs(90)) + .https_only(true) + .redirect(if redirects { + Policy::limited(10) + } else { + Policy::none() + }) + .user_agent("vector-package-publisher") + .build()?) +} +fn github(client: &Client, endpoint: &str) -> Result { + let mut request = client + .get(format!("https://api.github.com/{endpoint}")) + .header("Accept", "application/vnd.github+json") + .header("X-GitHub-Api-Version", "2022-11-28"); + if let Ok(token) = std::env::var("GH_TOKEN") { + request = request.bearer_auth(token); + } + let response = request.send()?.error_for_status()?; + ensure!( + response.status().is_success(), + "GitHub API redirects are forbidden" + ); + Ok(response.json()?) +} +pub(super) fn download( + client: &Client, + url: &str, + destination: &Path, + expected: &Entry, +) -> Result<()> { + ensure!(!destination.is_symlink(), "Invalid download destination"); + let mut response = client.get(url).send()?.error_for_status()?; + ensure!( + response.url().scheme() == "https", + "Download redirected away from HTTPS" + ); + let mut file = fs::File::create(destination)?; + let size = std::io::copy( + &mut Read::by_ref(&mut response).take(expected.size + 1), + &mut file, + )?; + file.flush()?; + ensure!( + size == expected.size && Entry::file(destination)? == *expected, + "Asset digest/size mismatch" + ); + Ok(()) +} +pub(super) fn fetch( + config: &Config, + work: &Path, + run_id: u64, + tag: &str, + channel: &str, +) -> Result<()> { + expected_packages(tag, channel)?; + let api = client(false)?; + let public = client(true)?; + let repo = &config.repository; + let run = github(&api, &format!("repos/{repo}/actions/runs/{run_id}"))?; + // Admitted tags contain only URL-path-safe ASCII. + let mut reference = + github(&api, &format!("repos/{repo}/git/ref/tags/{tag}"))?["object"].clone(); + for _ in 0..4 { + if reference["type"] == "commit" { + break; + } + ensure!(reference["type"] == "tag", "Ref does not identify a commit"); + reference = github( + &api, + &format!("repos/{repo}/git/tags/{}", string(&reference, "sha")?), + )?["object"] + .clone(); + } + ensure!(reference["type"] == "commit", "Too many tag indirections"); + validate_run(config, &run, tag, string(&reference, "sha")?)?; + let endpoint = format!("repos/{repo}/releases/tags/{tag}"); + let release = github(&api, &endpoint)?; + let assets = select_assets(&release, tag, channel)?; + let manifest = json!({"schema":1,"repository":repo,"repository_id":config.repository_id,"run":run_id,"commit":reference["sha"],"tag":tag,"channel":channel,"release":release["id"],"assets":assets}); + let data = canonical(&manifest)?; + let destination = work.join("input"); + fs::create_dir_all(&destination)?; + ensure!( + files(&destination)?.keys().all(|n| assets.contains_key(n)), + "Unexpected input files" + ); + if work.join("input.json").exists() { + ensure!( + fs::read(work.join("input.json"))? == data, + "Frozen inputs changed" + ); + } + for (name, asset) in &assets { + let path = destination.join(name); + let expected = entry_value(asset)?; + if path.exists() { + ensure!(Entry::file(&path)? == expected, "Downloaded asset changed"); + continue; + } + let temporary = work.join("download.partial"); + download( + &public, + &format!("https://github.com/{repo}/releases/download/{tag}/{name}"), + &temporary, + &expected, + )?; + fs::rename(temporary, path)?; + } + let after = github(&api, &endpoint)?; + ensure!( + after["id"] == release["id"] && select_assets(&after, tag, channel)? == assets, + "Release assets changed during download" + ); + fs::write(work.join("input.json"), data)?; + verify_input(work)?; + println!("Verified six repository packages and published checksums"); + Ok(()) +} +pub(super) fn verify_input(work: &Path) -> Result<(Value, String)> { + let data = fs::read(work.join("input.json"))?; + let manifest: Value = serde_json::from_slice(&data)?; + ensure!( + canonical(&manifest)? == data && manifest["schema"] == 1, + "Invalid input manifest" + ); + ensure!( + manifest["repository"] == "vectordotdev/vector" && manifest["repository_id"] == 146_327_667, + "Wrong input repository" + ); + let tag = string(&manifest, "tag")?; + let expected = expected_packages(tag, string(&manifest, "channel")?)?; + let sums_name = format!( + "vector-{}-SHA256SUMS", + tag.strip_prefix('v').context("Missing tag prefix")? + ); + let assets = manifest["assets"].as_object().context("Missing assets")?; + let wanted: BTreeSet<_> = expected.keys().chain(std::iter::once(&sums_name)).collect(); + ensure!( + assets.keys().collect::>() == wanted + && files(&work.join("input"))?.keys().collect::>() == wanted, + "Input file set changed" + ); + for (name, asset) in assets { + ensure!( + Entry::file(&work.join("input").join(name))? == entry_value(asset)?, + "Input asset changed: {name}" + ); + if let Some(identity) = expected.get(name) { + for (field, value) in identity.as_object().context("Invalid identity")? { + ensure!(asset.get(field) == Some(value), "Native identity changed"); + } + } + } + let sums = parse_checksums(&fs::read(work.join("input").join(sums_name))?)?; + for name in expected.keys() { + ensure!( + sums.get(name).map(String::as_str) == assets[name]["sha256"].as_str(), + "SHA256SUMS disagrees with GitHub digest" + ); + } + Ok((manifest, digest(&data))) +} diff --git a/vdev/src/commands/release/repositories/publisher/mod.rs b/vdev/src/commands/release/repositories/publisher/mod.rs new file mode 100644 index 0000000000000..c3b0bdd72584b --- /dev/null +++ b/vdev/src/commands/release/repositories/publisher/mod.rs @@ -0,0 +1,76 @@ +//! Container-side implementation of `vdev release repositories`. +// Repository object names are case-sensitive protocol identifiers. +#![allow(clippy::case_sensitive_file_extension_comparisons)] +mod bundles; +mod engine; +mod inputs; +mod model; +mod repository; +mod rollback; +mod storage; +use super::{Channel, Commands, Rollback}; +use anyhow::{Result, bail}; +use model::Config; +use std::path::Path; + +pub(super) fn exec(config: &Path, work: &Path, command: Commands) -> Result<()> { + let config = Config::load(config)?; + let work = work.canonicalize()?; + match command { + Commands::Fetch { run, tag, channel } => inputs::fetch( + &config, + &work, + run, + &tag, + match channel { + Channel::Stable => "stable", + Channel::Beta => "beta", + }, + ), + Commands::Base => engine::base(&config, &work), + Commands::Prepare => { + println!("Prepared snapshot {}", repository::prepare(&config, &work)?); + Ok(()) + } + Commands::Publish => engine::publish(&config, &work), + Commands::Verify => { + engine::verify_snapshot(&config, &engine::check_result(&config, &work)?.1) + } + Commands::Rollback { phase } => match phase { + Rollback::Base { snapshot } => rollback::base(&config, &work, &snapshot), + Rollback::Prepare => { + println!( + "Prepared rollback snapshot {}", + rollback::prepare(&config, &work)? + ); + Ok(()) + } + Rollback::Publish => rollback::publish(&config, &work), + Rollback::Verify => { + engine::verify_snapshot(&config, &rollback::check_result(&config, &work)?.1) + } + }, + Commands::Status | Commands::RecoverLock { .. } => { + let store = storage::S3Store::new(&config)?; + let mut publisher = storage::Publisher::new(&store, &config); + if let Commands::RecoverLock { + expected_version, + confirm_owner_stopped, + } = command + { + publisher.recover_lock(&expected_version, confirm_owner_stopped)?; + println!("Lock released; resume pending publication or roll back"); + } else { + let (state, head) = publisher.load_state()?; + println!( + "{}", + serde_json::to_string_pretty( + &serde_json::json!({"state":state,"version":head.map(|h|h.version)}) + )? + ); + } + Ok(()) + } + Commands::BuildImage => bail!("Build the image outside the container"), + } +} diff --git a/vdev/src/commands/release/repositories/publisher/model.rs b/vdev/src/commands/release/repositories/publisher/model.rs new file mode 100644 index 0000000000000..e76cb888535a3 --- /dev/null +++ b/vdev/src/commands/release/repositories/publisher/model.rs @@ -0,0 +1,375 @@ +//! Stable on-disk and S3 formats shared by publication, retries and rollback. +use anyhow::{Context, Result, ensure}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use std::{ + collections::BTreeMap, + fs, + io::{Read, Write}, + path::{Path, PathBuf}, +}; + +pub(super) const LIMIT: u64 = 5 * 1024 * 1024 * 1024; + +pub(super) fn canonical(value: &impl Serialize) -> Result> { + let mut value = serde_json::to_value(value)?; + value.sort_all_objects(); + let text = serde_json::to_string(&value)?; + // Preserve the original manifest encoding, including Python's ensure_ascii. + let mut bytes = Vec::new(); + for character in text.chars() { + if character.is_ascii() { + bytes.push(character as u8); + } else { + for unit in character.encode_utf16(&mut [0; 2]) { + write!(bytes, "\\u{unit:04x}")?; + } + } + } + bytes.push(b'\n'); + Ok(bytes) +} + +pub(super) fn digest(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) +} +pub(super) fn is_digest(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct Entry { + pub sha256: String, + pub size: u64, +} +impl Entry { + pub fn bytes(bytes: &[u8]) -> Self { + Self { + sha256: digest(bytes), + size: bytes.len() as u64, + } + } + pub fn file(path: &Path) -> Result { + ensure!( + path.symlink_metadata()?.is_file(), + "Expected a regular file: {}", + path.display() + ); + let mut file = fs::File::open(path)?; + let mut hash = Sha256::new(); + let mut size = 0; + let mut buffer = vec![0; 1024 * 1024]; + loop { + let count = file.read(&mut buffer)?; + if count == 0 { + break; + } + size += count as u64; + hash.update(&buffer[..count]); + } + Ok(Self { + sha256: hex::encode(hash.finalize()), + size, + }) + } +} + +pub(super) fn safe_relative(name: &str) -> Result<&Path> { + let parts: Vec<_> = name.split('/').collect(); + ensure!( + name.bytes() + .all(|b| b.is_ascii_alphanumeric() || b"_./+~-".contains(&b)) + && parts.len() >= 3 + && matches!(parts[0], "apt" | "yum") + && parts + .iter() + .all(|p| !p.is_empty() && *p != "." && *p != ".."), + "Invalid repository path: {name}" + ); + Ok(Path::new(name)) +} +pub(super) fn mutable(name: &str) -> bool { + matches!( + name.rsplit('/').next(), + Some( + "Packages" + | "Packages.gz" + | "Release" + | "Release.gpg" + | "InRelease" + | "repomd.xml" + | "repomd.xml.asc" + ) + ) +} +pub(super) fn root_order(name: &str) -> u8 { + match name.rsplit('/').next().unwrap_or("") { + "Release" => 0, + "Release.gpg" => 1, + "InRelease" => 2, + "repomd.xml" => 3, + "repomd.xml.asc" => 4, + _ => 5, + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Manifest { + schema: u32, + base: Option, + input: Option, + files: BTreeMap, +} +#[derive(Clone, Debug)] +pub(super) struct Snapshot { + pub entries: BTreeMap, + pub base: Option, + pub input: Option, + pub data: Vec, + pub id: String, + pub source: PathBuf, +} +impl Snapshot { + pub fn new( + entries: BTreeMap, + base: Option, + input: Option, + source: PathBuf, + ) -> Result { + ensure!( + !entries.is_empty() && entries.keys().any(|n| mutable(n)), + "Snapshot has no repository metadata" + ); + for (name, entry) in &entries { + safe_relative(name)?; + ensure!( + is_digest(&entry.sha256) && entry.size <= LIMIT, + "Invalid snapshot entry" + ); + } + for id in [&base, &input].into_iter().flatten() { + ensure!(is_digest(id), "Invalid snapshot identity"); + } + let data = canonical(&Manifest { + schema: 2, + base: base.clone(), + input: input.clone(), + files: entries.clone(), + })?; + Ok(Self { + entries, + base, + input, + id: digest(&data), + data, + source, + }) + } + pub fn decode(data: &[u8], expected: Option<&str>) -> Result { + let value: Manifest = serde_json::from_slice(data)?; + ensure!(value.schema == 2, "Unsupported snapshot schema"); + let snapshot = Self::new(value.files, value.base, value.input, PathBuf::new())?; + ensure!( + snapshot.data == data && expected.is_none_or(|id| id == snapshot.id), + "Noncanonical manifest or incorrect digest" + ); + Ok(snapshot) + } + pub fn local(source: &Path) -> Result { + let mut snapshot = Self::decode( + &fs::read( + source + .parent() + .context("Missing snapshot parent")? + .join("manifest.json"), + )?, + None, + )?; + let files = files(source)?; + for (name, path) in &files { + ensure!( + snapshot.entries.get(name) == Some(&Entry::file(path)?), + "Snapshot content differs: {name}" + ); + } + for name in snapshot.entries.keys() { + ensure!( + files.contains_key(name) || (!mutable(name) && snapshot.base.is_some()), + "Missing snapshot file: {name}" + ); + } + source.clone_into(&mut snapshot.source); + Ok(snapshot) + } +} + +/// Walk without following symlinks, including at the root. +pub(super) fn files(root: &Path) -> Result> { + fn visit(root: &Path, directory: &Path, result: &mut BTreeMap) -> Result<()> { + ensure!( + directory.symlink_metadata()?.is_dir(), + "Expected a regular directory: {}", + directory.display() + ); + for item in fs::read_dir(directory)? { + let path = item?.path(); + let kind = path.symlink_metadata()?.file_type(); + if kind.is_dir() { + visit(root, &path, result)?; + } else { + ensure!(kind.is_file(), "Nonregular file: {}", path.display()); + result.insert( + path.strip_prefix(root)? + .to_str() + .context("Non-UTF8 path")? + .replace('\\', "/"), + path, + ); + } + } + Ok(()) + } + let mut result = BTreeMap::new(); + visit(root, root, &mut result)?; + Ok(result) +} +pub(super) fn write(path: &Path, data: &[u8]) -> Result<()> { + fs::create_dir_all(path.parent().context("Missing parent")?)?; + ensure!(!path.is_symlink(), "Refusing a symlink output"); + fs::write(path, data)?; + Ok(()) +} +pub(super) fn copy(source: &Path, destination: &Path) -> Result<()> { + ensure!( + source.symlink_metadata()?.is_file() && !destination.is_symlink(), + "Copy requires regular files" + ); + fs::create_dir_all(destination.parent().context("Missing parent")?)?; + fs::copy(source, destination)?; + Ok(()) +} + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Config { + pub schema: u32, + #[serde(default = "default_true")] + pub require_existing_repository: bool, + pub repository: String, + pub repository_id: u64, + pub workflow: String, + pub account: String, + pub bucket: String, + pub region: String, + pub live_prefix: String, + pub archive_prefix: String, + pub bundle_prefix: String, + pub control_key: String, + pub public_key: String, + pub primary_fingerprint: String, + pub signing_fingerprint: String, + pub apt_url: String, + pub yum_url: String, + #[serde(skip)] + pub key_path: PathBuf, +} +fn default_true() -> bool { + true +} +impl Config { + pub fn load(path: &Path) -> Result { + let mut config: Self = serde_json::from_slice(&fs::read(path)?)?; + ensure!( + config.schema == 1 + && config.repository == "vectordotdev/vector" + && config.repository_id == 146_327_667, + "Wrong publisher configuration/source repository" + ); + ensure!( + config.account.len() == 12 && config.account.bytes().all(|b| b.is_ascii_digit()), + "Invalid AWS account" + ); + ensure!( + regex::Regex::new(r"^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$")?.is_match(&config.bucket), + "Invalid bucket" + ); + let paths = [ + &config.live_prefix, + &config.archive_prefix, + &config.bundle_prefix, + &config.control_key, + ]; + for (index, name) in paths.iter().enumerate() { + ensure!( + name.bytes() + .all(|b| b.is_ascii_alphanumeric() || b"_./-".contains(&b)) + && name + .trim_end_matches('/') + .split('/') + .all(|p| !p.is_empty() && p != "." && p != ".."), + "Invalid S3 path" + ); + ensure!(index == 3 || name.ends_with('/'), "Prefixes must end in /"); + for other in &paths[index + 1..] { + ensure!( + !name.starts_with(other.as_str()) && !other.starts_with(name.as_str()), + "S3 paths must be disjoint" + ); + } + } + if config.account == "486234852809" { + ensure!( + config.bucket == "dd-cose-releases" && config.live_prefix == "vector/repositories/", + "Unexpected production destination" + ); + ensure!( + paths[1..].iter().all(|p| !p.starts_with("vector/")), + "Internal state cannot use the public prefix" + ); + } + for fingerprint in [&config.primary_fingerprint, &config.signing_fingerprint] { + ensure!( + fingerprint.len() == 40 + && fingerprint + .bytes() + .all(|b| b.is_ascii_digit() || (b'A'..=b'F').contains(&b)), + "Invalid pinned fingerprint" + ); + } + ensure!( + !config.public_key.is_empty() + && Path::new(&config.public_key) + .file_name() + .is_some_and(|p| p == config.public_key.as_str()), + "Public key must be beside configuration" + ); + config.key_path = path + .parent() + .context("Missing configuration parent")? + .join(&config.public_key); + Ok(config) + } +} +pub(super) fn string<'a>(value: &'a Value, key: &str) -> Result<&'a str> { + value[key] + .as_str() + .with_context(|| format!("Missing/invalid {key}")) +} +pub(super) fn entry_value(value: &Value) -> Result { + Ok(serde_json::from_value( + json!({"size":value["size"], "sha256":value["sha256"]}), + )?) +} + +/// Publishing has its own extraction budgets and does not depend on Vector's +/// runtime crates. Cap the decompressed stream before tar/XML parsing. +#[allow(clippy::disallowed_types)] +pub(super) fn gzip(source: impl Read, limit: u64) -> impl Read { + flate2::read::GzDecoder::new(source).take(limit) +} diff --git a/vdev/src/commands/release/repositories/publisher/repository.rs b/vdev/src/commands/release/repositories/publisher/repository.rs new file mode 100644 index 0000000000000..9fcfb08dfe6f7 --- /dev/null +++ b/vdev/src/commands/release/repositories/publisher/repository.rs @@ -0,0 +1,999 @@ +//! Native package validation, `OpenPGP` signing and incremental repository metadata. +use super::{ + inputs::{DEB_ARCHES, RPM_ARCHES, verify_input}, + model::{ + Config, Entry, Snapshot, copy, digest, entry_value, files, mutable, safe_relative, string, + write, + }, +}; +use anyhow::{Context, Result, ensure}; +use chrono::{DateTime, Timelike, Utc}; +use flate2::{Compression, GzBuilder}; +use roxmltree::{Document, Node}; +use serde_json::Value; +use sha2::{Digest, Sha512}; +use std::fmt::Write as _; +use std::{ + collections::{BTreeMap, BTreeSet}, + ffi::OsStr, + fs, + io::{Read, Write}, + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; + +pub(super) fn command(program: &str) -> Command { + let mut command = Command::new(program); + // Never leak signing material or cloud credentials to packaging subprocesses. + for name in [ + "PACKAGE_SIGNING_SUBKEY", + "PACKAGE_SIGNING_PASSPHRASE", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "GH_TOKEN", + ] { + command.env_remove(name); + } + command +} +pub(super) fn run(command: &mut Command, input: Option<&[u8]>) -> Result> { + command + .stdin(if input.is_some() { + Stdio::piped() + } else { + Stdio::null() + }) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let mut child = command + .spawn() + .with_context(|| format!("Cannot start {}", command.get_program().to_string_lossy()))?; + if let Some(input) = input { + child + .stdin + .take() + .context("Missing command stdin")? + .write_all(input)?; + } + let output = child.wait_with_output()?; + ensure!( + output.status.success(), + "{} failed: {}", + command.get_program().to_string_lossy(), + String::from_utf8_lossy(&output.stderr) + .chars() + .rev() + .take(4000) + .collect::() + .chars() + .rev() + .collect::() + ); + Ok(output.stdout) +} +fn text(command: &mut Command) -> Result { + Ok(String::from_utf8(run(command, None)?)?) +} +pub(super) fn fields(paragraph: &str) -> Result> { + let mut result = BTreeMap::new(); + for line in paragraph.lines() { + if line.starts_with([' ', '\t']) { + continue; + } + let (key, value) = line.split_once(':').context("Malformed package metadata")?; + ensure!( + result + .insert(key.to_owned(), value.trim().to_owned()) + .is_none(), + "Duplicate metadata field" + ); + } + Ok(result) +} +pub(super) fn merge_packages(previous: &[u8], incoming: &[u8]) -> Result> { + let mut packages: BTreeMap<(String, String, String), &str> = BTreeMap::new(); + for data in [previous, incoming] { + for paragraph in std::str::from_utf8(data)? + .trim() + .split("\n\n") + .filter(|p| !p.is_empty()) + { + let item = fields(paragraph)?; + ensure!( + item.get("Package").map(String::as_str) == Some("vector"), + "Foreign Debian package" + ); + let key = ( + item.get("Package").context("Missing name")?.clone(), + item.get("Version").context("Missing version")?.clone(), + item.get("Architecture") + .context("Missing architecture")? + .clone(), + ); + if let Some(old) = packages.get(&key) { + ensure!(fields(old)? == item, "Conflicting Debian identity"); + } + packages.insert(key, paragraph); + } + } + let mut result = Vec::new(); + for paragraph in packages.values() { + result.extend_from_slice(paragraph.as_bytes()); + result.extend_from_slice(b"\n\n"); + } + Ok(result) +} +pub(super) fn lint_package(path: &Path, identity: &Value) -> Result<()> { + if identity["format"] == "deb" { + for (field, expected) in [ + ("Package", "vector"), + ("Version", string(identity, "version")?), + ("Architecture", string(identity, "arch")?), + ] { + ensure!( + text(command("dpkg-deb").arg("-f").arg(path).arg(field))?.trim() == expected, + "Debian native identity mismatch" + ); + } + let mut child = command("dpkg-deb") + .arg("--fsys-tarfile") + .arg(path) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn()?; + let result = (|| -> Result<()> { + let mut archive = + tar::Archive::new(child.stdout.take().context("Missing dpkg stdout")?); + let mut size: u64 = 0; + for member in archive.entries()? { + let member = member?; + size = size + .checked_add(member.size()) + .context("Package size overflow")?; + ensure!(size < 8 * 1024 * 1024 * 1024, "Unreasonably large payload"); + let kind = member.header().entry_type(); + ensure!( + kind.is_symlink() + || kind.is_hard_link() + || member.header().mode()? & 0o002 == 0, + "World-writable Debian entry" + ); + } + Ok(()) + })(); + if result.is_err() { + drop(child.kill()); + } + let status = child.wait()?; + result?; + ensure!(status.success(), "Cannot inspect Debian payload"); + } else { + ensure!(identity["format"] == "rpm", "Unsupported package format"); + let actual = text( + command("rpm") + .args(["-qp", "--qf", "%{NAME} %{VERSION}-%{RELEASE} %{ARCH}"]) + .arg(path), + )?; + ensure!( + actual + == format!( + "vector {} {}", + string(identity, "version")?, + string(identity, "arch")? + ), + "RPM native identity mismatch" + ); + for line in text( + command("rpm") + .args(["-qp", "--qf", "[%{FILEMODES:perms} %{FILENAMES}\\n]"]) + .arg(path), + )? + .lines() + { + ensure!( + line.len() >= 10 && (line.starts_with('l') || line.as_bytes()[8] != b'w'), + "Invalid or world-writable RPM entry" + ); + } + } + Ok(()) +} +fn fingerprints(data: &[u8], primary: &str, subkey: &str) -> Result, String)>> { + let mut pending = None; + let mut result = Vec::new(); + for line in std::str::from_utf8(data)?.lines() { + let parts: Vec<_> = line.split(':').map(str::to_owned).collect(); + if parts[0] == primary || parts[0] == subkey { + pending = Some(parts); + } else if parts[0] == "fpr" + && let Some(key) = pending.take() + { + result.push((key, parts.get(9).context("Missing fingerprint")?.clone())); + } + } + Ok(result) +} +pub(super) struct PublicKey<'a> { + pub config: &'a Config, + pub home: PathBuf, +} +impl<'a> PublicKey<'a> { + pub fn new(config: &'a Config, name: &str) -> Result { + use std::os::unix::fs::DirBuilderExt as _; + let home = Path::new("/run/keys").join(name); + fs::DirBuilder::new().mode(0o700).create(&home)?; + let public = Self { config, home }; + let keys = fingerprints( + &public.gpg( + [ + OsStr::new("--with-colons"), + OsStr::new("--show-keys"), + config.key_path.as_os_str(), + ], + None, + )?, + "pub", + "sub", + )?; + ensure!( + keys.iter() + .filter(|(p, _)| p[0] == "pub") + .map(|(_, f)| f.as_str()) + .collect::>() + == [config.primary_fingerprint.as_str()], + "Wrong public primary key" + ); + ensure!( + keys.iter() + .any(|(p, f)| p[0] == "sub" && *f == config.signing_fingerprint), + "Wrong signing subkey" + ); + public.gpg([OsStr::new("--import"), config.key_path.as_os_str()], None)?; + Ok(public) + } + fn gpg_command(&self) -> Command { + let mut cmd = command("gpg"); + cmd.args(["--batch", "--no-tty"]) + .env("GNUPGHOME", &self.home); + cmd + } + pub fn gpg( + &self, + args: impl IntoIterator>, + input: Option<&[u8]>, + ) -> Result> { + run(self.gpg_command().args(args), input) + } + pub fn verify(&self, signature: &Path, data: Option<&Path>) -> Result<()> { + let mut cmd = self.gpg_command(); + cmd.args(["--status-fd", "1", "--verify"]).arg(signature); + if let Some(data) = data { + cmd.arg(data); + } + let status = text(&mut cmd)?; + ensure!( + status.lines().any(|l| l.starts_with(&format!( + "[GNUPG:] VALIDSIG {} ", + self.config.signing_fingerprint + ))), + "Signature did not use pinned subkey" + ); + Ok(()) + } + pub fn rpmdb(&self) -> Result { + let path = self.home.join("rpmdb"); + run( + command("rpm") + .arg("--dbpath") + .arg(&path) + .arg("--import") + .arg(&self.config.key_path), + None, + )?; + Ok(path) + } +} +impl Drop for PublicKey<'_> { + fn drop(&mut self) { + drop( + command("gpgconf") + .args(["--kill", "all"]) + .env("GNUPGHOME", &self.home) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(), + ); + drop(fs::remove_dir_all(&self.home)); + } +} +pub(super) struct Signer<'a> { + pub public: PublicKey<'a>, + password: PathBuf, + pub rpmdb: PathBuf, +} +impl<'a> Signer<'a> { + pub fn new(config: &'a Config, name: &str) -> Result { + use std::os::unix::fs::OpenOptionsExt as _; + let private = + std::env::var("PACKAGE_SIGNING_SUBKEY").context("Missing encrypted signing subkey")?; + let passphrase = + std::env::var("PACKAGE_SIGNING_PASSPHRASE").context("Missing signing passphrase")?; + ensure!( + !private.is_empty() && !passphrase.is_empty() && !passphrase.contains(['\r', '\n']), + "Expected encrypted subkey and single-line passphrase" + ); + let public = PublicKey::new(config, name)?; + let password = public.home.join("passphrase"); + let mut output = fs::OpenOptions::new() + .create_new(true) + .write(true) + .mode(0o600) + .open(&password)?; + writeln!(output, "{passphrase}")?; + drop(output); + public.gpg(["--import"], Some(private.as_bytes()))?; + let keys = fingerprints( + &public.gpg(["--with-colons", "--list-secret-keys"], None)?, + "sec", + "ssb", + )?; + ensure!( + keys.len() == 2, + "Expected exactly one primary stub and one signing subkey" + ); + let (primary, pf) = &keys[0]; + let (subkey, sf) = &keys[1]; + ensure!( + *pf == config.primary_fingerprint + && primary[0] == "sec" + && primary.get(14).map(String::as_str) == Some("#"), + "Usable primary secret key is forbidden in CI" + ); + ensure!( + *sf == config.signing_fingerprint + && subkey[0] == "ssb" + && subkey.get(14).map(String::as_str) == Some("+") + && subkey.get(11).is_some_and(|s| s.contains('s')), + "Wrong or unavailable signing subkey" + ); + for (parts, _) in &keys { + ensure!( + !matches!(parts.get(1).map(String::as_str), Some("r" | "e" | "d")) + && parts.get(6).context("Missing key expiry")?.parse::()? + > Utc::now().timestamp(), + "Expired or revoked signing identity" + ); + } + let challenge = public.home.join("challenge"); + fs::write(&challenge, b"Vector repository signing capability check\n")?; + run( + command("gpgconf") + .args(["--kill", "gpg-agent"]) + .env("GNUPGHOME", &public.home), + None, + )?; + let wrong = run( + public + .gpg_command() + .args([ + "--pinentry-mode", + "loopback", + "--passphrase-fd", + "0", + "--local-user", + ]) + .arg(format!("{}!", config.signing_fingerprint)) + .arg("--output") + .arg(public.home.join("wrong-passphrase.asc")) + .arg("--detach-sign") + .arg(&challenge), + Some( + format!( + "{}{}\n", + uuid::Uuid::new_v4().simple(), + uuid::Uuid::new_v4().simple() + ) + .as_bytes(), + ), + ); + ensure!( + wrong.is_err(), + "Signing subkey must be passphrase-protected" + ); + let rpmdb = public.rpmdb()?; + let signer = Self { + public, + password, + rpmdb, + }; + signer.sign(&challenge, false, &challenge.with_extension("asc"))?; + signer + .public + .verify(&challenge.with_extension("asc"), Some(&challenge))?; + Ok(signer) + } + pub fn sign(&self, source: &Path, clear: bool, destination: &Path) -> Result<()> { + run( + self.public + .gpg_command() + .args(["--yes", "--pinentry-mode", "loopback", "--passphrase-file"]) + .arg(&self.password) + .arg("--local-user") + .arg(format!("{}!", self.public.config.signing_fingerprint)) + .args(["--digest-algo", "SHA256", "--armor", "--output"]) + .arg(destination) + .arg(if clear { + "--clearsign" + } else { + "--detach-sign" + }) + .arg(source), + None, + )?; + Ok(()) + } + pub fn rpm(&self, package: &Path) -> Result<()> { + run( + command("rpmsign") + .arg("--define") + .arg(format!( + "_gpg_name {}!", + self.public.config.signing_fingerprint + )) + .arg("--define") + .arg(format!("_gpg_path {}", self.public.home.display())) + .args([ + "--define", + "__gpg /usr/bin/gpg", + "--define", + "_gpg_digest_algo sha256", + "--define", + ]) + .arg(format!( + "_gpg_sign_cmd_extra_args --pinentry-mode loopback --passphrase-file {}", + self.password.display() + )) + .arg("--addsign") + .arg(package) + .env("GNUPGHOME", &self.public.home), + None, + )?; + verify_rpm(&self.rpmdb, package) + } +} +pub(super) fn verify_rpm(database: &Path, package: &Path) -> Result<()> { + ensure!( + text( + command("rpm") + .arg("--dbpath") + .arg(database) + .arg("--checksig") + .arg(package) + )? + .contains("digests signatures OK"), + "RPM signature verification failed" + ); + Ok(()) +} +fn child<'a>(node: Node<'a, 'a>, tag: &str) -> Result> { + node.children() + .find(|n| n.is_element() && n.tag_name().name() == tag) + .with_context(|| format!("Missing XML {tag}")) +} +fn location<'a>(node: Node<'a, 'a>) -> Result<&'a str> { + let location = child(node, "location")?; + ensure!( + location.attributes().len() == 1, + "RPM locations must be relative" + ); + let name = location.attribute("href").context("Missing href")?; + safe_relative(&format!("yum/metadata/{name}"))?; + Ok(name) +} +pub(super) fn repomd_locations(root: &Path) -> Result> { + let data = fs::read_to_string(root.join("repodata/repomd.xml"))?; + let document = Document::parse(&data)?; + let mut result = Vec::new(); + for entry in document + .root_element() + .children() + .filter(|n| n.has_tag_name(("http://linux.duke.edu/metadata/repo", "data"))) + { + let name = location(entry)?; + ensure!( + name.starts_with("repodata/") && name.split('/').count() == 2, + "Invalid RPM metadata location" + ); + result.push(name.to_owned()); + } + ensure!(!result.is_empty(), "Empty RPM root metadata"); + Ok(result) +} +pub(super) fn check_apt_index(data: &[u8], snapshot: &Snapshot) -> Result<()> { + for paragraph in std::str::from_utf8(data)? + .trim() + .split("\n\n") + .filter(|p| !p.is_empty()) + { + let item = fields(paragraph)?; + ensure!( + item.get("Package").map(String::as_str) == Some("vector"), + "Foreign package in apt history" + ); + let name = format!( + "apt/{}", + item.get("Filename").context("Missing package filename")? + ); + safe_relative(&name)?; + let entry = Entry { + sha256: item + .get("SHA256") + .context("Missing package digest")? + .clone(), + size: item.get("Size").context("Missing package size")?.parse()?, + }; + ensure!( + snapshot.entries.get(&name) == Some(&entry), + "Debian history differs from manifest" + ); + } + Ok(()) +} +#[allow(clippy::too_many_lines)] // Keep the signed-root/index/payload checks together. +pub(super) fn verify_metadata( + snapshot: &Snapshot, + root: &Path, + public: &PublicKey<'_>, +) -> Result<()> { + for name in snapshot.entries.keys() { + if name.ends_with("/InRelease") { + let suite = root + .join(name) + .parent() + .context("Missing suite")? + .to_owned(); + public.verify(&suite.join("InRelease"), None)?; + public.verify(&suite.join("Release.gpg"), Some(&suite.join("Release")))?; + let release = fs::read(suite.join("Release"))?; + ensure!( + public.gpg( + [OsStr::new("--decrypt"), suite.join("InRelease").as_os_str()], + None + )? == release, + "Inline and detached roots disagree" + ); + let mut sha256 = false; + let mut checked = BTreeSet::new(); + for line in std::str::from_utf8(&release)?.lines() { + if !line.starts_with(' ') { + sha256 = line == "SHA256:"; + } else if sha256 { + let pieces: Vec<_> = line.split_whitespace().collect(); + ensure!(pieces.len() == 3, "Malformed Release checksum"); + let relative = pieces[2]; + safe_relative(&format!("apt/index/{relative}"))?; + let entry = Entry { + sha256: pieces[0].to_owned(), + size: pieces[1].parse()?, + }; + let path = suite.join(relative); + let filename = path + .strip_prefix(root)? + .to_str() + .context("Invalid index path")?; + ensure!( + snapshot.entries.get(filename) == Some(&entry), + "Apt index differs from signed root" + ); + if relative.ends_with("/Packages") { + let data = fs::read(path)?; + ensure!(Entry::bytes(&data) == entry, "Apt checksum mismatch"); + check_apt_index(&data, snapshot)?; + checked.insert(relative); + } + } + } + ensure!(!checked.is_empty(), "Apt root has no SHA-256 indexes"); + } else if name.ends_with("/repomd.xml") { + let path = root.join(name); + let package_root = path + .parent() + .and_then(Path::parent) + .context("Invalid RPM root")?; + public.verify( + &PathBuf::from(format!("{}.asc", path.display())), + Some(&path), + )?; + let data = fs::read_to_string(&path)?; + let document = Document::parse(&data)?; + for item in document + .root_element() + .children() + .filter(|n| n.is_element() && n.tag_name().name() == "data") + { + let path = package_root.join(location(item)?); + let checksum = child(item, "checksum")?; + let data = fs::read(&path)?; + let algorithm = checksum.attribute("type").context("Missing hash type")?; + ensure!( + matches!(algorithm, "sha256" | "sha512"), + "RPM metadata needs SHA-2" + ); + ensure!( + snapshot + .entries + .get(path.strip_prefix(root)?.to_str().context("Invalid path")?) + == Some(&Entry::bytes(&data)) + && hash(&data, algorithm) == checksum.text().context("Missing hash")?, + "RPM metadata differs from signed root" + ); + if item.attribute("type") == Some("primary") { + ensure!( + path.extension() == Some(OsStr::new("gz")), + "Expected gzip primary metadata" + ); + let mut expanded = String::new(); + super::model::gzip(data.as_slice(), 512 * 1024 * 1024 + 1) + .read_to_string(&mut expanded)?; + ensure!( + expanded.len() <= 512 * 1024 * 1024, + "RPM primary metadata exceeds extraction budget" + ); + let primary = Document::parse(&expanded)?; + for package in primary + .root_element() + .children() + .filter(|n| n.is_element() && n.tag_name().name() == "package") + { + ensure!( + child(package, "name")?.text() == Some("vector"), + "Foreign RPM package" + ); + let path = package_root.join(location(package)?); + let checksum = child(package, "checksum")?; + ensure!( + checksum.attribute("type") == Some("sha256"), + "RPM package needs SHA-256" + ); + let entry = Entry { + sha256: checksum.text().context("Missing checksum")?.to_owned(), + size: child(package, "size")? + .attribute("package") + .context("Missing package size")? + .parse()?, + }; + ensure!( + snapshot + .entries + .get(path.strip_prefix(root)?.to_str().context("Invalid path")?) + == Some(&entry), + "RPM package differs from signed index" + ); + } + } + } + } + } + Ok(()) +} +fn hash(data: &[u8], algorithm: &str) -> String { + if algorithm == "sha256" { + digest(data) + } else { + hex::encode(Sha512::digest(data)) + } +} +pub(super) fn release_date(data: &[u8]) -> Result> { + let metadata = fields(std::str::from_utf8(data)?)?; + ensure!( + !metadata.contains_key("Valid-Until"), + "Repository expiry requires a separate policy" + ); + Ok( + DateTime::parse_from_rfc2822(metadata.get("Date").context("Missing repository date")?)? + .with_timezone(&Utc), + ) +} +pub(super) fn next_release_date(previous: &[Vec], now: DateTime) -> Result { + let now = now.with_nanosecond(0).context("Invalid clock")?; + let mut date = now; + for data in previous { + date = date.max(release_date(data)? + chrono::Duration::seconds(1)); + } + ensure!( + date <= now + chrono::Duration::seconds(5), + "Repository date too far ahead; check clock and retry later" + ); + Ok(date.to_rfc2822()) +} +pub(super) fn dated_release(content: &[u8], date: &str) -> Result> { + release_date(content)?; + Ok(std::str::from_utf8(content)? + .split_inclusive('\n') + .map(|line| { + if line.starts_with("Date:") { + format!("Date: {date}\n") + } else { + line.to_owned() + } + }) + .collect::() + .into_bytes()) +} +fn apt_indexes( + root: &Path, + channel: &str, + incoming: &BTreeMap>, + previous: &Path, + signer: &Signer<'_>, +) -> Result<()> { + let suite = root.join("apt/dists").join(channel); + let architectures = ["amd64", "arm64", "armhf", "x86_64", "all", "i386"]; + debug_assert_eq!(&architectures[..3], &DEB_ARCHES); + for arch in architectures { + let relative = format!("apt/dists/{channel}/vector-0/binary-{arch}/Packages"); + let old = previous.join(&relative); + let data = merge_packages( + &if old.exists() { + fs::read(old)? + } else { + Vec::new() + }, + incoming + .get(if arch == "x86_64" { "amd64" } else { arch }) + .map_or(&[], Vec::as_slice), + )?; + let index = root.join(relative); + write(&index, &data)?; + let mut compressed = GzBuilder::new() + .mtime(0) + .write(Vec::new(), Compression::best()); + compressed.write_all(&data)?; + write(&index.with_extension("gz"), &compressed.finish()?)?; + for path in [&index, &index.with_extension("gz")] { + let content = fs::read(path)?; + for algorithm in ["sha256", "sha512"] { + write( + &path + .parent() + .context("Missing index parent")? + .join("by-hash") + .join(algorithm.to_uppercase()) + .join(hash(&content, algorithm)), + &content, + )?; + } + } + } + let prior = previous.join(format!("apt/dists/{channel}/Release")); + let timestamp = next_release_date( + &if prior.exists() { + vec![fs::read(prior)?] + } else { + Vec::new() + }, + Utc::now(), + )?; + // Keep the existing repository identity to avoid an additional apt approval. + let mut release = format!( + "Suite:\nCodename: {channel}\nDate: {timestamp}\nArchitectures: {}\nComponents: vector-0\nAcquire-By-Hash: yes\n", + architectures.join(" ") + ); + for algorithm in ["sha256", "sha512"] { + writeln!(release, "{}:", algorithm.to_uppercase())?; + for arch in architectures { + for filename in ["Packages", "Packages.gz"] { + let relative = format!("vector-0/binary-{arch}/{filename}"); + let data = fs::read(suite.join(&relative))?; + writeln!( + release, + " {} {} {relative}", + hash(&data, algorithm), + data.len() + )?; + } + } + } + write(&suite.join("Release"), release.as_bytes())?; + signer.sign(&suite.join("Release"), true, &suite.join("InRelease"))?; + signer.sign(&suite.join("Release"), false, &suite.join("Release.gpg"))?; + signer.public.verify(&suite.join("InRelease"), None)?; + signer + .public + .verify(&suite.join("Release.gpg"), Some(&suite.join("Release")))?; + Ok(()) +} +#[allow(clippy::too_many_lines)] // Ordered admission, signing and final verification phase. +pub(super) fn prepare(config: &Config, work: &Path) -> Result { + let (manifest, input_id) = verify_input(work)?; + ensure!( + !work.join("result").exists(), + "Prepared output exists; reuse it or choose another workspace" + ); + let base_data = fs::read(work.join("base.json"))?; + let previous = if base_data == b"null\n" { + None + } else { + Some(Snapshot::decode(&base_data, None)?) + }; + let base_files = work.join("base"); + for (name, path) in files(&base_files)? { + ensure!( + previous.as_ref().and_then(|s| s.entries.get(&name)) == Some(&Entry::file(&path)?), + "Base metadata changed" + ); + } + let assets = manifest["assets"].as_object().context("Missing assets")?; + for (name, identity) in assets { + if identity.get("format").is_some() { + lint_package(&work.join("input").join(name), identity)?; + } + } + { + let public = PublicKey::new(config, "verify")?; + if let Some(previous) = &previous { + verify_metadata(previous, &base_files, &public)?; + } + } + let signer = Signer::new(config, "sign")?; + let result = work.join("result"); + let root = result.join("files"); + fs::create_dir_all(&root)?; + let mut entries = previous + .as_ref() + .map_or_else(BTreeMap::new, |s| s.entries.clone()); + for (name, source) in files(&base_files)? { + copy(&source, &root.join(name))?; + } + let channel = string(&manifest, "channel")?; + let mut incoming = BTreeMap::new(); + for (name, identity) in assets { + if identity.get("format").is_none() { + continue; + } + let is_deb = identity["format"] == "deb"; + let arch = string(identity, "arch")?; + let relative = if is_deb { + format!("apt/pool/v/ve/{name}") + } else { + format!("yum/{channel}/vector-0/{arch}/{name}") + }; + if let Some(entry) = entries.get(&relative) { + ensure!( + is_deb && *entry == entry_value(identity)?, + "Package already present; reuse signed publication" + ); + } + let destination = root.join(&relative); + copy(&work.join("input").join(name), &destination)?; + if is_deb { + incoming.insert( + arch.to_owned(), + run( + command("apt-ftparchive") + .arg("packages") + .arg( + relative + .strip_prefix("apt/") + .context("Invalid Debian path")?, + ) + .current_dir(root.join("apt")), + None, + )?, + ); + } else { + let cached = work.join("reuse").join(name); + if cached.exists() { + copy(&cached, &destination)?; + lint_package(&destination, identity)?; + verify_rpm(&signer.rpmdb, &destination)?; + } else { + signer.rpm(&destination)?; + } + } + } + let empty = BTreeMap::new(); + for selected in ["stable", "beta"] { + if previous.is_none() || selected == channel { + apt_indexes( + &root, + selected, + if selected == channel { + &incoming + } else { + &empty + }, + &base_files, + &signer, + )?; + for arch in RPM_ARCHES { + let relative = format!("yum/{selected}/vector-0/{arch}"); + let target = root.join(&relative); + fs::create_dir_all(&target)?; + if target.join("repodata").exists() { + fs::remove_dir_all(target.join("repodata"))?; + } + run( + command("createrepo_c") + .args([ + "--no-database", + "--checksum", + "sha256", + "--compress-type", + "gz", + ]) + .arg(&target), + None, + )?; + let old = base_files.join(relative); + if old.join("repodata/repomd.xml").exists() { + let temporary = tempfile::tempdir()?; + let uri = |path: &Path| -> Result { + Ok(reqwest::Url::from_file_path(path.canonicalize()?) + .map_err(|()| anyhow::anyhow!("Invalid file URI"))? + .to_string()) + }; + run( + command("mergerepo_c") + .args([ + "--all", + "--omit-baseurl", + "--no-database", + "--compress-type", + "gz", + "--repo", + ]) + .arg(uri(&old)?) + .arg("--repo") + .arg(uri(&target)?) + .arg("--outputdir") + .arg(temporary.path()), + None, + )?; + fs::remove_dir_all(target.join("repodata"))?; + for (name, path) in files(&temporary.path().join("repodata"))? { + copy(&path, &target.join("repodata").join(name))?; + } + } + repomd_locations(&target)?; + signer.sign( + &target.join("repodata/repomd.xml"), + false, + &target.join("repodata/repomd.xml.asc"), + )?; + signer.public.verify( + &target.join("repodata/repomd.xml.asc"), + Some(&target.join("repodata/repomd.xml")), + )?; + } + } + } + for (name, path) in files(&root)? { + let entry = Entry::file(&path)?; + ensure!( + mutable(&name) || entries.get(&name).is_none_or(|old| *old == entry), + "Immutable object changed" + ); + entries.insert(name, entry); + } + let snapshot = Snapshot::new( + entries, + previous.map(|s| s.id), + Some(input_id), + root.clone(), + )?; + let view = tempfile::tempdir()?; + for source in [&base_files, &root] { + for (name, path) in files(source)? { + if !name.ends_with(".deb") && !name.ends_with(".rpm") { + copy(&path, &view.path().join(name))?; + } + } + } + verify_metadata(&snapshot, view.path(), &signer.public)?; + write(&result.join("manifest.json"), &snapshot.data)?; + copy(&work.join("input.json"), &result.join("input.json"))?; + Snapshot::local(&root)?; + Ok(snapshot.id) +} diff --git a/vdev/src/commands/release/repositories/publisher/rollback.rs b/vdev/src/commands/release/repositories/publisher/rollback.rs new file mode 100644 index 0000000000000..c3381ebf6f52d --- /dev/null +++ b/vdev/src/commands/release/repositories/publisher/rollback.rs @@ -0,0 +1,335 @@ +//! Restore a historical selection with fresh, monotonically dated apt roots. +use super::{ + bundles::{checked_download, unpack_bundle}, + engine::{download_metadata, save_bundle}, + model::{Config, Entry, Snapshot, canonical, copy, digest, files, is_digest, mutable, write}, + repository::{ + PublicKey, Signer, dated_release, next_release_date, release_date, verify_metadata, + }, + storage::{Publisher, S3Store, Store}, +}; +use anyhow::{Context, Result, ensure}; +use chrono::Utc; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{BTreeMap, BTreeSet}, + ffi::OsStr, + fs, + path::{Path, PathBuf}, +}; +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Plan { + schema: u32, + operation: String, + pub target: String, + pub active: String, + pub pending: Option, +} +impl Plan { + pub fn decode(data: &[u8]) -> Result<(Self, String)> { + let plan: Self = serde_json::from_slice(data)?; + ensure!( + plan.schema == 1 && plan.operation == "rollback" && canonical(&plan)? == data, + "Invalid rollback request" + ); + ensure!( + is_digest(&plan.target) + && is_digest(&plan.active) + && plan.pending.as_ref().is_none_or(|s| is_digest(s)), + "Invalid rollback snapshot identity" + ); + ensure!( + plan.pending.is_none() || plan.target == plan.active, + "Abandoning pending publication must restore last completed selection" + ); + Ok((plan, digest(data))) + } + fn source_ids(&self) -> BTreeSet<&str> { + [ + Some(self.target.as_str()), + Some(self.active.as_str()), + self.pending.as_deref(), + ] + .into_iter() + .flatten() + .collect() + } +} +fn request_key(config: &Config, input: &str) -> String { + format!("{}rollback-requests/{input}.json", config.bundle_prefix) +} +fn bundle_key(config: &Config, input: &str) -> String { + format!("{}rollbacks/{input}.tar.gz", config.bundle_prefix) +} +pub(super) fn base(config: &Config, work: &Path, target: &str) -> Result<()> { + let store = S3Store::new(config)?; + let publisher = Publisher::new(&store, config); + let (state, _) = publisher.load_state()?; + ensure!(!state.held, "Rollback requires unlocked repository"); + let active = state + .active + .context("Rollback requires initialized repository")?; + let candidate = publisher.archived(state.pending.as_deref().unwrap_or(&active))?; + let mut saved = None; + if let Some(input) = &candidate.input { + let key = request_key(config, input); + if store.head(&key)?.is_some() { + let (data, _) = store.read(&key)?; + let (plan, identity) = Plan::decode(&data)?; + ensure!( + identity == *input, + "Saved rollback request checksum mismatch" + ); + if plan.target == target { + saved = Some(data); + } + } + } + let retry = saved.is_some(); + let data = saved.unwrap_or(canonical(&Plan { + schema: 1, + operation: "rollback".into(), + target: target.into(), + active, + pending: state.pending, + })?); + let (plan, input) = Plan::decode(&data)?; + write(&work.join("input.json"), &data)?; + for identity in plan.source_ids() { + let snapshot = publisher.archived(identity)?; + let directory = work.join("sources").join(identity); + ensure!(!directory.exists(), "Rollback source already exists"); + write(&directory.join("manifest.json"), &snapshot.data)?; + download_metadata( + &publisher, + &snapshot, + &directory.join("files"), + identity == plan.active || identity == plan.target, + )?; + } + let key = bundle_key(config, &input); + if store.head(&key)?.is_some() { + let archive = work.join("cached.tar.gz"); + checked_download(&store, &key, &archive, None)?; + let snapshot = unpack_bundle(&archive, &work.join("result"))?; + ensure!( + snapshot.input.as_deref() == Some(&input) + && snapshot.base.as_deref() == Some(&plan.active) + && fs::read(work.join("result/input.json"))? == data, + "Saved rollback belongs to another request" + ); + ensure!( + !retry || snapshot.id == candidate.id, + "Saved rollback differs from pending/completed generation" + ); + fs::remove_file(archive)?; + println!("Restored saved signed rollback; signing skipped"); + } else { + ensure!( + candidate.input.as_deref() != Some(&input), + "Required rollback retry bundle missing" + ); + println!("Downloaded archived selections and potentially exposed dates"); + } + Ok(()) +} +fn check_sources( + config: &Config, + work: &Path, +) -> Result<(Plan, String, BTreeMap)> { + let (plan, input) = Plan::decode(&fs::read(work.join("input.json"))?)?; + let mut sources = BTreeMap::new(); + let public = PublicKey::new(config, "rollback-sources")?; + for identity in plan.source_ids() { + let directory = work.join("sources").join(identity); + let mut snapshot = + Snapshot::decode(&fs::read(directory.join("manifest.json"))?, Some(identity))?; + let root = directory.join("files"); + let present = files(&root)?; + for (name, path) in &present { + ensure!( + !name.ends_with(".deb") + && !name.ends_with(".rpm") + && snapshot.entries.get(name) == Some(&Entry::file(path)?), + "Rollback source differs from metadata manifest" + ); + } + ensure!( + snapshot + .entries + .keys() + .filter(|n| mutable(n)) + .all(|n| present.contains_key(n)), + "Missing mutable rollback metadata" + ); + if identity == plan.active || identity == plan.target { + verify_metadata(&snapshot, &root, &public)?; + } else { + for (name, path) in &present { + if name.ends_with("/Release") { + let suite = path.parent().context("Missing suite")?; + public.verify(&suite.join("InRelease"), None)?; + public.verify(&suite.join("Release.gpg"), Some(path))?; + ensure!( + public.gpg( + [OsStr::new("--decrypt"), suite.join("InRelease").as_os_str()], + None + )? == fs::read(path)?, + "Pending apt roots disagree" + ); + } else if name.ends_with("/repomd.xml") { + public.verify( + &PathBuf::from(format!("{}.asc", path.display())), + Some(path), + )?; + } + } + } + snapshot.source = root; + sources.insert(identity.to_owned(), snapshot); + } + let selected = &sources[&plan.target]; + let names: BTreeSet<_> = selected.entries.keys().filter(|n| mutable(n)).collect(); + for source in sources.values() { + ensure!( + source + .entries + .keys() + .filter(|n| mutable(n)) + .collect::>() + == names, + "Rollback across layout changes requires a separate migration" + ); + } + let active = &sources[&plan.active]; + ensure!( + selected + .entries + .iter() + .filter(|(n, _)| !mutable(n)) + .all(|(n, e)| active.entries.get(n) == Some(e)), + "Selection absent from active immutable history" + ); + Ok((plan, input, sources)) +} +fn expected_entries(plan: &Plan, sources: &BTreeMap) -> BTreeMap { + let mut entries = sources[&plan.target].entries.clone(); + entries.extend( + sources[&plan.active] + .entries + .iter() + .filter(|(n, _)| !mutable(n)) + .map(|(n, e)| (n.clone(), e.clone())), + ); + entries +} +pub(super) fn prepare(config: &Config, work: &Path) -> Result { + let (plan, input, sources) = check_sources(config, work)?; + let selected = &sources[&plan.target]; + let root = work.join("result/files"); + ensure!( + !work.join("result").exists(), + "Prepared rollback already exists" + ); + fs::create_dir_all(&root)?; + for (name, path) in files(&selected.source)? { + copy(&path, &root.join(name))?; + } + let mut entries = expected_entries(&plan, &sources); + { + let signer = Signer::new(config, "rollback-signing")?; + for name in selected.entries.keys().filter(|n| n.ends_with("/Release")) { + let previous = sources + .values() + .map(|s| fs::read(s.source.join(name))) + .collect::>>()?; + let date = next_release_date(&previous, Utc::now())?; + let release = root.join(name); + let suite = release.parent().context("Missing suite")?; + fs::write(&release, dated_release(&fs::read(&release)?, &date)?)?; + signer.sign(&release, true, &suite.join("InRelease"))?; + signer.sign(&release, false, &suite.join("Release.gpg"))?; + } + } + for (name, path) in files(&root)? { + entries.insert(name, Entry::file(&path)?); + } + let snapshot = Snapshot::new(entries, Some(plan.active), Some(input), root)?; + write(&work.join("result/manifest.json"), &snapshot.data)?; + copy(&work.join("input.json"), &work.join("result/input.json"))?; + check_result(config, work)?; + Ok(snapshot.id) +} +pub(super) fn check_result(config: &Config, work: &Path) -> Result<(Plan, Snapshot)> { + let (plan, input, sources) = check_sources(config, work)?; + let snapshot = Snapshot::local(&work.join("result/files"))?; + ensure!( + snapshot.input.as_deref() == Some(&input) + && snapshot.base.as_deref() == Some(&plan.active) + && fs::read(work.join("result/input.json"))? == fs::read(work.join("input.json"))?, + "Signed rollback belongs to another request" + ); + let expected = expected_entries(&plan, &sources); + ensure!( + snapshot.entries.keys().eq(expected.keys()), + "Rollback changed file set" + ); + for (name, entry) in expected { + if !["/Release", "/InRelease", "/Release.gpg"] + .iter() + .any(|suffix| name.ends_with(suffix)) + { + ensure!( + snapshot.entries[&name] == entry, + "Rollback changed package bytes or indexes" + ); + } + if name.ends_with("/Release") { + let data = fs::read(snapshot.source.join(&name))?; + let timestamp = release_date(&data)?; + for source in sources.values() { + ensure!( + timestamp > release_date(&fs::read(source.source.join(&name))?)?, + "Rollback date must exceed every exposed root" + ); + } + ensure!( + timestamp <= Utc::now() + chrono::Duration::seconds(5), + "Rollback dated too far ahead" + ); + ensure!( + dated_release(&data, "DATE")? + == dated_release(&fs::read(sources[&plan.target].source.join(&name))?, "DATE")?, + "Rollback changed apt identity or indexes" + ); + } + } + let public = PublicKey::new(config, "rollback-result")?; + verify_metadata(&snapshot, &snapshot.source, &public)?; + Ok((plan, snapshot)) +} +pub(super) fn publish(config: &Config, work: &Path) -> Result<()> { + let (plan, snapshot) = check_result(config, work)?; + let store = S3Store::new(config)?; + let mut publisher = Publisher::new(&store, config); + for id in plan.source_ids() { + ensure!( + publisher.archived(id)?.data + == fs::read(work.join("sources").join(id).join("manifest.json"))?, + "Rollback source differs from saved archive" + ); + } + let input = snapshot + .input + .as_deref() + .context("Missing rollback identity")?; + save_bundle(&publisher, work, &bundle_key(config, input))?; + publisher.ensure_bytes( + &request_key(config, input), + &fs::read(work.join("input.json"))?, + )?; + publisher.publish(&snapshot, Some((&plan.active, plan.pending.as_deref())))?; + println!("Published fresh rollback snapshot {}", snapshot.id); + Ok(()) +} diff --git a/vdev/src/commands/release/repositories/publisher/storage.rs b/vdev/src/commands/release/repositories/publisher/storage.rs new file mode 100644 index 0000000000000..3fe6f673c4650 --- /dev/null +++ b/vdev/src/commands/release/repositories/publisher/storage.rs @@ -0,0 +1,684 @@ +//! Conditional S3 publication. Ambiguous writes deliberately retain the lock. +use super::model::{Config, Entry, Snapshot, canonical, is_digest, mutable, root_order}; +use anyhow::{Context, Result, ensure}; +use aws_sdk_s3::{ + Client, + config::{Credentials, Region, retry::RetryConfig, timeout::TimeoutConfig}, + primitives::ByteStream, + types::{ChecksumMode, ServerSideEncryption}, +}; +use aws_smithy_runtime_api::client::{ + interceptors::{Intercept, context::BeforeDeserializationInterceptorContextRef}, + runtime_components::RuntimeComponents, +}; +use aws_smithy_types::config_bag::ConfigBag; +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + collections::BTreeMap, + fs, + io::Write, + path::Path, + sync::{Arc, Mutex}, + time::Duration, +}; + +#[derive(Debug, Clone)] +struct ServerClock(Arc>>>); +impl Intercept for ServerClock { + fn name(&self) -> &'static str { + "PublicationServerClock" + } + fn read_before_deserialization( + &self, + context: &BeforeDeserializationInterceptorContextRef<'_>, + _: &RuntimeComponents, + _: &mut ConfigBag, + ) -> Result<(), aws_smithy_runtime_api::box_error::BoxError> { + let date = context + .response() + .headers() + .get("date") + .ok_or("AWS response has no server date")?; + *self.0.lock().map_err(|_| "Server clock poisoned")? = + Some(DateTime::parse_from_rfc2822(date)?.with_timezone(&Utc)); + Ok(()) + } +} +#[derive(Clone, Debug)] +pub(super) struct Head { + pub entry: Entry, + pub etag: String, + pub version: String, +} +pub(super) enum Body<'a> { + Bytes(&'a [u8]), + File(&'a Path), +} +pub(super) trait Store { + fn head(&self, key: &str) -> Result>; + fn read(&self, key: &str) -> Result<(Vec, Head)>; + fn heads(&self, keys: &[String]) -> Result>> { + keys.iter().map(|key| self.head(key)).collect() + } + fn put( + &self, + key: &str, + body: Body<'_>, + entry: &Entry, + previous: Option<&Head>, + immutable: bool, + ) -> Result; + fn download(&self, key: &str, path: &Path, limit: u64) -> Result; + fn server_time(&self) -> Result>; + fn owner(&self) -> &str; +} +pub(super) struct S3Store { + runtime: tokio::runtime::Runtime, + client: Client, + bucket: String, + live: String, + pub owner: String, + clock: ServerClock, +} +fn checksum(value: Option<&str>, size: Option) -> Result { + let hash = STANDARD.decode(value.context("Missing stored SHA-256 checksum")?)?; + ensure!(hash.len() == 32, "Invalid stored SHA-256 checksum"); + Ok(Entry { + sha256: hex::encode(hash), + size: u64::try_from(size.context("Missing object size")?)?, + }) +} +impl S3Store { + pub fn new(config: &Config) -> Result { + let role_arn = std::env::var("PACKAGE_REPOSITORIES_ROLE_ARN") + .context("Set the publishing role ARN")?; + ensure!( + regex::Regex::new(&format!( + r"^arn:aws:iam::{}:role/[A-Za-z0-9_+=,.@-]+$", + config.account + ))? + .is_match(&role_arn), + "Wrong publishing role ARN" + ); + // Snapshot one temporary session, with no credential-provider renewal. + let required = |name| -> Result { + let value = std::env::var(name).context("A fixed temporary AWS session is required")?; + ensure!(!value.is_empty(), "Empty AWS credential"); + Ok(value) + }; + let credentials = Credentials::new( + required("AWS_ACCESS_KEY_ID")?, + required("AWS_SECRET_ACCESS_KEY")?, + Some(required("AWS_SESSION_TOKEN")?), + None, + "fixed-publisher-session", + ); + let runtime = tokio::runtime::Runtime::new()?; + let sdk = runtime.block_on( + aws_config::defaults(aws_config::BehaviorVersion::latest()) + .region(Region::new(config.region.clone())) + .credentials_provider(credentials) + .retry_config(RetryConfig::disabled()) + .timeout_config( + TimeoutConfig::builder() + .connect_timeout(Duration::from_secs(10)) + .read_timeout(Duration::from_secs(90)) + .operation_timeout(Duration::from_mins(5)) + .build(), + ) + .load(), + ); + let identity = + runtime.block_on(aws_sdk_sts::Client::new(&sdk).get_caller_identity().send())?; + let role = role_arn.rsplit('/').next().context("Invalid role")?; + let owner = identity.arn().context("Missing caller ARN")?.to_owned(); + ensure!( + identity.account() == Some(config.account.as_str()) + && owner.starts_with(&format!( + "arn:aws:sts::{}:assumed-role/{role}/", + config.account + )), + "Wrong AWS account or role" + ); + let policy = runtime.block_on( + aws_sdk_iam::Client::new(&sdk) + .get_role() + .role_name(role) + .send(), + )?; + ensure!( + policy + .role() + .context("Missing role")? + .max_session_duration() + .is_some_and(|seconds| seconds <= 3600), + "Role exceeds the one-hour recovery bound" + ); + let clock = ServerClock(Arc::new(Mutex::new(None))); + let client = Client::from_conf( + aws_sdk_s3::config::Builder::from(&sdk) + .interceptor(clock.clone()) + .build(), + ); + Ok(Self { + runtime, + client, + bucket: config.bucket.clone(), + live: config.live_prefix.clone(), + owner, + clock, + }) + } +} +impl Store for S3Store { + fn heads(&self, keys: &[String]) -> Result>> { + if keys.is_empty() { + return Ok(Vec::new()); + } + // Join every request before returning a conflict and releasing the lock. + std::thread::scope(|scope| { + let workers: Vec<_> = keys + .chunks(keys.len().div_ceil(16)) + .map(|chunk| { + scope.spawn(move || { + chunk + .iter() + .map(|key| self.head(key)) + .collect::>>() + }) + }) + .collect(); + let completed: Vec<_> = workers + .into_iter() + .map(|worker| { + worker + .join() + .map_err(|_| anyhow::anyhow!("S3 inspection worker panicked"))? + }) + .collect(); + Ok(completed + .into_iter() + .collect::>>()? + .into_iter() + .flatten() + .collect()) + }) + } + fn owner(&self) -> &str { + &self.owner + } + fn server_time(&self) -> Result> { + self.clock + .0 + .lock() + .map_err(|_| anyhow::anyhow!("Server clock poisoned"))? + .context("No AWS server time") + } + fn head(&self, key: &str) -> Result> { + let result = self.runtime.block_on( + self.client + .head_object() + .bucket(&self.bucket) + .key(key) + .checksum_mode(ChecksumMode::Enabled) + .send(), + ); + let output = match result { + Ok(value) => value, + Err(error) => { + let status = error.raw_response().map(|r| r.status().as_u16()); + if status == Some(404) { + return Ok(None); + } + if status == Some(403) { + let listed = self.runtime.block_on( + self.client + .list_objects_v2() + .bucket(&self.bucket) + .prefix(key) + .max_keys(1) + .send(), + )?; + if listed.contents().iter().all(|o| o.key() != Some(key)) { + return Ok(None); + } + } + return Err(error.into()); + } + }; + Ok(Some(Head { + entry: checksum(output.checksum_sha256(), output.content_length())?, + etag: output.e_tag().context("Missing ETag")?.to_owned(), + version: output + .version_id() + .context("Bucket versioning is required")? + .to_owned(), + })) + } + fn read(&self, key: &str) -> Result<(Vec, Head)> { + let output = self.runtime.block_on( + self.client + .get_object() + .bucket(&self.bucket) + .key(key) + .checksum_mode(ChecksumMode::Enabled) + .send(), + )?; + let head = Head { + entry: checksum(output.checksum_sha256(), output.content_length())?, + etag: output.e_tag().context("Missing ETag")?.to_owned(), + version: output + .version_id() + .context("Bucket versioning is required")? + .to_owned(), + }; + ensure!(head.entry.size < super::model::LIMIT, "Oversized S3 object"); + let data = self + .runtime + .block_on(output.body.collect())? + .into_bytes() + .to_vec(); + ensure!( + Entry::bytes(&data) == head.entry, + "S3 read failed checksum verification" + ); + Ok((data, head)) + } + fn download(&self, key: &str, destination: &Path, limit: u64) -> Result { + ensure!(!destination.is_symlink(), "Invalid download destination"); + let output = self.runtime.block_on( + self.client + .get_object() + .bucket(&self.bucket) + .key(key) + .checksum_mode(ChecksumMode::Enabled) + .send(), + )?; + let expected = checksum(output.checksum_sha256(), output.content_length())?; + ensure!(expected.size <= limit, "S3 object grew before download"); + let mut target = fs::File::create(destination)?; + let mut digest = Sha256::new(); + let mut size = 0; + self.runtime.block_on(async { + let mut body = output.body; + while let Some(chunk) = body.try_next().await? { + size += chunk.len() as u64; + ensure!(size <= limit, "Oversized download"); + digest.update(&chunk); + target.write_all(&chunk)?; + } + Ok::<_, anyhow::Error>(()) + })?; + let actual = Entry { + sha256: hex::encode(digest.finalize()), + size, + }; + ensure!(actual == expected, "S3 body failed checksum verification"); + Ok(actual) + } + fn put( + &self, + key: &str, + body: Body<'_>, + entry: &Entry, + previous: Option<&Head>, + immutable: bool, + ) -> Result { + let body = match body { + Body::Bytes(bytes) => ByteStream::from(bytes.to_owned()), + Body::File(path) => self.runtime.block_on(ByteStream::from_path(path))?, + }; + let mut request = self + .client + .put_object() + .bucket(&self.bucket) + .key(key) + .body(body) + .content_length(i64::try_from(entry.size)?) + .checksum_sha256(STANDARD.encode(hex::decode(&entry.sha256)?)) + .server_side_encryption(ServerSideEncryption::Aes256) + .content_type(content_type(key)) + .cache_control(format!( + "{},{}", + if key.starts_with(&self.live) { + "public" + } else { + "private" + }, + if immutable { + "max-age=31536000,immutable" + } else { + "max-age=0,no-cache,must-revalidate" + } + )); + request = if let Some(head) = previous { + request.if_match(&head.etag) + } else { + request.if_none_match("*") + }; + let output = self.runtime.block_on(request.send())?; + Ok(Head { + entry: entry.clone(), + etag: output.e_tag().context("Missing ETag")?.to_owned(), + version: output + .version_id() + .context("Bucket versioning is required")? + .to_owned(), + }) + } +} +fn content_type(key: &str) -> &'static str { + if key.ends_with(".deb") { + "application/vnd.debian.binary-package" + } else if key.ends_with(".rpm") { + "application/x-rpm" + } else if key.ends_with(".json") { + "application/json" + } else if key.ends_with(".gz") { + "application/gzip" + } else if key.ends_with(".xml") { + "application/xml" + } else { + "application/octet-stream" + } +} +#[derive(Debug)] +struct Conflict(String); +impl std::fmt::Display for Conflict { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} +impl std::error::Error for Conflict {} +fn conflict(message: impl Into) -> anyhow::Error { + Conflict(message.into()).into() +} +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(super) struct State { + schema: u32, + pub held: bool, + pub active: Option, + pub pending: Option, + #[serde(skip_serializing_if = "Option::is_none", default)] + owner: Option, + #[serde(skip_serializing_if = "Option::is_none", default)] + recover_after: Option>, + #[serde(skip_serializing_if = "Option::is_none", default)] + nonce: Option, +} +pub(super) struct Publisher<'a> { + pub store: &'a dyn Store, + pub config: &'a Config, + state: Option, + head: Option, +} +impl<'a> Publisher<'a> { + pub fn new(store: &'a dyn Store, config: &'a Config) -> Self { + Self { + store, + config, + state: None, + head: None, + } + } + pub fn load_state(&self) -> Result<(State, Option)> { + if self.store.head(&self.config.control_key)?.is_none() { + return Ok(( + State { + schema: 1, + held: false, + active: None, + pending: None, + owner: None, + recover_after: None, + nonce: None, + }, + None, + )); + } + let (data, head) = self.store.read(&self.config.control_key)?; + let state: State = serde_json::from_slice(&data)?; + ensure!( + state.schema == 1 + && [&state.active, &state.pending] + .into_iter() + .flatten() + .all(|s| is_digest(s)), + "Invalid publication state" + ); + ensure!( + !state.held || (state.recover_after.is_some() && state.owner.is_some()), + "Incomplete held lock" + ); + Ok((state, Some(head))) + } + fn write_state(&mut self) -> Result<()> { + let state = self.state.as_mut().context("Lock not initialized")?; + state.nonce = Some(uuid::Uuid::new_v4().simple().to_string()); + let data = canonical(state)?; + self.head = Some(self.store.put( + &self.config.control_key, + Body::Bytes(&data), + &Entry::bytes(&data), + self.head.as_ref(), + false, + )?); + Ok(()) + } + fn acquire(&mut self) -> Result<()> { + let (mut state, head) = self.load_state()?; + ensure!( + !state.held, + "Publisher busy; locks are never stolen on timeout" + ); + state.held = true; + state.owner = Some(self.store.owner().to_owned()); + state.recover_after = Some(self.store.server_time()? + chrono::Duration::minutes(65)); + self.state = Some(state); + self.head = head; + self.write_state() + } + fn release(&mut self) -> Result<()> { + self.state.as_mut().context("No lock")?.held = false; + self.write_state() + } + pub fn recover_lock(&mut self, version: &str, owner_stopped: bool) -> Result<()> { + let (state, head) = self.load_state()?; + ensure!( + owner_stopped, + "Confirm that the previous workflow/process has stopped" + ); + ensure!( + state.held && head.as_ref().is_some_and(|h| h.version == version), + "Lock version changed or is released" + ); + ensure!( + self.store.server_time()? >= state.recover_after.context("Missing recovery time")?, + "Recovery waits 65 minutes for the previous fixed STS session to expire" + ); + self.state = Some(state); + self.head = head; + self.release() + } + pub fn ensure_bytes(&self, key: &str, data: &[u8]) -> Result<()> { + self.ensure_immutable(key, Body::Bytes(data), &Entry::bytes(data)) + } + pub fn ensure_file(&self, key: &str, path: &Path, entry: &Entry) -> Result<()> { + self.ensure_immutable(key, Body::File(path), entry) + } + fn ensure_immutable(&self, key: &str, body: Body<'_>, entry: &Entry) -> Result<()> { + if let Some(head) = self.store.head(key)? { + if head.entry != *entry { + return Err(conflict(format!("Immutable object conflict: {key}"))); + } + return Ok(()); + } + self.store.put(key, body, entry, None, true)?; + Ok(()) + } + pub fn archived(&self, id: &str) -> Result { + ensure!(is_digest(id), "Invalid snapshot ID"); + Snapshot::decode( + &self + .store + .read(&format!("{}{id}/manifest.json", self.config.archive_prefix))? + .0, + Some(id), + ) + } + fn preflight(&self, snapshot: &Snapshot) -> Result<()> { + let active = &self.state.as_ref().context("No lock")?.active; + if active.as_deref() != Some(&snapshot.id) { + if *active != snapshot.base { + return Err(conflict( + "Repository advanced after preparation; regenerate against current base", + )); + } + if let Some(id) = active { + for (name, entry) in self.archived(id)?.entries { + if !mutable(&name) && snapshot.entries.get(&name) != Some(&entry) { + return Err(conflict(format!( + "Publication would remove historical data: {name}" + ))); + } + } + } + } + let keys: Vec<_> = snapshot + .entries + .keys() + .map(|name| format!("{}{name}", self.config.live_prefix)) + .collect(); + let heads = self.store.heads(&keys)?; + for ((name, entry), old) in snapshot.entries.iter().zip(heads) { + if !mutable(name) { + if old.as_ref().is_some_and(|h| h.entry != *entry) { + return Err(conflict(format!("Immutable object conflict: {name}"))); + } + if old.is_none() && !snapshot.source.join(name).is_file() { + return Err(conflict(format!("Inherited object is missing: {name}"))); + } + } + } + Ok(()) + } + fn publish_locked( + &mut self, + snapshot: &Snapshot, + recovery: Option<(&str, Option<&str>)>, + ) -> Result<()> { + let state = self.state.as_ref().context("No lock")?; + if let Some((active, pending)) = recovery { + let pair = (state.active.as_deref(), state.pending.as_deref()); + if snapshot.base.as_deref() != Some(active) + || ![ + (Some(active), pending), + (Some(active), Some(snapshot.id.as_str())), + (Some(snapshot.id.as_str()), None), + ] + .contains(&pair) + { + return Err(conflict("Repository changed after rollback preparation")); + } + } else if state.pending.as_ref().is_some_and(|p| p != &snapshot.id) { + return Err(conflict("Recover pending publication or roll back")); + } + self.preflight(snapshot)?; + let archive = format!("{}{}/", self.config.archive_prefix, snapshot.id); + for (name, entry) in &snapshot.entries { + if mutable(name) { + self.ensure_file( + &format!("{archive}files/{name}"), + &snapshot.source.join(name), + entry, + )?; + } + } + self.ensure_bytes(&format!("{archive}manifest.json"), &snapshot.data)?; + let mut metadata = BTreeMap::new(); + for (name, entry) in &snapshot.entries { + if mutable(name) { + let data = self.store.read(&format!("{archive}files/{name}"))?.0; + if Entry::bytes(&data) != *entry { + return Err(conflict(format!( + "Archived metadata checksum mismatch: {name}" + ))); + } + metadata.insert(name, data); + } + } + self.state.as_mut().context("No lock")?.pending = Some(snapshot.id.clone()); + self.write_state()?; + for (name, entry) in &snapshot.entries { + if !mutable(name) && snapshot.source.join(name).exists() { + self.ensure_file( + &format!("{}{name}", self.config.live_prefix), + &snapshot.source.join(name), + entry, + )?; + } + } + let mut names: Vec<_> = metadata.keys().copied().collect(); + names.sort_by_key(|name| { + ( + if name.ends_with("/Packages") || name.ends_with("/Packages.gz") { + 0 + } else { + 1 + root_order(name) + }, + *name, + ) + }); + for name in names { + let key = format!("{}{name}", self.config.live_prefix); + let entry = &snapshot.entries[name]; + let previous = self.store.head(&key)?; + if previous.as_ref().is_some_and(|h| h.entry == *entry) { + continue; + } + self.store.put( + &key, + Body::Bytes(&metadata[name]), + entry, + previous.as_ref(), + false, + )?; + } + let keys: Vec<_> = snapshot + .entries + .keys() + .map(|name| format!("{}{name}", self.config.live_prefix)) + .collect(); + for ((name, entry), head) in snapshot.entries.iter().zip(self.store.heads(&keys)?) { + ensure!( + head.is_some_and(|h| h.entry == *entry), + "Published object failed final verification: {name}" + ); + } + let state = self.state.as_mut().context("No lock")?; + state.active = Some(snapshot.id.clone()); + state.pending = None; + self.write_state() + } + pub fn publish( + &mut self, + snapshot: &Snapshot, + recovery: Option<(&str, Option<&str>)>, + ) -> Result<()> { + self.acquire()?; + match self.publish_locked(snapshot, recovery) { + Ok(()) => self.release(), + Err(error) if error.is::() => { + self.release()?; + Err(error) + } + Err(error) => { + Err(error.context("Publication stopped; lock remains held for operator recovery")) + } + } + } +} From a292175e937cb42969394ab829d703e9423d59bc Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 20 Oct 2026 17:37:16 -0400 Subject: [PATCH 08/23] fix(releasing): reuse signed bundles across archive encodings --- .../release/repositories/publisher/engine.rs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/vdev/src/commands/release/repositories/publisher/engine.rs b/vdev/src/commands/release/repositories/publisher/engine.rs index 924c1a10c9cad..8f4c9e7a5b41c 100644 --- a/vdev/src/commands/release/repositories/publisher/engine.rs +++ b/vdev/src/commands/release/repositories/publisher/engine.rs @@ -193,6 +193,29 @@ pub(super) fn save_bundle(publisher: &Publisher<'_>, work: &Path, key: &str) -> entry.size < LIMIT, "Prepared bundle exceeds single-PUT limit" ); + if let Some(saved) = publisher.store.head(key)? { + if saved.entry == entry { + return Ok(()); + } + // Compression or tar encodings can change between tool versions. Keep + // the saved archive when every signed file and manifest is identical. + let temporary = tempfile::tempdir_in(work)?; + let saved_archive = temporary.path().join("saved.tar.gz"); + checked_download(publisher.store, key, &saved_archive, Some(&saved.entry))?; + let saved_result = temporary.path().join("result"); + unpack_bundle(&saved_archive, &saved_result)?; + let contents = |root: &Path| -> Result> { + files(root)? + .into_iter() + .map(|(name, path)| Ok((name, Entry::file(&path)?))) + .collect() + }; + ensure!( + contents(&saved_result)? == contents(&work.join("result"))?, + "Saved signed bundle contains different files" + ); + return Ok(()); + } publisher.ensure_file(key, &archive, &entry) } pub(super) fn publish(config: &Config, work: &Path) -> Result<()> { From 92355eb983bd0e17d50ca29cff067eec2fca41f4 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 21 Oct 2026 08:40:12 -0400 Subject: [PATCH 09/23] docs(releasing): explain package repository jobs inline --- .../package-repositories-maintenance.yml | 8 + .github/workflows/package-repositories.yml | 10 + docs/README.md | 3 - docs/reference/package-repositories.md | 254 ------------------ 4 files changed, 18 insertions(+), 257 deletions(-) delete mode 100644 docs/reference/package-repositories.md diff --git a/.github/workflows/package-repositories-maintenance.yml b/.github/workflows/package-repositories-maintenance.yml index ce2f726e83597..9a8207ea78854 100644 --- a/.github/workflows/package-repositories-maintenance.yml +++ b/.github/workflows/package-repositories-maintenance.yml @@ -26,6 +26,9 @@ concurrency: cancel-in-progress: false jobs: + # Inspect S3 state, prepare a rollback, or recover a lock with the publishing role. + # Available even when automated publication is disabled. Recovery requires the exact + # control version, a stopped owner and an expired 65-minute recovery window. maintain: if: github.repository_id == '146327667' && github.ref == 'refs/heads/master' runs-on: ubuntu-24.04 @@ -109,6 +112,8 @@ jobs: retention-days: 1 if-no-files-found: error + # Refresh apt dates/signatures so cached clients accept the archived package selection. + # Reuse saved rollback output on retries; the signing job has no AWS credentials. sign: needs: maintain if: inputs.operation == 'rollback' @@ -146,6 +151,8 @@ jobs: retention-days: 1 if-no-files-found: error + # Restore the selected repository state under the shared S3 lock, rejecting stale plans. + # Uses AWS OIDC without the signing key; installed clients are not downgraded. publish: needs: sign runs-on: ubuntu-24.04 @@ -184,6 +191,7 @@ jobs: echo '```' } >> "$GITHUB_STEP_SUMMARY" + # Verify the restored indexes and fresh signed roots through public HTTPS. endpoints: needs: publish runs-on: ubuntu-24.04 diff --git a/.github/workflows/package-repositories.yml b/.github/workflows/package-repositories.yml index 36cc62f4c4a94..efe2d9b4818b7 100644 --- a/.github/workflows/package-repositories.yml +++ b/.github/workflows/package-repositories.yml @@ -28,6 +28,8 @@ concurrency: cancel-in-progress: false jobs: + # Freeze verified release assets and build the tools reused by every job. + # Enable only after historical backfill and the repository/key migration are ready. inputs: if: >- github.repository_id == '146327667' && @@ -82,6 +84,8 @@ jobs: retention-days: 1 if-no-files-found: error + # Read current S3 metadata or restore the exact signed output of a retry. + # Uses AWS OIDC without access to the signing key. base: needs: inputs runs-on: ubuntu-24.04 @@ -117,6 +121,8 @@ jobs: retention-days: 1 if-no-files-found: error + # Sign new packages and repository metadata; retries reuse saved signatures. + # The protected signing job has no AWS credentials. sign: needs: [inputs, base] runs-on: ubuntu-24.04 @@ -153,6 +159,8 @@ jobs: retention-days: 1 if-no-files-found: error + # Upload immutable packages/indexes before switching signed repository roots. + # Serialize writes with the S3 control lock; this job has no signing key. publish: needs: [inputs, sign] runs-on: ubuntu-24.04 @@ -182,6 +190,7 @@ jobs: PACKAGE_REPOSITORIES_ROLE_ARN: ${{ vars.PACKAGE_REPOSITORIES_ROLE_ARN }} run: '"$VDEV" release repositories publish' + # Check public HTTPS metadata and package bytes against the signed snapshot. endpoints: needs: [inputs, publish] runs-on: ubuntu-24.04 @@ -201,6 +210,7 @@ jobs: - name: Verify public HTTPS metadata and new package bytes run: '"$VDEV" release repositories verify' + # Install the selected version through apt/DNF on amd64 and arm64 using pinned trust. clients: needs: [inputs, endpoints] runs-on: ${{ matrix.runner }} diff --git a/docs/README.md b/docs/README.md index 429ff8a57d8fe..443c706e38faa 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,9 +22,6 @@ contributors: - **[REVIEWING.md](REVIEWING.md)** - Code review expectations and guidelines. - **[USER_EXPERIENCE_DESIGN.md](USER_EXPERIENCE_DESIGN.md)** - User experience principles and guidelines. -- **[Package repository publishing](reference/package-repositories.md)** - COSE - signing, publication and recovery commands; activation requires the migration - cutover. ## Project policies diff --git a/docs/reference/package-repositories.md b/docs/reference/package-repositories.md deleted file mode 100644 index b64e8e3d9c8b9..0000000000000 --- a/docs/reference/package-repositories.md +++ /dev/null @@ -1,254 +0,0 @@ -# Package repository publishing - -The `Package Repositories` workflow consumes existing GitHub release assets and -publishes apt and yum repositories. `cargo vdev release repositories` provides the -maintainer commands and the Rust publisher implementation in -`vdev/src/commands/release/repositories/publisher`. The tools image is built within -each workflow run from a pinned Rust builder and Ubuntu runtime. It contains -`vdev`, GPG and native Debian/RPM metadata tools; Python and pip are not required. -The `package-repositories` Cargo feature keeps AWS SDK dependencies out of ordinary -vdev builds. `distribution/repositories` holds the image definition, production -configuration, public key and standalone shell installer. - -This workflow is disabled unless the repository variable -`PACKAGE_REPOSITORIES_ENABLED` is `true`. It requires a completed historical -backfill. Enabling it is part of the repository migration cutover, after the new -public key, installation instructions and client migration have been published. -The normal Test Suite checks the vdev CLI and builds the publisher image when -this tooling changes. Those checks require no signing secrets or AWS credentials. -Publisher regressions and migration rehearsals live in the private CI sandbox. - -## Inputs and job boundaries - -Automatic publication follows a successful `Release Suite` run for a final -`v0.x.y` tag. Manual dispatch requires that run's ID, the same immutable tag, and -an explicit stable or beta channel. Stable rejects prereleases. The input job -checks the repository ID, workflow path, tag push, commit, release state, asset -set, asset digests and SHA256SUMS. It records the six packages in an immutable -manifest. Debian `armel` remains a raw release download, outside these indexes. - -The workflow checks out its default-branch code, not the release tag. It builds -one tools image and vdev binary and transfers those exact tools between jobs. -Signing has no AWS credentials. Publishing has no private signing key. Client -verification has neither. Package installation happens only in client jobs. - -The following protected environments must exist before activation. Restrict both -to `master`, require the team's release reviewers, and prevent self-approval. - -| Environment | Configuration | -| --- | --- | -| `package-repositories-signing` | `PACKAGE_SIGNING_SUBKEY` and `PACKAGE_SIGNING_PASSPHRASE` secrets | -| `package-repositories-publishing` | `PACKAGE_REPOSITORIES_ROLE_ARN` variable, assumed with GitHub OIDC | - -Import only the encrypted signing-subkey export from the production custody -record. The primary secret key and revocation certificate do not belong in CI. -The code rejects an available primary secret key, an unexpected subkey, expired -keys, and an unprotected signing subkey. Temporary keyrings and the passphrase -file exist only in container tmpfs and are excluded from artifacts. - -The public identity is pinned in `distribution/repositories/production.json`: - -- Primary: `FBC83025C7F26E39DBCCDCD9ED518768DAA61968`. -- Signing subkey: `E49714E41E4A57313AACFC333C9F4A9B1ACBAD72`. - -## Storage and publication - -The production configuration targets `dd-cose-releases` in account -`486234852809`, region `us-east-1`. Live content uses -`vector/repositories/{apt,yum}/`. Internal state, signed retry bundles and -metadata archives use `package-repositories/`, outside the publicly readable -`vector/` prefix. Hosting must map the apt/yum domain roots to the corresponding -live prefixes. - -The role needs only scoped S3 listing, reads, conditional writes and access to its -own IAM role metadata. Keep bucket versioning, TLS enforcement and encryption -enabled. Deny object deletion and changes outside these prefixes. The OIDC trust -must bind the repository's immutable ID and the publishing environment. The -role's maximum session duration must be at most one hour; workflow jobs request -15-minute sessions. Credentials are fixed for a publisher invocation and never -refreshed while holding a lock. - -Each generation records all live paths, sizes and SHA-256 digests. Preparation -downloads current metadata and new packages; historical package payloads stay -in S3. Apt indexes are merged by native package identity. RPM indexes are merged -with all versions retained. Previous apt SHA-256/SHA-512 by-hash objects and yum -metadata remain available. An outdated preparation or a conflicting immutable -object fails before live metadata changes. - -Signed RPMs are saved by the source artifact digest. Signed publication bundles -are saved by input and base-generation digest before live publication. Retries -restore those bytes and skip signing; rebasing against newer metadata can reuse -the saved RPM signatures. Debian package bytes are unchanged. - -Publication archives signed metadata, writes immutable packages and indexes, -then replaces canonical indexes and roots. All writers use one conditional S3 -control object, including stable, beta, backfill and recovery. Control objects -must have no lifecycle expiration. Retain internal bundles and immutable content -for as long as releases can be retried or rolled back. - -The RPM root and detached signature are separate objects. Clients can observe a -temporary mismatch and must fail signature verification during that interval. -Origin writes are not an atomic CDN update. Mutable objects use revalidation -headers; the actual CDN caching and retry behavior must be validated before -cutover. Workflow success also requires public HTTPS checksum verification and -native apt/DNF installation on amd64 and arm64. - -## Client setup and migration - -`distribution/repositories/setup.sh` is the standalone apt/yum setup script. -It configures a repository without installing or upgrading Vector. A new source -defaults to stable and COSE-only trust. An existing `vector.list` or `vector.repo` -keeps its URL, channel, architecture restrictions and other options while adding -COSE trust. `--migrate` requires an existing managed source. - -A pre-cutover migration retains the old public keys in Vector's own key file. -It leaves shared Datadog keyrings and RPM database entries unchanged. On RPM -clients, old remote public keys are copied into the local Vector key file so -subsequent refreshes no longer need Datadog's key endpoint. Both `gpgcheck` and -`repo_gpgcheck` are required, including on custom mirrors. The old RHEL/CentOS -8.1 exception is not retained. - -Only after hosting has switched to COSE, `--migrate --cose-only` can remove legacy -trust from Vector's own configuration. This mode also supports late migration -when the old key files or endpoints are unavailable. It never deletes shared -Datadog key files, RPM keys or keys cached by DNF. - -The default public key URL is -`https://dd-cose-releases.s3.us-east-1.amazonaws.com/vector/keys/vector-packages.asc`. -The script verifies its full primary fingerprint before changing managed files. -It uses HTTPS with certificate verification for key downloads, backs up original -managed files with `.pre-cose`, installs replacements atomically, and refreshes -only the Vector repository. A failed final refresh leaves the saved configuration -available for inspection and retry. The script rejects ambiguous source layouts, -symlinked managed files, unavailable legacy trust and unsupported overrides -before replacing source files. Custom filenames and deb822 sources require -manual migration. - -Fresh setup retains `VECTOR_DIST_CHANNEL`, `VECTOR_MAJOR_VERSION`, -`VEC_REPO_URL`, `TESTING_APT_URL`, `TESTING_YUM_URL`, -`TESTING_APT_REPO_VERSION` and `TESTING_YUM_VERSION_PATH`. -`TESTING_KEYS_URL` is a compatibility alias for the HTTPS directory containing -`vector-packages.asc`. An owned staging mirror can supply -`VECTOR_PUBLIC_KEY_URL` and an explicit full -`VECTOR_PUBLIC_KEY_FINGERPRINT`. These are public trust settings, not secrets. -Existing sources reject fresh-setup overrides rather than switching their -channel on a repeated invocation. `CSM_MIGRATE=true` disables the known -`timber-vector` file with a backup. - -The publisher keeps the existing apt identity: no `Origin` or `Label`, an empty -`Suite`, and the channel as `Codename`. This permits the signing-key and hosting -transition without a separate release-information approval. The observed -legacy fields and the regression check comparing them with publisher output -are kept in the private migration test suite. - -Before announcement, deploy the reviewed script at -`vector/setup.sh` and the public key at `vector/keys/vector-packages.asc` -in the COSE bucket, then publish the migration guide. These paths are separate -from the package publisher's scope and need an appropriately scoped deployment. -Keep the live `setup.vector.dev` redirect and normal installation pages unchanged -until the repository hosting cutover. At cutover, point `setup.vector.dev` at the -new script and update those installation pages together. - -The protected production client job runs this setup script before installing a -release. Activation therefore requires the public key endpoint to be available. -Private sandbox tests use separate ephemeral legacy and COSE signing identities -for disposable HTTPS clients. Private keys stay in tmpfs; client bundles contain -only public keys and signed packages. - -## Maintainer commands - -Build the tools once from reviewed source: - -```sh -cargo vdev release repositories build-image -``` - -The default workspace is `target/package-repositories`. Use a different `--work` -directory for each release. A configuration override is intended for isolated -staging; it must include the corresponding pinned public key. - -```sh -cargo vdev release repositories --work target/repository-v0.60.0 fetch \ - --run 39289726838 --tag v0.60.0 --channel stable -cargo vdev release repositories --work target/repository-v0.60.0 base -cargo vdev release repositories --work target/repository-v0.60.0 prepare -cargo vdev release repositories --work target/repository-v0.60.0 publish -cargo vdev release repositories --work target/repository-v0.60.0 verify -``` - -`fetch` accepts `GH_TOKEN` for GitHub API access. AWS commands require fixed -`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN`, and -`PACKAGE_REPOSITORIES_ROLE_ARN`. `prepare` requires the two signing secrets only -when `base` did not restore a signed result. Never put secret values in command -arguments, a tracked configuration file or a release artifact. - -For normal retries, dispatch the workflow with the original run, tag and channel. -Do not rebuild packages or move the tag. An older already-published release must -not replace the current repository generation; use an explicit rollback when -that is the intended operation. - -## Recovery - -Use the manually dispatched `Package Repository Maintenance` workflow on `master` -for `status`, `rollback` or `recover-lock`. It shares the release workflow's -concurrency group. Status and lock recovery use only the protected publishing -environment and its OIDC role. Rollback downloads archives there, signs in the -protected signing environment, then publishes and verifies public HTTPS in -separate jobs. Signing never receives AWS credentials; publishing never receives -the private key. - -Maintenance remains available when `PACKAGE_REPOSITORIES_ENABLED` is disabled. -The protected environments and role must be configured before it can run. -Rollback requires an exact archived snapshot digest. Lock recovery also requires -the exact control-object version and confirmation that the old owner stopped. -A successful operation records the resulting control state in its summary. - -Inspect the state before acting: - -```sh -cargo vdev release repositories status -``` - -An unexpected error retains the lock because a timed-out S3 write may have -completed. Locks are never stolen automatically. Stop the previous workflow or -process and ensure its in-flight writes have quiesced. Credential expiry alone -does not cancel requests already accepted by S3. After the 65-minute safety -window, recover using the exact inspected S3 control-object version: - -```sh -cargo vdev release repositories recover-lock \ - --expected-version VERSION_ID --confirm-owner-stopped -``` - -Resume the pending signed publication, or select an archived package set. When -abandoning an incomplete publication, the selection must be the last completed -snapshot. The maintenance workflow performs these phases with separate -credentials; the equivalent CLI sequence uses a fresh workspace: - -```sh -cargo vdev release repositories --work target/repository-rollback rollback base SNAPSHOT_SHA256 -cargo vdev release repositories --work target/repository-rollback rollback prepare -cargo vdev release repositories --work target/repository-rollback rollback publish -cargo vdev release repositories --work target/repository-rollback rollback verify -``` - -Skip `prepare` when `base` restores a saved signed result. Rollback signs a new apt -Release date, newer than the selected, active and any pending roots. It preserves -the selected indexes and repository identity, reuses the selected yum signatures, -and retains all completed immutable history. Publication rejects a changed base -or pending state. Interrupted and completed retries restore the same signed -bundle without signing again or creating extra live-object versions. - -Replaying old apt signatures alone is insufficient: clients can keep their newer -cached indexes when the restored Release date is older, as shown in -[apt's metadata verification](https://sources.debian.org/src/apt/2.2.4/apt-pkg/acquire-item.cc/). -Neither cache deletion nor disabling date/signature checks is part of recovery. -A clock more than five seconds behind a required new date stops preparation; -correct the clock or retry later. Repositories with `Valid-Until` or a changed -metadata layout require a separate recovery policy. - -Rollback changes which versions the indexes advertise. It does not downgrade -installed packages or restore Datadog-only trust. To restore a newer package -selection afterward, dispatch rollback with that newer archived snapshot; do not -replay its old publication. Run native client checks after public verification. -Never delete a control object to bypass recovery checks. From 9b15dbc1942fd220d5507332b38d83a3b1fc6905 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 21 Oct 2026 09:47:33 -0400 Subject: [PATCH 10/23] refactor(releasing): simplify package repository verification --- .../release/repositories/publisher/engine.rs | 18 +------- .../release/repositories/publisher/model.rs | 6 ++- .../repositories/publisher/repository.rs | 42 +++++++------------ .../repositories/publisher/rollback.rs | 11 +---- 4 files changed, 21 insertions(+), 56 deletions(-) diff --git a/vdev/src/commands/release/repositories/publisher/engine.rs b/vdev/src/commands/release/repositories/publisher/engine.rs index 8f4c9e7a5b41c..987f6b31a3e81 100644 --- a/vdev/src/commands/release/repositories/publisher/engine.rs +++ b/vdev/src/commands/release/repositories/publisher/engine.rs @@ -8,7 +8,6 @@ use super::{ }; use anyhow::{Context, Result, ensure}; use serde_json::Value; -use sha2::{Digest, Sha256}; use std::{collections::BTreeSet, fs, io::Read, path::Path}; fn bundle_key(config: &Config, input: &str, base: Option<&str>) -> String { @@ -285,23 +284,8 @@ pub(super) fn verify_snapshot(config: &Config, snapshot: &Snapshot) -> Result<() response.url().scheme() == "https", "Repository redirected away from HTTPS" ); - let mut source = response.take(expected.size + 1); - let mut hash = Sha256::new(); - let mut size = 0; - let mut buffer = vec![0; 1024 * 1024]; - loop { - let count = source.read(&mut buffer)?; - if count == 0 { - break; - } - size += count as u64; - hash.update(&buffer[..count]); - } ensure!( - Entry { - sha256: hex::encode(hash.finalize()), - size - } == *expected, + Entry::read(response.take(expected.size + 1))? == *expected, "Endpoint differs from snapshot: {name}" ); } diff --git a/vdev/src/commands/release/repositories/publisher/model.rs b/vdev/src/commands/release/repositories/publisher/model.rs index e76cb888535a3..ed409e12e7a86 100644 --- a/vdev/src/commands/release/repositories/publisher/model.rs +++ b/vdev/src/commands/release/repositories/publisher/model.rs @@ -60,12 +60,14 @@ impl Entry { "Expected a regular file: {}", path.display() ); - let mut file = fs::File::open(path)?; + Self::read(fs::File::open(path)?) + } + pub fn read(mut source: impl Read) -> Result { let mut hash = Sha256::new(); let mut size = 0; let mut buffer = vec![0; 1024 * 1024]; loop { - let count = file.read(&mut buffer)?; + let count = source.read(&mut buffer)?; if count == 0 { break; } diff --git a/vdev/src/commands/release/repositories/publisher/repository.rs b/vdev/src/commands/release/repositories/publisher/repository.rs index 9fcfb08dfe6f7..3cfc82279eef5 100644 --- a/vdev/src/commands/release/repositories/publisher/repository.rs +++ b/vdev/src/commands/release/repositories/publisher/repository.rs @@ -281,6 +281,19 @@ impl<'a> PublicKey<'a> { ); Ok(()) } + pub fn verify_apt_release(&self, suite: &Path) -> Result> { + self.verify(&suite.join("InRelease"), None)?; + self.verify(&suite.join("Release.gpg"), Some(&suite.join("Release")))?; + let release = fs::read(suite.join("Release"))?; + ensure!( + self.gpg( + [OsStr::new("--decrypt"), suite.join("InRelease").as_os_str()], + None + )? == release, + "Inline and detached roots disagree" + ); + Ok(release) + } pub fn rpmdb(&self) -> Result { let path = self.home.join("rpmdb"); run( @@ -553,16 +566,7 @@ pub(super) fn verify_metadata( .parent() .context("Missing suite")? .to_owned(); - public.verify(&suite.join("InRelease"), None)?; - public.verify(&suite.join("Release.gpg"), Some(&suite.join("Release")))?; - let release = fs::read(suite.join("Release"))?; - ensure!( - public.gpg( - [OsStr::new("--decrypt"), suite.join("InRelease").as_os_str()], - None - )? == release, - "Inline and detached roots disagree" - ); + let release = public.verify_apt_release(&suite)?; let mut sha256 = false; let mut checked = BTreeSet::new(); for line in std::str::from_utf8(&release)?.lines() { @@ -798,10 +802,6 @@ fn apt_indexes( write(&suite.join("Release"), release.as_bytes())?; signer.sign(&suite.join("Release"), true, &suite.join("InRelease"))?; signer.sign(&suite.join("Release"), false, &suite.join("Release.gpg"))?; - signer.public.verify(&suite.join("InRelease"), None)?; - signer - .public - .verify(&suite.join("Release.gpg"), Some(&suite.join("Release")))?; Ok(()) } #[allow(clippy::too_many_lines)] // Ordered admission, signing and final verification phase. @@ -962,10 +962,6 @@ pub(super) fn prepare(config: &Config, work: &Path) -> Result { false, &target.join("repodata/repomd.xml.asc"), )?; - signer.public.verify( - &target.join("repodata/repomd.xml.asc"), - Some(&target.join("repodata/repomd.xml")), - )?; } } } @@ -983,15 +979,7 @@ pub(super) fn prepare(config: &Config, work: &Path) -> Result { Some(input_id), root.clone(), )?; - let view = tempfile::tempdir()?; - for source in [&base_files, &root] { - for (name, path) in files(source)? { - if !name.ends_with(".deb") && !name.ends_with(".rpm") { - copy(&path, &view.path().join(name))?; - } - } - } - verify_metadata(&snapshot, view.path(), &signer.public)?; + verify_metadata(&snapshot, &root, &signer.public)?; write(&result.join("manifest.json"), &snapshot.data)?; copy(&work.join("input.json"), &result.join("input.json"))?; Snapshot::local(&root)?; diff --git a/vdev/src/commands/release/repositories/publisher/rollback.rs b/vdev/src/commands/release/repositories/publisher/rollback.rs index c3381ebf6f52d..efff6dfd417be 100644 --- a/vdev/src/commands/release/repositories/publisher/rollback.rs +++ b/vdev/src/commands/release/repositories/publisher/rollback.rs @@ -13,7 +13,6 @@ use chrono::Utc; use serde::{Deserialize, Serialize}; use std::{ collections::{BTreeMap, BTreeSet}, - ffi::OsStr, fs, path::{Path, PathBuf}, }; @@ -169,15 +168,7 @@ fn check_sources( for (name, path) in &present { if name.ends_with("/Release") { let suite = path.parent().context("Missing suite")?; - public.verify(&suite.join("InRelease"), None)?; - public.verify(&suite.join("Release.gpg"), Some(path))?; - ensure!( - public.gpg( - [OsStr::new("--decrypt"), suite.join("InRelease").as_os_str()], - None - )? == fs::read(path)?, - "Pending apt roots disagree" - ); + public.verify_apt_release(suite)?; } else if name.ends_with("/repomd.xml") { public.verify( &PathBuf::from(format!("{}.asc", path.display())), From 7bfc6bf8fcfa297956125630030680caae4ee3c9 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 21 Oct 2026 16:30:50 -0400 Subject: [PATCH 11/23] Use the standard vdev installer for package repository jobs --- .../package-repository-tools/action.yml | 6 ++-- .../package-repositories-maintenance.yml | 31 ++++++++++++----- .github/workflows/package-repositories.yml | 34 ++++++++++++++----- Cargo.lock | 2 +- vdev/Cargo.toml | 2 +- 5 files changed, 52 insertions(+), 23 deletions(-) diff --git a/.github/actions/package-repository-tools/action.yml b/.github/actions/package-repository-tools/action.yml index 77289e8e2d058..925c72c633d93 100644 --- a/.github/actions/package-repository-tools/action.yml +++ b/.github/actions/package-repository-tools/action.yml @@ -1,5 +1,5 @@ -name: Restore package repository tools -description: Restore the publisher image and vdev built by this workflow run. +name: Restore package repository image +description: Restore the publisher image built by this workflow run. runs: using: composite steps: @@ -10,5 +10,3 @@ runs: - shell: bash run: | gzip -dc "$RUNNER_TEMP/package-repository-tools/image.tar.gz" | docker load - chmod +x "$RUNNER_TEMP/package-repository-tools/vdev" - echo "VDEV=$RUNNER_TEMP/package-repository-tools/vdev" >> "$GITHUB_ENV" diff --git a/.github/workflows/package-repositories-maintenance.yml b/.github/workflows/package-repositories-maintenance.yml index 9a8207ea78854..bae4ed49127ad 100644 --- a/.github/workflows/package-repositories-maintenance.yml +++ b/.github/workflows/package-repositories-maintenance.yml @@ -55,10 +55,12 @@ jobs: recover-lock) test -n "$EXPECTED_VERSION" && test "$OWNER_STOPPED" = true ;; *) exit 1 ;; esac - - name: Build the reviewed maintenance tools - run: | - cargo build --locked --package vdev - target/debug/vdev release repositories build-image + - uses: ./.github/actions/setup + with: + vdev: true + mold: false + - name: Build the reviewed publisher image + run: '"$VDEV" release repositories build-image' - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 with: role-to-assume: ${{ vars.PACKAGE_REPOSITORIES_ROLE_ARN }} @@ -77,25 +79,24 @@ jobs: case "$OPERATION" in status) ;; rollback) - target/debug/vdev release repositories rollback base "$SNAPSHOT" + "$VDEV" release repositories rollback base "$SNAPSHOT" ;; recover-lock) - target/debug/vdev release repositories recover-lock \ + "$VDEV" release repositories recover-lock \ --expected-version "$EXPECTED_VERSION" --confirm-owner-stopped ;; esac - target/debug/vdev release repositories status > target/package-repository-status.json + "$VDEV" release repositories status > target/package-repository-status.json cat target/package-repository-status.json { echo '```json' cat target/package-repository-status.json echo '```' } >> "$GITHUB_STEP_SUMMARY" - - name: Save the reviewed tools for the rollback jobs + - name: Save the publisher image for the rollback jobs if: inputs.operation == 'rollback' run: | mkdir -p target/package-repository-tools - cp target/debug/vdev target/package-repository-tools/vdev docker save vector-package-publisher:local | gzip > target/package-repository-tools/image.tar.gz - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 if: inputs.operation == 'rollback' @@ -127,6 +128,10 @@ jobs: with: ref: ${{ github.sha }} persist-credentials: false + - uses: ./.github/actions/setup + with: + vdev: true + mold: false - uses: ./.github/actions/package-repository-tools - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: @@ -166,6 +171,10 @@ jobs: with: ref: ${{ github.sha }} persist-credentials: false + - uses: ./.github/actions/setup + with: + vdev: true + mold: false - uses: ./.github/actions/package-repository-tools - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: @@ -203,6 +212,10 @@ jobs: with: ref: ${{ github.sha }} persist-credentials: false + - uses: ./.github/actions/setup + with: + vdev: true + mold: false - uses: ./.github/actions/package-repository-tools - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: diff --git a/.github/workflows/package-repositories.yml b/.github/workflows/package-repositories.yml index efe2d9b4818b7..65f15777bea07 100644 --- a/.github/workflows/package-repositories.yml +++ b/.github/workflows/package-repositories.yml @@ -28,7 +28,7 @@ concurrency: cancel-in-progress: false jobs: - # Freeze verified release assets and build the tools reused by every job. + # Freeze verified release assets and build the publisher image reused by each job. # Enable only after historical backfill and the repository/key migration are ready. inputs: if: >- @@ -55,21 +55,23 @@ jobs: persist-credentials: false - id: source run: echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - - name: Build the reviewed tools - run: | - cargo build --locked --package vdev - target/debug/vdev release repositories build-image + - uses: ./.github/actions/setup + with: + vdev: true + mold: false + - name: Build the reviewed publisher image + run: '"$VDEV" release repositories build-image' - name: Freeze and verify release inputs env: GH_TOKEN: ${{ github.token }} SOURCE_RUN: ${{ inputs.run_id || github.event.workflow_run.id }} SOURCE_TAG: ${{ inputs.tag || github.event.workflow_run.head_branch }} CHANNEL: ${{ inputs.channel || 'stable' }} - run: target/debug/vdev release repositories fetch --run "$SOURCE_RUN" --tag "$SOURCE_TAG" --channel "$CHANNEL" - - name: Save the exact tools used by this run + run: >- + "$VDEV" release repositories fetch --run "$SOURCE_RUN" --tag "$SOURCE_TAG" --channel "$CHANNEL" + - name: Save the publisher image used by this run run: | mkdir -p target/package-repository-tools - cp target/debug/vdev target/package-repository-tools/vdev docker save vector-package-publisher:local | gzip > target/package-repository-tools/image.tar.gz - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: @@ -99,6 +101,10 @@ jobs: with: ref: ${{ needs.inputs.outputs.tooling_commit }} persist-credentials: false + - uses: ./.github/actions/setup + with: + vdev: true + mold: false - uses: ./.github/actions/package-repository-tools - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: @@ -135,6 +141,10 @@ jobs: with: ref: ${{ needs.inputs.outputs.tooling_commit }} persist-credentials: false + - uses: ./.github/actions/setup + with: + vdev: true + mold: false - uses: ./.github/actions/package-repository-tools - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: @@ -174,6 +184,10 @@ jobs: with: ref: ${{ needs.inputs.outputs.tooling_commit }} persist-credentials: false + - uses: ./.github/actions/setup + with: + vdev: true + mold: false - uses: ./.github/actions/package-repository-tools - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: @@ -202,6 +216,10 @@ jobs: with: ref: ${{ needs.inputs.outputs.tooling_commit }} persist-credentials: false + - uses: ./.github/actions/setup + with: + vdev: true + mold: false - uses: ./.github/actions/package-repository-tools - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: diff --git a/Cargo.lock b/Cargo.lock index 6769a4858671e..7c7f0a398474c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13523,7 +13523,7 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vdev" -version = "0.3.33" +version = "0.3.34" dependencies = [ "anyhow", "aws-config", diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index 91c33633b7bfa..7eb18459a2ff4 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vdev" -version = "0.3.33" +version = "0.3.34" edition = "2024" authors = ["Vector Contributors "] license = "MPL-2.0" From 8f4dc4f57d900fecae7cf14ebd7e22d8b994ea86 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 21 Oct 2026 18:33:34 -0400 Subject: [PATCH 12/23] fix(releasing): correct ARMv7 repository setup and reject unsupported majors --- .github/workflows/package-repositories.yml | 2 ++ distribution/repositories/setup.sh | 29 ++++++++++++++-------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/.github/workflows/package-repositories.yml b/.github/workflows/package-repositories.yml index 65f15777bea07..8fdb33eb21d92 100644 --- a/.github/workflows/package-repositories.yml +++ b/.github/workflows/package-repositories.yml @@ -29,6 +29,8 @@ concurrency: jobs: # Freeze verified release assets and build the publisher image reused by each job. + # Trust authorized GitHub release publishers: run/tag and checksum checks do not + # prove build provenance. Stronger provenance verification is deferred. # Enable only after historical backfill and the repository/key migration are ready. inputs: if: >- diff --git a/distribution/repositories/setup.sh b/distribution/repositories/setup.sh index 027d040819736..70087364679da 100644 --- a/distribution/repositories/setup.sh +++ b/distribution/repositories/setup.sh @@ -15,12 +15,12 @@ Do not use --cose-only while the repository is still signed by Datadog. Fresh setup options: VECTOR_DIST_CHANNEL stable (default) or beta - VECTOR_MAJOR_VERSION 0 (default) + VECTOR_MAJOR_VERSION 0 (only supported major) VEC_REPO_URL vector.dev (default repository domain) TESTING_APT_URL HTTPS apt host/path or URL TESTING_YUM_URL HTTPS yum host/path or URL - TESTING_APT_REPO_VERSION " vector-" - TESTING_YUM_VERSION_PATH "/vector-" + TESTING_APT_REPO_VERSION " vector-0" + TESTING_YUM_VERSION_PATH "/vector-0" Both modes: VECTOR_PUBLIC_KEY_URL HTTPS armored public-key URL @@ -28,8 +28,9 @@ Both modes: TESTING_KEYS_URL Compatibility alias: HTTPS host/path containing vector-packages.asc CSM_MIGRATE=true Disable the old timber-vector source, with backup -Existing sources keep their channel, URL and options; fresh setup overrides are -rejected during migration. Package and repository signatures remain required. +Existing sources keep their channel and options; the standard ARMv7 repository +URL is corrected to armv7hl. Fresh setup overrides are rejected during migration. +Package and repository signatures remain required. Other source filenames and deb822 files require manual migration. EOF } @@ -61,6 +62,11 @@ if [[ -f /etc/debian_version ]] && command -v apt-get >/dev/null; then timber_file=/etc/apt/sources.list.d/timber-vector.list elif command -v rpm >/dev/null && { command -v dnf >/dev/null || command -v yum >/dev/null; }; then family=rpm + # DNF calls ARM hard-float "armhfp"; Vector publishes ARMv7 under "armv7hl". + rpm_arch="\$basearch" + case "$(rpm --eval '%{_arch}')" in + armv7hl|armv7hnl|armv8hl|armv8hnl|armv8hcnl) rpm_arch=armv7hl ;; + esac source_file=/etc/yum.repos.d/vector.repo key_file=/etc/pki/rpm-gpg/RPM-GPG-KEY-vector timber_file=/etc/yum.repos.d/timber-vector.repo @@ -196,7 +202,7 @@ elif "$existing"; then fi # DNF maintains a separate metadata keyring, so configure gpgkey as well as # importing the combined public keys into the RPM database. - awk -v key="file://$key_file" ' + awk -v key="file://$key_file" -v arch="$rpm_arch" ' BEGIN { keys=0; seen=0 } /^[[:space:]]*gpgkey[[:space:]]*=/ { if (seen++) exit 2 @@ -208,6 +214,9 @@ elif "$existing"; then /^[[:space:]]*([#;]|$)/ { print; next } { keys=0 } /^[[:space:]]*(gpgcheck|repo_gpgcheck)[[:space:]]*=/ { next } + /^[[:space:]]*baseurl[[:space:]]*=/ && arch=="armv7hl" { + gsub(/\/vector-0\/\$(basearch|\{basearch\})\//, "/vector-0/armv7hl/") + } { print } END { if (!seen) print "gpgkey=" key @@ -218,22 +227,22 @@ else channel=${VECTOR_DIST_CHANNEL:-stable} major=${VECTOR_MAJOR_VERSION:-0} [[ "$channel" == stable || "$channel" == beta ]] || fail "VECTOR_DIST_CHANNEL must be stable or beta." - [[ "$major" =~ ^(0|[1-9][0-9]*)$ ]] || fail "VECTOR_MAJOR_VERSION must be a nonnegative integer." + [[ "$major" == 0 ]] || fail "VECTOR_MAJOR_VERSION must be 0; other repository majors are not published." domain=${VEC_REPO_URL:-vector.dev} [[ "$domain" =~ ^[A-Za-z0-9][A-Za-z0-9.-]*$ ]] || fail "VEC_REPO_URL must be a domain name." if [[ "$family" == apt ]]; then url=$(https_url "${TESTING_APT_URL:-apt.$domain}") selection=${TESTING_APT_REPO_VERSION:-$channel vector-$major} - [[ "$selection" =~ ^(stable|beta)\ vector-(0|[1-9][0-9]*)$ ]] || fail "Invalid apt channel/component override." + [[ "$selection" =~ ^(stable|beta)\ vector-0$ ]] || fail "Expected stable/beta and vector-0 in the apt override." printf 'deb [signed-by=%s] %s/ %s\n' "$key_file" "$url" "$selection" > "$work/source" else url=$(https_url "${TESTING_YUM_URL:-yum.$domain}") selection=${TESTING_YUM_VERSION_PATH:-$channel/vector-$major} - [[ "$selection" =~ ^(stable|beta)/vector-(0|[1-9][0-9]*)$ ]] || fail "Invalid yum channel/component override." + [[ "$selection" =~ ^(stable|beta)/vector-0$ ]] || fail "Expected stable/beta and vector-0 in the yum override." cat > "$work/source" < Date: Thu, 22 Oct 2026 16:03:52 -0400 Subject: [PATCH 13/23] Use a manually published GHCR image for package repositories --- .../package-repository-tools/action.yml | 38 ++++++-- .../workflows/package-repositories-image.yml | 90 +++++++++++++++++++ .../package-repositories-maintenance.yml | 39 ++++---- .github/workflows/package-repositories.yml | 38 +++++--- 4 files changed, 168 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/package-repositories-image.yml diff --git a/.github/actions/package-repository-tools/action.yml b/.github/actions/package-repository-tools/action.yml index 925c72c633d93..5a4257bdcd30d 100644 --- a/.github/actions/package-repository-tools/action.yml +++ b/.github/actions/package-repository-tools/action.yml @@ -1,12 +1,34 @@ -name: Restore package repository image -description: Restore the publisher image built by this workflow run. +name: Pull package repository image +description: Pull the selected immutable publisher image from private GHCR storage. +inputs: + image: + description: Publisher image pinned by its full GHCR SHA-256 digest + required: true + token: + description: GitHub token with read access to the container package + required: true +outputs: + image: + description: Exact image reference used by this job + value: ${{ steps.pull.outputs.image }} runs: using: composite steps: - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: package-repository-tools - path: ${{ runner.temp }}/package-repository-tools - - shell: bash + - id: pull + shell: bash + env: + PUBLISHER_IMAGE: ${{ inputs.image }} + GHCR_TOKEN: ${{ inputs.token }} run: | - gzip -dc "$RUNNER_TEMP/package-repository-tools/image.tar.gz" | docker load + [[ "$PUBLISHER_IMAGE" =~ ^ghcr\.io/vectordotdev/[a-z0-9._/-]+@sha256:[0-9a-f]{64}$ ]] || { + echo 'Set PACKAGE_REPOSITORIES_IMAGE to the digest reported by Package Repository Image.' >&2 + exit 1 + } + # Keep registry credentials in memory and remove them before any signing/AWS step. + export DOCKER_CONFIG + DOCKER_CONFIG=$(mktemp -d /dev/shm/package-registry.XXXXXX) + trap 'rm -rf "$DOCKER_CONFIG"' EXIT + printf '%s' "$GHCR_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + docker pull "$PUBLISHER_IMAGE" + docker tag "$PUBLISHER_IMAGE" vector-package-publisher:local + echo "image=$PUBLISHER_IMAGE" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/package-repositories-image.yml b/.github/workflows/package-repositories-image.yml new file mode 100644 index 0000000000000..8d80cb16c52c0 --- /dev/null +++ b/.github/workflows/package-repositories-image.yml @@ -0,0 +1,90 @@ +name: Package Repository Image + +on: + workflow_dispatch: + +permissions: {} + +# Rebuild manually after publisher/dependency changes or to refresh OS packages. +# A successful build does not change the image selected for production jobs. +concurrency: + group: package-repositories-image + cancel-in-progress: false + +jobs: + # Build without package-signing or AWS credentials; publish only from master. + build: + if: github.repository_id == '146327667' && github.ref == 'refs/heads/master' + runs-on: ubuntu-24.04 + timeout-minutes: 45 + permissions: + contents: read + packages: write + env: + IMAGE_NAME: ghcr.io/vectordotdev/vector-package-publisher + outputs: + image: ${{ steps.publish.outputs.image }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Build, check and publish a private image + id: publish + env: + GH_TOKEN: ${{ github.token }} + run: | + package="orgs/vectordotdev/packages/container/${IMAGE_NAME##*/}" + metadata="$RUNNER_TEMP/package-image.json" + gh api "$package" > "$metadata" || jq -e '.status == "404"' "$metadata" >/dev/null + jq -e '.status == "404" or .visibility == "private"' "$metadata" >/dev/null + + revision=$(git rev-parse HEAD) + tag="$IMAGE_NAME:$revision-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + docker build --pull --no-cache --tag "$tag" \ + --label "org.opencontainers.image.source=https://github.com/$GITHUB_REPOSITORY" \ + --label "org.opencontainers.image.revision=$revision" \ + --file distribution/repositories/Dockerfile . + docker run --rm "$tag" --help + docker run --rm --entrypoint sh "$tag" -ec \ + 'for tool in vdev apt-ftparchive dpkg-deb createrepo_c rpm rpmsign gpg; do command -v "$tool"; done' + + # Registry credentials stay in memory; the new GHCR package is private by default. + export DOCKER_CONFIG + DOCKER_CONFIG=$(mktemp -d /dev/shm/package-registry.XXXXXX) + trap 'rm -rf "$DOCKER_CONFIG"' EXIT + printf '%s' "$GH_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + docker push "$tag" + test "$(gh api "$package" --jq .visibility)" = private + image=$(docker inspect --format '{{index .RepoDigests 0}}' "$tag") + [[ "$image" == "$IMAGE_NAME@sha256:"* ]] + echo "image=$image" >> "$GITHUB_OUTPUT" + + # Pull from GHCR on a fresh runner before showing the digest operators can select. + verify: + needs: build + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + packages: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: ./.github/actions/package-repository-tools + with: + image: ${{ needs.build.outputs.image }} + token: ${{ github.token }} + - name: Check the pulled image and report the manual selection command + env: + PUBLISHER_IMAGE: ${{ needs.build.outputs.image }} + run: | + docker run --rm vector-package-publisher:local --help + { + echo 'Set the repository variable below to select this image for future publishing and maintenance runs.' + echo '```sh' + printf 'gh variable set PACKAGE_REPOSITORIES_IMAGE --repo %s --body %s\n' "$GITHUB_REPOSITORY" "$PUBLISHER_IMAGE" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/package-repositories-maintenance.yml b/.github/workflows/package-repositories-maintenance.yml index bae4ed49127ad..70e0b69bb5739 100644 --- a/.github/workflows/package-repositories-maintenance.yml +++ b/.github/workflows/package-repositories-maintenance.yml @@ -27,15 +27,19 @@ concurrency: jobs: # Inspect S3 state, prepare a rollback, or recover a lock with the publishing role. + # Capture PACKAGE_REPOSITORIES_IMAGE once so every rollback job uses the same digest. # Available even when automated publication is disabled. Recovery requires the exact # control version, a stopped owner and an expired 65-minute recovery window. maintain: if: github.repository_id == '146327667' && github.ref == 'refs/heads/master' runs-on: ubuntu-24.04 timeout-minutes: 30 + outputs: + publisher_image: ${{ steps.image.outputs.image }} environment: package-repositories-publishing permissions: contents: read + packages: read id-token: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -59,8 +63,11 @@ jobs: with: vdev: true mold: false - - name: Build the reviewed publisher image - run: '"$VDEV" release repositories build-image' + - uses: ./.github/actions/package-repository-tools + id: image + with: + image: ${{ vars.PACKAGE_REPOSITORIES_IMAGE }} + token: ${{ github.token }} - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 with: role-to-assume: ${{ vars.PACKAGE_REPOSITORIES_ROLE_ARN }} @@ -93,18 +100,6 @@ jobs: cat target/package-repository-status.json echo '```' } >> "$GITHUB_STEP_SUMMARY" - - name: Save the publisher image for the rollback jobs - if: inputs.operation == 'rollback' - run: | - mkdir -p target/package-repository-tools - docker save vector-package-publisher:local | gzip > target/package-repository-tools/image.tar.gz - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - if: inputs.operation == 'rollback' - with: - name: package-repository-tools - path: target/package-repository-tools/ - retention-days: 1 - if-no-files-found: error - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 if: inputs.operation == 'rollback' with: @@ -123,6 +118,7 @@ jobs: timeout-minutes: 20 permissions: contents: read + packages: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -133,6 +129,9 @@ jobs: vdev: true mold: false - uses: ./.github/actions/package-repository-tools + with: + image: ${{ needs.maintain.outputs.publisher_image }} + token: ${{ github.token }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: package-repository-rollback-base @@ -159,12 +158,13 @@ jobs: # Restore the selected repository state under the shared S3 lock, rejecting stale plans. # Uses AWS OIDC without the signing key; installed clients are not downgraded. publish: - needs: sign + needs: [maintain, sign] runs-on: ubuntu-24.04 environment: package-repositories-publishing timeout-minutes: 20 permissions: contents: read + packages: read id-token: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -176,6 +176,9 @@ jobs: vdev: true mold: false - uses: ./.github/actions/package-repository-tools + with: + image: ${{ needs.maintain.outputs.publisher_image }} + token: ${{ github.token }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: package-repository-rollback-signed @@ -202,11 +205,12 @@ jobs: # Verify the restored indexes and fresh signed roots through public HTTPS. endpoints: - needs: publish + needs: [maintain, publish] runs-on: ubuntu-24.04 timeout-minutes: 20 permissions: contents: read + packages: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -217,6 +221,9 @@ jobs: vdev: true mold: false - uses: ./.github/actions/package-repository-tools + with: + image: ${{ needs.maintain.outputs.publisher_image }} + token: ${{ github.token }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: package-repository-rollback-signed diff --git a/.github/workflows/package-repositories.yml b/.github/workflows/package-repositories.yml index 8fdb33eb21d92..b31a36919bf5d 100644 --- a/.github/workflows/package-repositories.yml +++ b/.github/workflows/package-repositories.yml @@ -28,7 +28,8 @@ concurrency: cancel-in-progress: false jobs: - # Freeze verified release assets and build the publisher image reused by each job. + # Freeze verified release assets and the manually selected publisher image digest. + # Rebuild via Package Repository Image; select its digest with PACKAGE_REPOSITORIES_IMAGE. # Trust authorized GitHub release publishers: run/tag and checksum checks do not # prove build provenance. Stronger provenance verification is deferred. # Enable only after historical backfill and the repository/key migration are ready. @@ -46,9 +47,11 @@ jobs: timeout-minutes: 30 permissions: contents: read + packages: read actions: read outputs: tooling_commit: ${{ steps.source.outputs.commit }} + publisher_image: ${{ steps.image.outputs.image }} steps: # workflow_run executes reviewed default-branch code, never release/tag code. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -61,8 +64,11 @@ jobs: with: vdev: true mold: false - - name: Build the reviewed publisher image - run: '"$VDEV" release repositories build-image' + - uses: ./.github/actions/package-repository-tools + id: image + with: + image: ${{ vars.PACKAGE_REPOSITORIES_IMAGE }} + token: ${{ github.token }} - name: Freeze and verify release inputs env: GH_TOKEN: ${{ github.token }} @@ -71,16 +77,6 @@ jobs: CHANNEL: ${{ inputs.channel || 'stable' }} run: >- "$VDEV" release repositories fetch --run "$SOURCE_RUN" --tag "$SOURCE_TAG" --channel "$CHANNEL" - - name: Save the publisher image used by this run - run: | - mkdir -p target/package-repository-tools - docker save vector-package-publisher:local | gzip > target/package-repository-tools/image.tar.gz - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: package-repository-tools - path: target/package-repository-tools/ - retention-days: 1 - if-no-files-found: error - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: package-repository-inputs @@ -97,6 +93,7 @@ jobs: timeout-minutes: 20 permissions: contents: read + packages: read id-token: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -108,6 +105,9 @@ jobs: vdev: true mold: false - uses: ./.github/actions/package-repository-tools + with: + image: ${{ needs.inputs.outputs.publisher_image }} + token: ${{ github.token }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: package-repository-inputs @@ -138,6 +138,7 @@ jobs: timeout-minutes: 20 permissions: contents: read + packages: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -148,6 +149,9 @@ jobs: vdev: true mold: false - uses: ./.github/actions/package-repository-tools + with: + image: ${{ needs.inputs.outputs.publisher_image }} + token: ${{ github.token }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: package-repository-base @@ -180,6 +184,7 @@ jobs: timeout-minutes: 20 permissions: contents: read + packages: read id-token: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -191,6 +196,9 @@ jobs: vdev: true mold: false - uses: ./.github/actions/package-repository-tools + with: + image: ${{ needs.inputs.outputs.publisher_image }} + token: ${{ github.token }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: package-repository-signed @@ -213,6 +221,7 @@ jobs: timeout-minutes: 20 permissions: contents: read + packages: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -223,6 +232,9 @@ jobs: vdev: true mold: false - uses: ./.github/actions/package-repository-tools + with: + image: ${{ needs.inputs.outputs.publisher_image }} + token: ${{ github.token }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: package-repository-signed From 60927170e0ca79fd4e19de6148c524e8c99a7e9a Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 26 Oct 2026 12:37:17 -0400 Subject: [PATCH 14/23] fix(releasing): migrate ARMv7 repository URLs without trailing slashes --- distribution/repositories/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/repositories/setup.sh b/distribution/repositories/setup.sh index 70087364679da..c0a2c81e65854 100644 --- a/distribution/repositories/setup.sh +++ b/distribution/repositories/setup.sh @@ -215,7 +215,7 @@ elif "$existing"; then { keys=0 } /^[[:space:]]*(gpgcheck|repo_gpgcheck)[[:space:]]*=/ { next } /^[[:space:]]*baseurl[[:space:]]*=/ && arch=="armv7hl" { - gsub(/\/vector-0\/\$(basearch|\{basearch\})\//, "/vector-0/armv7hl/") + gsub(/\/vector-0\/\$(basearch|\{basearch\})(\/|$)/, "/vector-0/armv7hl/") } { print } END { From c676cd6b110cc7102f55760923c76f7c6e7c8481 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 26 Oct 2026 12:41:14 -0400 Subject: [PATCH 15/23] fix(ci): create the package repository status output directory --- .github/workflows/package-repositories-maintenance.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/package-repositories-maintenance.yml b/.github/workflows/package-repositories-maintenance.yml index 70e0b69bb5739..95c7ad308a863 100644 --- a/.github/workflows/package-repositories-maintenance.yml +++ b/.github/workflows/package-repositories-maintenance.yml @@ -93,6 +93,7 @@ jobs: --expected-version "$EXPECTED_VERSION" --confirm-owner-stopped ;; esac + mkdir -p target "$VDEV" release repositories status > target/package-repository-status.json cat target/package-repository-status.json { From 6c34189d8f993b5bd4ca9b873424c7539f332d20 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 26 Oct 2026 12:54:46 -0400 Subject: [PATCH 16/23] fix(releasing): use Ring for publisher AWS clients --- Cargo.lock | 38 +------------------ LICENSE-3rdparty.csv | 2 - vdev/Cargo.toml | 12 +++--- .../release/repositories/publisher/storage.rs | 8 ++++ 4 files changed, 16 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 380daefe6980a..7dfdd16ec97b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1279,29 +1279,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "aws-lc-rs" -version = "1.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13" -dependencies = [ - "aws-lc-sys", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc" -dependencies = [ - "cc", - "cmake", - "dunce", - "fs_extra", - "pkg-config", -] - [[package]] name = "aws-runtime" version = "1.9.2" @@ -4207,12 +4184,6 @@ dependencies = [ "shared_child", ] -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - [[package]] name = "dyn-clone" version = "1.0.20" @@ -4812,12 +4783,6 @@ dependencies = [ "num", ] -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "fsevent-sys" version = "4.1.0" @@ -10580,7 +10545,6 @@ version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ - "aws-lc-rs", "log", "once_cell", "ring", @@ -10663,7 +10627,6 @@ version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ - "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -13315,6 +13278,7 @@ dependencies = [ "aws-sdk-iam", "aws-sdk-s3", "aws-sdk-sts", + "aws-smithy-http-client", "aws-smithy-runtime-api", "aws-smithy-types", "base64 0.23.1", diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index d7d77c23e6b06..95258f66380d5 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -71,8 +71,6 @@ atoi,https://github.com/pacman82/atoi-rs,MIT,Markus Klein atomic-waker,https://github.com/smol-rs/atomic-waker,Apache-2.0 OR MIT,"Stjepan Glavina , Contributors to futures-rs" aws-config,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-credential-types,https://github.com/smithy-lang/smithy-rs,Apache-2.0,AWS Rust SDK Team -aws-lc-rs,https://github.com/aws/aws-lc-rs,ISC AND (Apache-2.0 OR ISC),AWS-LibCrypto -aws-lc-sys,https://github.com/aws/aws-lc-rs,ISC AND (Apache-2.0 OR ISC) AND Apache-2.0 AND MIT AND BSD-3-Clause AND (Apache-2.0 OR ISC OR MIT) AND (Apache-2.0 OR ISC OR MIT-0),AWS-LC aws-runtime,https://github.com/smithy-lang/smithy-rs,Apache-2.0,AWS Rust SDK Team aws-sdk-cloudwatch,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-cloudwatchlogs,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index c80734074ec28..7cc6c358abc7e 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -24,15 +24,17 @@ workspace = true # clients and publisher implementation inside that container. package-repositories = [ "dep:aws-config", "dep:aws-sdk-s3", "dep:aws-sdk-sts", "dep:aws-sdk-iam", - "dep:aws-smithy-runtime-api", "dep:aws-smithy-types", "dep:base64", "dep:flate2", "dep:reqwest", "dep:roxmltree", "dep:tar", + "dep:aws-smithy-http-client", "dep:aws-smithy-runtime-api", "dep:aws-smithy-types", "dep:base64", "dep:flate2", "dep:reqwest", "dep:roxmltree", "dep:tar", "dep:tokio", "dep:uuid", ] [dependencies] -aws-config = { version = "=1.12.0", default-features = false, features = ["behavior-version-latest", "rt-tokio", "default-https-client"], optional = true } -aws-sdk-s3 = { version = "=1.146.0", default-features = false, features = ["behavior-version-latest", "rt-tokio", "default-https-client"], optional = true } -aws-sdk-sts = { version = "=1.114.0", default-features = false, features = ["behavior-version-latest", "rt-tokio", "default-https-client"], optional = true } -aws-sdk-iam = { version = "1.0", default-features = false, features = ["behavior-version-latest", "rt-tokio", "default-https-client"], optional = true } +aws-config = { version = "=1.12.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-s3 = { version = "=1.146.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-sts = { version = "=1.114.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-iam = { version = "1.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +# Match Vector's Ring provider when both packages are built together. +aws-smithy-http-client = { version = "1.4", default-features = false, features = ["rustls-ring"], optional = true } aws-smithy-runtime-api = { version = "1.16", features = ["client"], optional = true } aws-smithy-types = { version = "1", optional = true } base64 = { workspace = true, optional = true } diff --git a/vdev/src/commands/release/repositories/publisher/storage.rs b/vdev/src/commands/release/repositories/publisher/storage.rs index 3fe6f673c4650..bf58a42150bd6 100644 --- a/vdev/src/commands/release/repositories/publisher/storage.rs +++ b/vdev/src/commands/release/repositories/publisher/storage.rs @@ -7,6 +7,10 @@ use aws_sdk_s3::{ primitives::ByteStream, types::{ChecksumMode, ServerSideEncryption}, }; +use aws_smithy_http_client::{ + Builder, + tls::{Provider, rustls_provider::CryptoMode}, +}; use aws_smithy_runtime_api::client::{ interceptors::{Intercept, context::BeforeDeserializationInterceptorContextRef}, runtime_components::RuntimeComponents, @@ -117,8 +121,12 @@ impl S3Store { "fixed-publisher-session", ); let runtime = tokio::runtime::Runtime::new()?; + let http_client = Builder::new() + .tls_provider(Provider::Rustls(CryptoMode::Ring)) + .build_https(); let sdk = runtime.block_on( aws_config::defaults(aws_config::BehaviorVersion::latest()) + .http_client(http_client) .region(Region::new(config.region.clone())) .credentials_provider(credentials) .retry_config(RetryConfig::disabled()) From 2162f20fb098a98e45c94790df6de8ed6b37c29b Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 26 Oct 2026 13:28:49 -0400 Subject: [PATCH 17/23] feat(vdev): add package repository publishing commands --- .github/workflows/changes.yml | 12 + .github/workflows/test.yml | 18 + Cargo.lock | 83 +- distribution/repositories/Dockerfile | 19 + vdev/Cargo.toml | 26 +- vdev/src/commands/release/mod.rs | 3 + vdev/src/commands/release/repositories.rs | 299 ++++++ .../release/repositories/publisher/bundles.rs | 95 ++ .../release/repositories/publisher/engine.rs | 298 ++++++ .../release/repositories/publisher/inputs.rs | 319 ++++++ .../release/repositories/publisher/mod.rs | 76 ++ .../release/repositories/publisher/model.rs | 377 +++++++ .../repositories/publisher/repository.rs | 987 ++++++++++++++++++ .../repositories/publisher/rollback.rs | 326 ++++++ .../release/repositories/publisher/storage.rs | 692 ++++++++++++ 15 files changed, 3628 insertions(+), 2 deletions(-) create mode 100644 distribution/repositories/Dockerfile create mode 100644 vdev/src/commands/release/repositories.rs create mode 100644 vdev/src/commands/release/repositories/publisher/bundles.rs create mode 100644 vdev/src/commands/release/repositories/publisher/engine.rs create mode 100644 vdev/src/commands/release/repositories/publisher/inputs.rs create mode 100644 vdev/src/commands/release/repositories/publisher/mod.rs create mode 100644 vdev/src/commands/release/repositories/publisher/model.rs create mode 100644 vdev/src/commands/release/repositories/publisher/repository.rs create mode 100644 vdev/src/commands/release/repositories/publisher/rollback.rs create mode 100644 vdev/src/commands/release/repositories/publisher/storage.rs diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index c905a4e297cb5..5425373ac512a 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -52,6 +52,8 @@ on: value: ${{ jobs.source.outputs.website_tests }} install: value: ${{ jobs.source.outputs.install }} + package_repositories: + value: ${{ jobs.source.outputs.package_repositories }} actionlint: value: ${{ jobs.source.outputs.actionlint }} prettier: @@ -187,6 +189,7 @@ jobs: markdown: ${{ steps.filter.outputs.markdown }} scripts: ${{ steps.filter.outputs.scripts }} install: ${{ steps.filter.outputs.install }} + package_repositories: ${{ steps.filter.outputs.package_repositories }} k8s: ${{ steps.filter.outputs.k8s }} website_only: ${{ steps.filter.outputs.website == 'true' && steps.filter.outputs.not_website == 'false' }} website_tests: ${{ steps.website_filter.outputs.website_tests }} @@ -284,6 +287,15 @@ jobs: - ".github/workflows/install-sh.yml" - "distribution/install.sh" - ".github/workflows/changes.yml" + package_repositories: + - "distribution/repositories/**" + - "vdev/**" + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - ".github/actions/package-repository-tools/**" + - ".github/workflows/package-repositories*.yml" + - ".github/workflows/changes.yml" k8s: - "src/sources/kubernetes_logs/**" - "lib/k8s-e2e-tests/**" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4177a9d334037..f959cee482e0f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -69,6 +69,23 @@ jobs: cargo-cache: true - run: make check-clippy + check-package-repositories: + name: Check package repository tooling + runs-on: ubuntu-24.04 + timeout-minutes: 45 + if: ${{ needs.changes.outputs.package_repositories == 'true' || needs.changes.outputs.test-yml == 'true' }} + needs: changes + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # CLI and image checks require neither signing material nor AWS access. + - run: | + cargo test --locked --package vdev commands::release::repositories:: + cargo build --locked --package vdev + target/debug/vdev release repositories build-image + docker run --rm vector-package-publisher:local --help + test: needs: changes permissions: @@ -214,6 +231,7 @@ jobs: - check-fmt - check-actionlint - check-clippy + - check-package-repositories - test - check-scripts - check-events diff --git a/Cargo.lock b/Cargo.lock index 69b752d4f83c1..11d00b2ebe016 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1416,6 +1416,33 @@ dependencies = [ "tracing 0.1.44", ] +[[package]] +name = "aws-sdk-iam" +version = "1.123.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabfa8ac26d6684cd0a021cf775bafb972dfa0dcc8ab151354de5e806d4f7d8a" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "http 1.5.0", + "regex-lite", + "tracing 0.1.44", +] + [[package]] name = "aws-sdk-kinesis" version = "1.119.0" @@ -1795,10 +1822,19 @@ dependencies = [ "h2 0.3.26", "h2 0.4.19", "http 0.2.12", + "http 1.5.0", "http-body 0.4.6", "hyper 0.14.32", + "hyper 1.11.1", + "hyper-rustls", + "hyper-util", "pin-project-lite", + "rustls", + "rustls-native-certs", + "rustls-pki-types", "tokio", + "tokio-rustls", + "tower 0.5.3", "tracing 0.1.44", ] @@ -4591,6 +4627,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -11915,6 +11961,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tcp-stream" version = "0.34.2" @@ -13213,9 +13270,17 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vdev" -version = "0.3.35" +version = "0.3.36" dependencies = [ "anyhow", + "aws-config", + "aws-sdk-iam", + "aws-sdk-s3", + "aws-sdk-sts", + "aws-smithy-http-client", + "aws-smithy-runtime-api", + "aws-smithy-types", + "base64 0.23.1", "cfg-if", "chrono", "clap", @@ -13223,6 +13288,7 @@ dependencies = [ "clap_complete", "convert_case 0.12.0", "directories", + "flate2", "git2", "glob", "handlebars", @@ -13237,6 +13303,8 @@ dependencies = [ "proc-macro2 1.0.107", "quote 1.0.47", "regex", + "reqwest 0.11.26", + "roxmltree", "semver", "serde", "serde_json", @@ -13244,9 +13312,12 @@ dependencies = [ "sha2 0.11.0", "strum 0.28.0", "syn 3.0.5", + "tar", "tempfile", + "tokio", "toml", "toml_edit 0.23.9", + "uuid", ] [[package]] @@ -15032,6 +15103,16 @@ dependencies = [ "tap", ] +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix 1.1.4", +] + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/distribution/repositories/Dockerfile b/distribution/repositories/Dockerfile new file mode 100644 index 0000000000000..012915d13ff72 --- /dev/null +++ b/distribution/repositories/Dockerfile @@ -0,0 +1,19 @@ +FROM rust:1.96.1-bookworm@sha256:a339861ae23e9abb272cea45dfafde21760d2ce6577a70f8a926153677902663 AS build +RUN apt-get update && apt-get install -y --no-install-recommends cmake libssl-dev pkg-config \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /src +COPY . . +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + cargo build --locked --package vdev --features package-repositories \ + && strip target/debug/vdev + +FROM ubuntu:24.04@sha256:68ee22004a9109b44973313b02ff7b1de250f1de0f5ecb628d73f579aaff33ac +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + apt-utils ca-certificates createrepo-c dpkg-dev gnupg rpm libssl3t64 \ + && rm -rf /var/lib/apt/lists/* +COPY --from=build /src/target/debug/vdev /usr/local/bin/vdev +# vdev normally locates its checkout before dispatching a command. +COPY --from=build /src/Cargo.toml /app/Cargo.toml +WORKDIR /app +ENTRYPOINT ["vdev", "release", "repositories"] diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index e314d17c7067f..7cc6c358abc7e 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vdev" -version = "0.3.35" +version = "0.3.36" edition = "2024" authors = ["Vector Contributors "] license = "MPL-2.0" @@ -19,7 +19,31 @@ include = [ [lints] workspace = true +[features] +# The ordinary developer CLI only launches the tools container. Compile the AWS +# clients and publisher implementation inside that container. +package-repositories = [ + "dep:aws-config", "dep:aws-sdk-s3", "dep:aws-sdk-sts", "dep:aws-sdk-iam", + "dep:aws-smithy-http-client", "dep:aws-smithy-runtime-api", "dep:aws-smithy-types", "dep:base64", "dep:flate2", "dep:reqwest", "dep:roxmltree", "dep:tar", + "dep:tokio", "dep:uuid", +] + [dependencies] +aws-config = { version = "=1.12.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-s3 = { version = "=1.146.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-sts = { version = "=1.114.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-iam = { version = "1.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +# Match Vector's Ring provider when both packages are built together. +aws-smithy-http-client = { version = "1.4", default-features = false, features = ["rustls-ring"], optional = true } +aws-smithy-runtime-api = { version = "1.16", features = ["client"], optional = true } +aws-smithy-types = { version = "1", optional = true } +base64 = { workspace = true, optional = true } +flate2 = { workspace = true, optional = true } +reqwest = { workspace = true, features = ["blocking"], optional = true } +roxmltree = { version = "0.21", optional = true } +tar = { version = "0.4", optional = true } +tokio = { workspace = true, features = ["rt-multi-thread"], optional = true } +uuid = { version = "1", features = ["v4"], optional = true } anyhow.workspace = true chrono.workspace = true clap.workspace = true diff --git a/vdev/src/commands/release/mod.rs b/vdev/src/commands/release/mod.rs index 1f4fea8d2c4c9..5d8689bb88bd2 100644 --- a/vdev/src/commands/release/mod.rs +++ b/vdev/src/commands/release/mod.rs @@ -2,6 +2,7 @@ mod channel; mod generate_cue; mod github; mod prepare; +mod repositories; mod workflow; use anyhow::{Result, ensure}; @@ -40,6 +41,7 @@ enum Commands { GenerateCue(generate_cue::Cli), Github(github::Cli), Prepare(prepare::Cli), + Repositories(repositories::Cli), Workflow(workflow::Cli), /// Uploads archives and packages to AWS S3 S3(ScriptArgs), @@ -53,6 +55,7 @@ impl Cli { Commands::GenerateCue(cli) => cli.exec(), Commands::Github(cli) => cli.exec(), Commands::Prepare(cli) => cli.exec(), + Commands::Repositories(cli) => cli.exec(), Commands::Workflow(cli) => cli.exec(), Commands::S3(args) => args.exec("release-s3.sh"), } diff --git a/vdev/src/commands/release/repositories.rs b/vdev/src/commands/release/repositories.rs new file mode 100644 index 0000000000000..a51238de831d1 --- /dev/null +++ b/vdev/src/commands/release/repositories.rs @@ -0,0 +1,299 @@ +#[cfg(feature = "package-repositories")] +mod publisher; + +use std::{path::PathBuf, process::Command}; + +use anyhow::{Context as _, Result, ensure}; + +use crate::app::{self, CommandExt as _}; + +/// Build, sign, publish, and recover the apt/yum repositories. +#[derive(clap::Args, Debug)] +pub(super) struct Cli { + /// Container built with `build-image` from the reviewed publisher source. + #[arg(long, default_value = "vector-package-publisher:local", global = true)] + image: String, + /// Run the publisher inside the prepared tools image. + #[arg(long, hide = true, global = true)] + in_container: bool, + /// Configuration and public key directory; never contains a private key. + #[arg( + long, + default_value = "distribution/repositories/production.json", + global = true + )] + config: PathBuf, + /// Persistent workspace exchanged between input, signing, and publishing jobs. + #[arg(long, default_value = "target/package-repositories", global = true)] + work: PathBuf, + #[command(subcommand)] + command: Commands, +} + +#[derive(clap::Subcommand, Debug)] +enum Commands { + /// Build the tools image locally; no registry publication is required. + BuildImage, + /// Validate a completed Release Suite run and freeze its release assets. + Fetch { + #[arg(long)] + run: u64, + #[arg(long)] + tag: String, + #[arg(long, value_enum, default_value = "stable")] + channel: Channel, + }, + /// Read current metadata or restore the previously signed retry bundle. + Base, + /// Validate inputs, import the signing subkey, and generate signed metadata. + Prepare, + /// Save the signed retry bundle and publish with conditional S3 writes. + Publish, + /// Verify the configured public HTTPS endpoints against the signed snapshot. + Verify, + /// Print the publication state and its S3 version for operator inspection. + Status, + /// Restore an archived package selection with fresh signed metadata. + Rollback { + #[command(subcommand)] + phase: Rollback, + }, + /// Release an abandoned lock after the fixed credential safety window. + RecoverLock { + #[arg(long)] + expected_version: String, + /// Confirm the previous process and its in-flight writes have stopped. + #[arg(long)] + confirm_owner_stopped: bool, + }, +} + +#[derive(clap::Subcommand, Debug)] +enum Rollback { + /// Download the selected archive and the current/pending roots, or a saved retry. + Base { snapshot: String }, + /// Sign fresh apt roots for the selected indexes; package bytes stay unchanged. + Prepare, + /// Publish the prepared rollback only if its expected state still matches. + Publish, + /// Verify public HTTPS objects against the prepared rollback. + Verify, +} + +#[derive(clap::ValueEnum, Clone, Copy, Debug)] +enum Channel { + Stable, + Beta, +} + +impl Cli { + pub fn exec(self) -> Result<()> { + if self.in_container { + #[cfg(feature = "package-repositories")] + return publisher::exec(&self.config, &self.work, self.command); + #[cfg(not(feature = "package-repositories"))] + anyhow::bail!("The publisher requires the package-repositories Cargo feature"); + } + app::set_repo_dir()?; + if matches!(self.command, Commands::BuildImage) { + return Command::new("docker") + .args([ + "build", + "--tag", + &self.image, + "--file", + "distribution/repositories/Dockerfile", + ".", + ]) + .check_run(); + } + std::fs::create_dir_all(&self.work)?; + let work = self.work.canonicalize()?; + let config = self + .config + .canonicalize() + .context("Publisher configuration is missing")?; + let directory = config.parent().context("Configuration has no parent")?; + ensure!(config.is_file(), "Configuration must be a regular file"); + let mut command = Command::new("docker"); + command.args(["run", "--rm", "--init"]); + let key_mount = { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt as _; + + let owner = work.metadata()?; + command.args(["--user", &format!("{}:{}", owner.uid(), owner.gid())]); + format!( + "/run/keys:rw,noexec,nosuid,size=16m,mode=0700,uid={},gid={}", + owner.uid(), + owner.gid() + ) + } + #[cfg(not(unix))] + { + "/run/keys:rw,noexec,nosuid,size=16m,mode=0700".to_owned() + } + }; + command.args([ + "--cap-drop=ALL", + "--security-opt=no-new-privileges", + "--tmpfs", + &key_mount, + "--env", + "HOME=/run/keys", + "--mount", + &format!("type=bind,source={},target=/work", work.display()), + "--mount", + &format!( + "type=bind,source={},target=/config,readonly", + directory.display() + ), + ]); + // Forward names, never values: command logging must not expose credentials. + for name in self.command.credentials() { + command.args(["--env", name]); + } + command.arg(&self.image).args([ + "--in-container", + "--work", + "/work", + "--config", + &format!( + "/config/{}", + config + .file_name() + .context("Missing configuration filename")? + .to_string_lossy() + ), + ]); + self.command.add_arguments(&mut command); + command.check_run() + } +} + +impl Commands { + fn credentials(&self) -> &'static [&'static str] { + match self { + Self::Fetch { .. } => &["GH_TOKEN"], + Self::Prepare + | Self::Rollback { + phase: Rollback::Prepare, + } => &["PACKAGE_SIGNING_SUBKEY", "PACKAGE_SIGNING_PASSPHRASE"], + Self::Base + | Self::Publish + | Self::Status + | Self::Rollback { + phase: Rollback::Base { .. } | Rollback::Publish, + } + | Self::RecoverLock { .. } => &[ + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "PACKAGE_REPOSITORIES_ROLE_ARN", + ], + _ => &[], + } + } + + fn add_arguments(self, command: &mut Command) { + match self { + Commands::Fetch { run, tag, channel } => { + command.args([ + "fetch", + "--run", + &run.to_string(), + "--tag", + &tag, + "--channel", + match channel { + Channel::Stable => "stable", + Channel::Beta => "beta", + }, + ]); + } + Commands::Base => { + command.arg("base"); + } + Commands::Prepare => { + command.arg("prepare"); + } + Commands::Publish => { + command.arg("publish"); + } + Commands::Verify => { + command.arg("verify"); + } + Commands::Status => { + command.arg("status"); + } + Commands::Rollback { phase } => { + command.arg("rollback"); + match phase { + Rollback::Base { snapshot } => { + command.args(["base", &snapshot]); + } + Rollback::Prepare => { + command.arg("prepare"); + } + Rollback::Publish => { + command.arg("publish"); + } + Rollback::Verify => { + command.arg("verify"); + } + } + } + Commands::RecoverLock { + expected_version, + confirm_owner_stopped, + } => { + command.args(["recover-lock", "--expected-version", &expected_version]); + if confirm_owner_stopped { + command.arg("--confirm-owner-stopped"); + } + } + Commands::BuildImage => unreachable!(), + } + } +} + +#[cfg(test)] +mod tests { + use super::{Commands, Rollback}; + + #[test] + fn signing_and_publishing_receive_disjoint_credentials() { + let signing = Commands::Prepare.credentials(); + let publishing = Commands::Publish.credentials(); + assert_eq!( + signing, + ["PACKAGE_SIGNING_SUBKEY", "PACKAGE_SIGNING_PASSPHRASE"] + ); + assert!(publishing.contains(&"AWS_SESSION_TOKEN")); + assert!(signing.iter().all(|name| !publishing.contains(name))); + for phase in [ + Rollback::Base { + snapshot: "test".into(), + }, + Rollback::Publish, + ] { + assert_eq!(Commands::Rollback { phase }.credentials(), publishing); + } + assert_eq!( + Commands::Rollback { + phase: Rollback::Prepare + } + .credentials(), + signing + ); + assert!( + Commands::Rollback { + phase: Rollback::Verify + } + .credentials() + .is_empty() + ); + assert!(Commands::Verify.credentials().is_empty()); + } +} diff --git a/vdev/src/commands/release/repositories/publisher/bundles.rs b/vdev/src/commands/release/repositories/publisher/bundles.rs new file mode 100644 index 0000000000000..eb8ad5a7e5819 --- /dev/null +++ b/vdev/src/commands/release/repositories/publisher/bundles.rs @@ -0,0 +1,95 @@ +//! Bounded, checksum-verified transport of signed workspaces. +use super::{ + model::{Entry, LIMIT, Snapshot, files, safe_relative}, + storage::Store, +}; +use anyhow::{Result, ensure}; +use flate2::{Compression, GzBuilder}; +use std::{collections::BTreeSet, fs, io::Write, path::Path}; +pub(super) fn checked_download( + store: &dyn Store, + key: &str, + path: &Path, + entry: Option<&Entry>, +) -> Result<()> { + let head = store + .head(key)? + .ok_or_else(|| anyhow::anyhow!("Missing S3 input: {key}"))?; + ensure!(head.entry.size < LIMIT, "Oversized S3 input"); + fs::create_dir_all( + path.parent() + .ok_or_else(|| anyhow::anyhow!("Missing parent"))?, + )?; + let actual = store.download(key, path, head.entry.size)?; + ensure!( + head.entry == actual && entry.is_none_or(|e| *e == actual), + "Downloaded S3 checksum mismatch" + ); + Ok(()) +} +pub(super) fn unpack_bundle(path: &Path, destination: &Path) -> Result { + fs::create_dir(destination)?; + let mut archive = tar::Archive::new(super::model::gzip(fs::File::open(path)?, LIMIT)); + let mut names = BTreeSet::new(); + let mut total: u64 = 0; + for member in archive.entries()? { + let mut member = member?; + let name = member + .path()? + .to_str() + .ok_or_else(|| anyhow::anyhow!("Non-UTF8 archive name"))? + .to_owned(); + ensure!( + member.header().entry_type().is_file() && names.insert(name.clone()), + "Duplicate or nonregular bundle member" + ); + if let Some(relative) = name.strip_prefix("files/") { + safe_relative(relative)?; + } else { + ensure!( + matches!(name.as_str(), "input.json" | "manifest.json"), + "Unexpected bundle member" + ); + } + total = total + .checked_add(member.size()) + .ok_or_else(|| anyhow::anyhow!("Archive size overflow"))?; + ensure!(total < LIMIT, "Bundle extraction budget exceeded"); + let target = destination.join(name); + fs::create_dir_all(target.parent().unwrap())?; + let mut output = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(target)?; + std::io::copy(&mut member, &mut output)?; + output.flush()?; + } + ensure!( + names.contains("input.json") && names.contains("manifest.json"), + "Incomplete signed bundle" + ); + Snapshot::local(&destination.join("files")) +} +pub(super) fn pack_bundle(source: &Path, target: &Path) -> Result<()> { + let output = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(target)?; + let compressed = GzBuilder::new() + .mtime(0) + .write(output, Compression::default()); + let mut archive = tar::Builder::new(compressed); + for (name, path) in files(source)? { + let mut header = tar::Header::new_gnu(); + header.set_size(fs::metadata(&path)?.len()); + header.set_uid(0); + header.set_gid(0); + header.set_mtime(0); + header.set_mode(0o644); + header.set_entry_type(tar::EntryType::Regular); + header.set_cksum(); + archive.append_data(&mut header, name, fs::File::open(path)?)?; + } + archive.into_inner()?.finish()?; + Ok(()) +} diff --git a/vdev/src/commands/release/repositories/publisher/engine.rs b/vdev/src/commands/release/repositories/publisher/engine.rs new file mode 100644 index 0000000000000..987f6b31a3e81 --- /dev/null +++ b/vdev/src/commands/release/repositories/publisher/engine.rs @@ -0,0 +1,298 @@ +//! Publication phases shared by the protected CI jobs. +use super::{ + bundles::{checked_download, pack_bundle, unpack_bundle}, + inputs::{client, verify_input}, + model::{Config, Entry, LIMIT, Snapshot, files, mutable, string, write}, + repository::{PublicKey, lint_package, repomd_locations, verify_metadata, verify_rpm}, + storage::{Publisher, S3Store, Store}, +}; +use anyhow::{Context, Result, ensure}; +use serde_json::Value; +use std::{collections::BTreeSet, fs, io::Read, path::Path}; + +fn bundle_key(config: &Config, input: &str, base: Option<&str>) -> String { + format!( + "{}bundles/{input}/{}.tar.gz", + config.bundle_prefix, + base.unwrap_or("initial") + ) +} +fn rpm_cache_key(config: &Config, asset: &Value) -> Result { + Ok(format!( + "{}rpms/{}.rpm", + config.bundle_prefix, + string(asset, "sha256")? + )) +} +pub(super) fn download_metadata( + publisher: &Publisher<'_>, + snapshot: &Snapshot, + destination: &Path, + indexes: bool, +) -> Result<()> { + fs::create_dir_all(destination)?; + for (name, entry) in &snapshot.entries { + if mutable(name) { + checked_download( + publisher.store, + &format!( + "{}{}/files/{name}", + publisher.config.archive_prefix, snapshot.id + ), + &destination.join(name), + Some(entry), + )?; + } + } + if indexes { + for name in snapshot + .entries + .keys() + .filter(|n| n.ends_with("/repomd.xml")) + { + let path = destination.join(name); + let root = path + .parent() + .and_then(Path::parent) + .context("Invalid RPM root")?; + for relative in repomd_locations(root)? { + let path = root.join(relative); + let name = path + .strip_prefix(destination)? + .to_str() + .context("Invalid index path")?; + let entry = snapshot + .entries + .get(name) + .context("RPM metadata absent from manifest")?; + checked_download( + publisher.store, + &format!("{}{name}", publisher.config.live_prefix), + &path, + Some(entry), + )?; + } + } + } + Ok(()) +} +pub(super) fn base(config: &Config, work: &Path) -> Result<()> { + let (manifest, input_id) = verify_input(work)?; + let store = S3Store::new(config)?; + let publisher = Publisher::new(&store, config); + let (state, _) = publisher.load_state()?; + ensure!(!state.held, "Publisher locked; inspect status"); + let active = state + .active + .as_deref() + .map(|id| publisher.archived(id)) + .transpose()?; + ensure!( + active.is_some() || !config.require_existing_repository, + "Historical backfill required before publication" + ); + let candidate = if let Some(pending) = &state.pending { + let snapshot = publisher.archived(pending)?; + ensure!( + snapshot.input.as_deref() == Some(&input_id), + "Resume or roll back pending publication" + ); + Some(snapshot) + } else { + active + .as_ref() + .filter(|s| s.input.as_deref() == Some(&input_id)) + .cloned() + }; + let base = candidate.as_ref().map_or_else( + || active.as_ref().map(|s| s.id.as_str()), + |s| s.base.as_deref(), + ); + let key = bundle_key(config, &input_id, base); + if store.head(&key)?.is_some() { + let archive = work.join("cached.tar.gz"); + checked_download(&store, &key, &archive, None)?; + let snapshot = unpack_bundle(&archive, &work.join("result"))?; + ensure!( + snapshot.input.as_deref() == Some(&input_id) + && snapshot.base.as_deref() == base + && fs::read(work.join("result/input.json"))? == fs::read(work.join("input.json"))?, + "Cached publication belongs to another input/base" + ); + fs::remove_file(archive)?; + println!("Restored saved signed publication; signing skipped"); + return Ok(()); + } + ensure!( + candidate.is_none(), + "Required signed retry bundle is missing" + ); + write( + &work.join("base.json"), + active + .as_ref() + .map_or(b"null\n".as_slice(), |s| s.data.as_slice()), + )?; + let destination = work.join("base"); + fs::create_dir(&destination)?; + if let Some(active) = active { + download_metadata(&publisher, &active, &destination, true)?; + } + let reuse = work.join("reuse"); + fs::create_dir(&reuse)?; + for (name, asset) in manifest["assets"].as_object().context("Missing assets")? { + if asset["format"] == "rpm" { + let key = rpm_cache_key(config, asset)?; + if store.head(&key)?.is_some() { + checked_download(&store, &key, &reuse.join(name), None)?; + } + } + } + println!("Downloaded current metadata and saved RPM signatures"); + Ok(()) +} +pub(super) fn check_result(config: &Config, work: &Path) -> Result<(Value, Snapshot)> { + let (manifest, input_id) = verify_input(work)?; + let snapshot = Snapshot::local(&work.join("result/files"))?; + ensure!( + snapshot.input.as_deref() == Some(&input_id) + && fs::read(work.join("result/input.json"))? == fs::read(work.join("input.json"))?, + "Signed output belongs to another input" + ); + let public = PublicKey::new(config, "check-result")?; + verify_metadata(&snapshot, &snapshot.source, &public)?; + let rpmdb = public.rpmdb()?; + for (name, asset) in manifest["assets"].as_object().context("Missing assets")? { + if asset["format"] == "deb" { + ensure!( + snapshot.entries.get(&format!("apt/pool/v/ve/{name}")) + == Some(&super::model::entry_value(asset)?), + "Debian package changed" + ); + } else if asset["format"] == "rpm" { + let path = snapshot.source.join(format!( + "yum/{}/vector-0/{}/{name}", + string(&manifest, "channel")?, + string(asset, "arch")? + )); + lint_package(&path, asset)?; + verify_rpm(&rpmdb, &path)?; + } + } + Ok((manifest, snapshot)) +} +pub(super) fn save_bundle(publisher: &Publisher<'_>, work: &Path, key: &str) -> Result<()> { + let archive = work.join("prepared.tar.gz"); + if archive.exists() { + fs::remove_file(&archive)?; + } + pack_bundle(&work.join("result"), &archive)?; + let entry = Entry::file(&archive)?; + ensure!( + entry.size < LIMIT, + "Prepared bundle exceeds single-PUT limit" + ); + if let Some(saved) = publisher.store.head(key)? { + if saved.entry == entry { + return Ok(()); + } + // Compression or tar encodings can change between tool versions. Keep + // the saved archive when every signed file and manifest is identical. + let temporary = tempfile::tempdir_in(work)?; + let saved_archive = temporary.path().join("saved.tar.gz"); + checked_download(publisher.store, key, &saved_archive, Some(&saved.entry))?; + let saved_result = temporary.path().join("result"); + unpack_bundle(&saved_archive, &saved_result)?; + let contents = |root: &Path| -> Result> { + files(root)? + .into_iter() + .map(|(name, path)| Ok((name, Entry::file(&path)?))) + .collect() + }; + ensure!( + contents(&saved_result)? == contents(&work.join("result"))?, + "Saved signed bundle contains different files" + ); + return Ok(()); + } + publisher.ensure_file(key, &archive, &entry) +} +pub(super) fn publish(config: &Config, work: &Path) -> Result<()> { + let (manifest, snapshot) = check_result(config, work)?; + let store = S3Store::new(config)?; + let mut publisher = Publisher::new(&store, config); + ensure!( + publisher.load_state()?.0.active.is_some() || !config.require_existing_repository, + "Historical backfill required" + ); + for (name, asset) in manifest["assets"].as_object().context("Missing assets")? { + if asset["format"] == "rpm" { + let relative = format!( + "yum/{}/vector-0/{}/{name}", + string(&manifest, "channel")?, + string(asset, "arch")? + ); + publisher.ensure_file( + &rpm_cache_key(config, asset)?, + &snapshot.source.join(&relative), + &snapshot.entries[&relative], + )?; + } + } + save_bundle( + &publisher, + work, + &bundle_key( + config, + snapshot + .input + .as_deref() + .context("Missing input identity")?, + snapshot.base.as_deref(), + ), + )?; + publisher.publish(&snapshot, None)?; + println!("Published snapshot {}", snapshot.id); + Ok(()) +} +pub(super) fn verify_snapshot(config: &Config, snapshot: &Snapshot) -> Result<()> { + let mut names: BTreeSet<_> = snapshot + .entries + .keys() + .filter(|n| mutable(n)) + .cloned() + .collect(); + names.extend(files(&snapshot.source)?.into_keys()); + let client = client(true)?; + for name in &names { + let (repository, relative) = name.split_once('/').context("Invalid repository path")?; + let root = if repository == "apt" { + &config.apt_url + } else { + &config.yum_url + }; + ensure!( + root.starts_with("https://") && root.ends_with('/'), + "Verification requires HTTPS repository roots" + ); + let expected = &snapshot.entries[name]; + let response = client + .get(format!("{root}{relative}")) + .send()? + .error_for_status()?; + ensure!( + response.url().scheme() == "https", + "Repository redirected away from HTTPS" + ); + ensure!( + Entry::read(response.take(expected.size + 1))? == *expected, + "Endpoint differs from snapshot: {name}" + ); + } + println!( + "Verified {} endpoint objects for {}", + names.len(), + snapshot.id + ); + Ok(()) +} diff --git a/vdev/src/commands/release/repositories/publisher/inputs.rs b/vdev/src/commands/release/repositories/publisher/inputs.rs new file mode 100644 index 0000000000000..2986bd81d105a --- /dev/null +++ b/vdev/src/commands/release/repositories/publisher/inputs.rs @@ -0,0 +1,319 @@ +//! Admission of already-built, immutable GitHub release assets. +use super::model::{Config, Entry, canonical, digest, entry_value, files, is_digest, string}; +use anyhow::{Context, Result, ensure}; +use reqwest::{blocking::Client, redirect::Policy}; +use serde_json::{Value, json}; +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + io::{Read, Write}, + path::Path, + time::Duration, +}; + +pub(super) const DEB_ARCHES: [&str; 3] = ["amd64", "arm64", "armhf"]; +pub(super) const RPM_ARCHES: [&str; 3] = ["x86_64", "aarch64", "armv7hl"]; +pub(super) fn versions(tag: &str, channel: &str) -> Result<(String, String, String)> { + ensure!( + regex::Regex::new( + r"^v0\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)(?:-([0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*))?$" + )? + .is_match(tag), + "Expected a Vector 0.x release tag" + ); + ensure!( + matches!(channel, "stable" | "beta") && (channel != "stable" || !tag.contains('-')), + "Invalid channel or stable prerelease" + ); + let raw = tag + .strip_prefix('v') + .context("Missing tag prefix")? + .to_owned(); + let native = raw.replacen('-', "~", 1); + Ok(( + raw, + format!("{native}-1"), + format!("{}-1", native.replace('-', ".")), + )) +} +pub(super) fn expected_packages(tag: &str, channel: &str) -> Result> { + let (_, deb, rpm) = versions(tag, channel)?; + Ok(DEB_ARCHES + .into_iter() + .map(|arch| { + ( + format!("vector_{deb}_{arch}.deb"), + json!({"format":"deb","arch":arch,"version":deb}), + ) + }) + .chain(RPM_ARCHES.into_iter().map(|arch| { + ( + format!("vector-{rpm}.{arch}.rpm"), + json!({"format":"rpm","arch":arch,"version":rpm}), + ) + })) + .collect()) +} +pub(super) fn validate_run(config: &Config, run: &Value, tag: &str, commit: &str) -> Result<()> { + ensure!( + run["repository"]["id"] == config.repository_id + && run["head_repository"]["id"] == config.repository_id, + "Wrong source repository or fork run" + ); + ensure!( + run["path"] == config.workflow && run["event"] == "push" && run["head_branch"] == tag, + "Source must be the release workflow tag push" + ); + ensure!( + run["status"] == "completed" && run["conclusion"] == "success", + "Source run did not succeed" + ); + ensure!( + run["head_sha"] == commit + && commit.len() == 40 + && commit + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)), + "Run and tag commit differ" + ); + Ok(()) +} +pub(super) fn select_assets( + release: &Value, + tag: &str, + channel: &str, +) -> Result> { + let (raw, deb, _) = versions(tag, channel)?; + ensure!( + release["tag_name"] == tag + && release["draft"] == false + && release["published_at"] + .as_str() + .is_some_and(|s| !s.is_empty()), + "Release must be published and match tag" + ); + ensure!( + channel != "stable" || release["prerelease"] == false, + "Stable release marked prerelease" + ); + let expected = expected_packages(tag, channel)?; + let sums = format!("vector-{raw}-SHA256SUMS"); + let mut seen = BTreeSet::new(); + let mut selected = BTreeMap::new(); + for asset in release["assets"] + .as_array() + .context("Missing release assets")? + { + let name = string(asset, "name")?; + ensure!(seen.insert(name), "Duplicate release asset"); + ensure!( + !name.ends_with(".deb") && !name.ends_with(".rpm") + || expected.contains_key(name) + || name == format!("vector_{deb}_armel.deb"), + "Unexpected Linux package: {name}" + ); + if !expected.contains_key(name) && name != sums { + continue; + } + let hash = string(asset, "digest")? + .strip_prefix("sha256:") + .context("Asset lacks SHA-256 digest")?; + ensure!(is_digest(hash), "Invalid asset digest"); + let size = asset["size"].as_u64().context("Invalid asset size")?; + ensure!( + size > 0 && size < 1024 * 1024 * 1024 && asset["id"].as_u64().is_some_and(|id| id > 0), + "Invalid asset size or ID" + ); + let mut item = expected.get(name).cloned().unwrap_or_else(|| json!({})); + item["id"] = asset["id"].clone(); + item["size"] = json!(size); + item["sha256"] = json!(hash); + selected.insert(name.to_owned(), item); + } + ensure!( + selected.len() == expected.len() + 1, + "Missing required packages or SHA256SUMS" + ); + Ok(selected) +} +pub(super) fn parse_checksums(bytes: &[u8]) -> Result> { + let expression = regex::Regex::new(r"^([0-9a-f]{64}) [ *]([^/\\ \t\r\n\x0b\x0c]+)$")?; + let mut sums = BTreeMap::new(); + for line in std::str::from_utf8(bytes)?.lines() { + let captures = expression + .captures(line) + .context("Malformed SHA256SUMS entry")?; + ensure!( + sums.insert(captures[2].to_owned(), captures[1].to_owned()) + .is_none(), + "Duplicate SHA256SUMS entry" + ); + } + Ok(sums) +} +pub(super) fn client(redirects: bool) -> Result { + Ok(Client::builder() + .timeout(Duration::from_secs(90)) + .https_only(true) + .redirect(if redirects { + Policy::limited(10) + } else { + Policy::none() + }) + .user_agent("vector-package-publisher") + .build()?) +} +fn github(client: &Client, endpoint: &str) -> Result { + let mut request = client + .get(format!("https://api.github.com/{endpoint}")) + .header("Accept", "application/vnd.github+json") + .header("X-GitHub-Api-Version", "2022-11-28"); + if let Ok(token) = std::env::var("GH_TOKEN") { + request = request.bearer_auth(token); + } + let response = request.send()?.error_for_status()?; + ensure!( + response.status().is_success(), + "GitHub API redirects are forbidden" + ); + Ok(response.json()?) +} +pub(super) fn download( + client: &Client, + url: &str, + destination: &Path, + expected: &Entry, +) -> Result<()> { + ensure!(!destination.is_symlink(), "Invalid download destination"); + let mut response = client.get(url).send()?.error_for_status()?; + ensure!( + response.url().scheme() == "https", + "Download redirected away from HTTPS" + ); + let mut file = fs::File::create(destination)?; + let size = std::io::copy( + &mut Read::by_ref(&mut response).take(expected.size + 1), + &mut file, + )?; + file.flush()?; + ensure!( + size == expected.size && Entry::file(destination)? == *expected, + "Asset digest/size mismatch" + ); + Ok(()) +} +pub(super) fn fetch( + config: &Config, + work: &Path, + run_id: u64, + tag: &str, + channel: &str, +) -> Result<()> { + expected_packages(tag, channel)?; + let api = client(false)?; + let public = client(true)?; + let repo = &config.repository; + let run = github(&api, &format!("repos/{repo}/actions/runs/{run_id}"))?; + // Admitted tags contain only URL-path-safe ASCII. + let mut reference = + github(&api, &format!("repos/{repo}/git/ref/tags/{tag}"))?["object"].clone(); + for _ in 0..4 { + if reference["type"] == "commit" { + break; + } + ensure!(reference["type"] == "tag", "Ref does not identify a commit"); + reference = github( + &api, + &format!("repos/{repo}/git/tags/{}", string(&reference, "sha")?), + )?["object"] + .clone(); + } + ensure!(reference["type"] == "commit", "Too many tag indirections"); + validate_run(config, &run, tag, string(&reference, "sha")?)?; + let endpoint = format!("repos/{repo}/releases/tags/{tag}"); + let release = github(&api, &endpoint)?; + let assets = select_assets(&release, tag, channel)?; + let manifest = json!({"schema":1,"repository":repo,"repository_id":config.repository_id,"run":run_id,"commit":reference["sha"],"tag":tag,"channel":channel,"release":release["id"],"assets":assets}); + let data = canonical(&manifest)?; + let destination = work.join("input"); + fs::create_dir_all(&destination)?; + ensure!( + files(&destination)?.keys().all(|n| assets.contains_key(n)), + "Unexpected input files" + ); + if work.join("input.json").exists() { + ensure!( + fs::read(work.join("input.json"))? == data, + "Frozen inputs changed" + ); + } + for (name, asset) in &assets { + let path = destination.join(name); + let expected = entry_value(asset)?; + if path.exists() { + ensure!(Entry::file(&path)? == expected, "Downloaded asset changed"); + continue; + } + let temporary = work.join("download.partial"); + download( + &public, + &format!("https://github.com/{repo}/releases/download/{tag}/{name}"), + &temporary, + &expected, + )?; + fs::rename(temporary, path)?; + } + let after = github(&api, &endpoint)?; + ensure!( + after["id"] == release["id"] && select_assets(&after, tag, channel)? == assets, + "Release assets changed during download" + ); + fs::write(work.join("input.json"), data)?; + verify_input(work)?; + println!("Verified six repository packages and published checksums"); + Ok(()) +} +pub(super) fn verify_input(work: &Path) -> Result<(Value, String)> { + let data = fs::read(work.join("input.json"))?; + let manifest: Value = serde_json::from_slice(&data)?; + ensure!( + canonical(&manifest)? == data && manifest["schema"] == 1, + "Invalid input manifest" + ); + ensure!( + manifest["repository"] == "vectordotdev/vector" && manifest["repository_id"] == 146_327_667, + "Wrong input repository" + ); + let tag = string(&manifest, "tag")?; + let expected = expected_packages(tag, string(&manifest, "channel")?)?; + let sums_name = format!( + "vector-{}-SHA256SUMS", + tag.strip_prefix('v').context("Missing tag prefix")? + ); + let assets = manifest["assets"].as_object().context("Missing assets")?; + let wanted: BTreeSet<_> = expected.keys().chain(std::iter::once(&sums_name)).collect(); + ensure!( + assets.keys().collect::>() == wanted + && files(&work.join("input"))?.keys().collect::>() == wanted, + "Input file set changed" + ); + for (name, asset) in assets { + ensure!( + Entry::file(&work.join("input").join(name))? == entry_value(asset)?, + "Input asset changed: {name}" + ); + if let Some(identity) = expected.get(name) { + for (field, value) in identity.as_object().context("Invalid identity")? { + ensure!(asset.get(field) == Some(value), "Native identity changed"); + } + } + } + let sums = parse_checksums(&fs::read(work.join("input").join(sums_name))?)?; + for name in expected.keys() { + ensure!( + sums.get(name).map(String::as_str) == assets[name]["sha256"].as_str(), + "SHA256SUMS disagrees with GitHub digest" + ); + } + Ok((manifest, digest(&data))) +} diff --git a/vdev/src/commands/release/repositories/publisher/mod.rs b/vdev/src/commands/release/repositories/publisher/mod.rs new file mode 100644 index 0000000000000..c3b0bdd72584b --- /dev/null +++ b/vdev/src/commands/release/repositories/publisher/mod.rs @@ -0,0 +1,76 @@ +//! Container-side implementation of `vdev release repositories`. +// Repository object names are case-sensitive protocol identifiers. +#![allow(clippy::case_sensitive_file_extension_comparisons)] +mod bundles; +mod engine; +mod inputs; +mod model; +mod repository; +mod rollback; +mod storage; +use super::{Channel, Commands, Rollback}; +use anyhow::{Result, bail}; +use model::Config; +use std::path::Path; + +pub(super) fn exec(config: &Path, work: &Path, command: Commands) -> Result<()> { + let config = Config::load(config)?; + let work = work.canonicalize()?; + match command { + Commands::Fetch { run, tag, channel } => inputs::fetch( + &config, + &work, + run, + &tag, + match channel { + Channel::Stable => "stable", + Channel::Beta => "beta", + }, + ), + Commands::Base => engine::base(&config, &work), + Commands::Prepare => { + println!("Prepared snapshot {}", repository::prepare(&config, &work)?); + Ok(()) + } + Commands::Publish => engine::publish(&config, &work), + Commands::Verify => { + engine::verify_snapshot(&config, &engine::check_result(&config, &work)?.1) + } + Commands::Rollback { phase } => match phase { + Rollback::Base { snapshot } => rollback::base(&config, &work, &snapshot), + Rollback::Prepare => { + println!( + "Prepared rollback snapshot {}", + rollback::prepare(&config, &work)? + ); + Ok(()) + } + Rollback::Publish => rollback::publish(&config, &work), + Rollback::Verify => { + engine::verify_snapshot(&config, &rollback::check_result(&config, &work)?.1) + } + }, + Commands::Status | Commands::RecoverLock { .. } => { + let store = storage::S3Store::new(&config)?; + let mut publisher = storage::Publisher::new(&store, &config); + if let Commands::RecoverLock { + expected_version, + confirm_owner_stopped, + } = command + { + publisher.recover_lock(&expected_version, confirm_owner_stopped)?; + println!("Lock released; resume pending publication or roll back"); + } else { + let (state, head) = publisher.load_state()?; + println!( + "{}", + serde_json::to_string_pretty( + &serde_json::json!({"state":state,"version":head.map(|h|h.version)}) + )? + ); + } + Ok(()) + } + Commands::BuildImage => bail!("Build the image outside the container"), + } +} diff --git a/vdev/src/commands/release/repositories/publisher/model.rs b/vdev/src/commands/release/repositories/publisher/model.rs new file mode 100644 index 0000000000000..ed409e12e7a86 --- /dev/null +++ b/vdev/src/commands/release/repositories/publisher/model.rs @@ -0,0 +1,377 @@ +//! Stable on-disk and S3 formats shared by publication, retries and rollback. +use anyhow::{Context, Result, ensure}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use std::{ + collections::BTreeMap, + fs, + io::{Read, Write}, + path::{Path, PathBuf}, +}; + +pub(super) const LIMIT: u64 = 5 * 1024 * 1024 * 1024; + +pub(super) fn canonical(value: &impl Serialize) -> Result> { + let mut value = serde_json::to_value(value)?; + value.sort_all_objects(); + let text = serde_json::to_string(&value)?; + // Preserve the original manifest encoding, including Python's ensure_ascii. + let mut bytes = Vec::new(); + for character in text.chars() { + if character.is_ascii() { + bytes.push(character as u8); + } else { + for unit in character.encode_utf16(&mut [0; 2]) { + write!(bytes, "\\u{unit:04x}")?; + } + } + } + bytes.push(b'\n'); + Ok(bytes) +} + +pub(super) fn digest(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) +} +pub(super) fn is_digest(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct Entry { + pub sha256: String, + pub size: u64, +} +impl Entry { + pub fn bytes(bytes: &[u8]) -> Self { + Self { + sha256: digest(bytes), + size: bytes.len() as u64, + } + } + pub fn file(path: &Path) -> Result { + ensure!( + path.symlink_metadata()?.is_file(), + "Expected a regular file: {}", + path.display() + ); + Self::read(fs::File::open(path)?) + } + pub fn read(mut source: impl Read) -> Result { + let mut hash = Sha256::new(); + let mut size = 0; + let mut buffer = vec![0; 1024 * 1024]; + loop { + let count = source.read(&mut buffer)?; + if count == 0 { + break; + } + size += count as u64; + hash.update(&buffer[..count]); + } + Ok(Self { + sha256: hex::encode(hash.finalize()), + size, + }) + } +} + +pub(super) fn safe_relative(name: &str) -> Result<&Path> { + let parts: Vec<_> = name.split('/').collect(); + ensure!( + name.bytes() + .all(|b| b.is_ascii_alphanumeric() || b"_./+~-".contains(&b)) + && parts.len() >= 3 + && matches!(parts[0], "apt" | "yum") + && parts + .iter() + .all(|p| !p.is_empty() && *p != "." && *p != ".."), + "Invalid repository path: {name}" + ); + Ok(Path::new(name)) +} +pub(super) fn mutable(name: &str) -> bool { + matches!( + name.rsplit('/').next(), + Some( + "Packages" + | "Packages.gz" + | "Release" + | "Release.gpg" + | "InRelease" + | "repomd.xml" + | "repomd.xml.asc" + ) + ) +} +pub(super) fn root_order(name: &str) -> u8 { + match name.rsplit('/').next().unwrap_or("") { + "Release" => 0, + "Release.gpg" => 1, + "InRelease" => 2, + "repomd.xml" => 3, + "repomd.xml.asc" => 4, + _ => 5, + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Manifest { + schema: u32, + base: Option, + input: Option, + files: BTreeMap, +} +#[derive(Clone, Debug)] +pub(super) struct Snapshot { + pub entries: BTreeMap, + pub base: Option, + pub input: Option, + pub data: Vec, + pub id: String, + pub source: PathBuf, +} +impl Snapshot { + pub fn new( + entries: BTreeMap, + base: Option, + input: Option, + source: PathBuf, + ) -> Result { + ensure!( + !entries.is_empty() && entries.keys().any(|n| mutable(n)), + "Snapshot has no repository metadata" + ); + for (name, entry) in &entries { + safe_relative(name)?; + ensure!( + is_digest(&entry.sha256) && entry.size <= LIMIT, + "Invalid snapshot entry" + ); + } + for id in [&base, &input].into_iter().flatten() { + ensure!(is_digest(id), "Invalid snapshot identity"); + } + let data = canonical(&Manifest { + schema: 2, + base: base.clone(), + input: input.clone(), + files: entries.clone(), + })?; + Ok(Self { + entries, + base, + input, + id: digest(&data), + data, + source, + }) + } + pub fn decode(data: &[u8], expected: Option<&str>) -> Result { + let value: Manifest = serde_json::from_slice(data)?; + ensure!(value.schema == 2, "Unsupported snapshot schema"); + let snapshot = Self::new(value.files, value.base, value.input, PathBuf::new())?; + ensure!( + snapshot.data == data && expected.is_none_or(|id| id == snapshot.id), + "Noncanonical manifest or incorrect digest" + ); + Ok(snapshot) + } + pub fn local(source: &Path) -> Result { + let mut snapshot = Self::decode( + &fs::read( + source + .parent() + .context("Missing snapshot parent")? + .join("manifest.json"), + )?, + None, + )?; + let files = files(source)?; + for (name, path) in &files { + ensure!( + snapshot.entries.get(name) == Some(&Entry::file(path)?), + "Snapshot content differs: {name}" + ); + } + for name in snapshot.entries.keys() { + ensure!( + files.contains_key(name) || (!mutable(name) && snapshot.base.is_some()), + "Missing snapshot file: {name}" + ); + } + source.clone_into(&mut snapshot.source); + Ok(snapshot) + } +} + +/// Walk without following symlinks, including at the root. +pub(super) fn files(root: &Path) -> Result> { + fn visit(root: &Path, directory: &Path, result: &mut BTreeMap) -> Result<()> { + ensure!( + directory.symlink_metadata()?.is_dir(), + "Expected a regular directory: {}", + directory.display() + ); + for item in fs::read_dir(directory)? { + let path = item?.path(); + let kind = path.symlink_metadata()?.file_type(); + if kind.is_dir() { + visit(root, &path, result)?; + } else { + ensure!(kind.is_file(), "Nonregular file: {}", path.display()); + result.insert( + path.strip_prefix(root)? + .to_str() + .context("Non-UTF8 path")? + .replace('\\', "/"), + path, + ); + } + } + Ok(()) + } + let mut result = BTreeMap::new(); + visit(root, root, &mut result)?; + Ok(result) +} +pub(super) fn write(path: &Path, data: &[u8]) -> Result<()> { + fs::create_dir_all(path.parent().context("Missing parent")?)?; + ensure!(!path.is_symlink(), "Refusing a symlink output"); + fs::write(path, data)?; + Ok(()) +} +pub(super) fn copy(source: &Path, destination: &Path) -> Result<()> { + ensure!( + source.symlink_metadata()?.is_file() && !destination.is_symlink(), + "Copy requires regular files" + ); + fs::create_dir_all(destination.parent().context("Missing parent")?)?; + fs::copy(source, destination)?; + Ok(()) +} + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Config { + pub schema: u32, + #[serde(default = "default_true")] + pub require_existing_repository: bool, + pub repository: String, + pub repository_id: u64, + pub workflow: String, + pub account: String, + pub bucket: String, + pub region: String, + pub live_prefix: String, + pub archive_prefix: String, + pub bundle_prefix: String, + pub control_key: String, + pub public_key: String, + pub primary_fingerprint: String, + pub signing_fingerprint: String, + pub apt_url: String, + pub yum_url: String, + #[serde(skip)] + pub key_path: PathBuf, +} +fn default_true() -> bool { + true +} +impl Config { + pub fn load(path: &Path) -> Result { + let mut config: Self = serde_json::from_slice(&fs::read(path)?)?; + ensure!( + config.schema == 1 + && config.repository == "vectordotdev/vector" + && config.repository_id == 146_327_667, + "Wrong publisher configuration/source repository" + ); + ensure!( + config.account.len() == 12 && config.account.bytes().all(|b| b.is_ascii_digit()), + "Invalid AWS account" + ); + ensure!( + regex::Regex::new(r"^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$")?.is_match(&config.bucket), + "Invalid bucket" + ); + let paths = [ + &config.live_prefix, + &config.archive_prefix, + &config.bundle_prefix, + &config.control_key, + ]; + for (index, name) in paths.iter().enumerate() { + ensure!( + name.bytes() + .all(|b| b.is_ascii_alphanumeric() || b"_./-".contains(&b)) + && name + .trim_end_matches('/') + .split('/') + .all(|p| !p.is_empty() && p != "." && p != ".."), + "Invalid S3 path" + ); + ensure!(index == 3 || name.ends_with('/'), "Prefixes must end in /"); + for other in &paths[index + 1..] { + ensure!( + !name.starts_with(other.as_str()) && !other.starts_with(name.as_str()), + "S3 paths must be disjoint" + ); + } + } + if config.account == "486234852809" { + ensure!( + config.bucket == "dd-cose-releases" && config.live_prefix == "vector/repositories/", + "Unexpected production destination" + ); + ensure!( + paths[1..].iter().all(|p| !p.starts_with("vector/")), + "Internal state cannot use the public prefix" + ); + } + for fingerprint in [&config.primary_fingerprint, &config.signing_fingerprint] { + ensure!( + fingerprint.len() == 40 + && fingerprint + .bytes() + .all(|b| b.is_ascii_digit() || (b'A'..=b'F').contains(&b)), + "Invalid pinned fingerprint" + ); + } + ensure!( + !config.public_key.is_empty() + && Path::new(&config.public_key) + .file_name() + .is_some_and(|p| p == config.public_key.as_str()), + "Public key must be beside configuration" + ); + config.key_path = path + .parent() + .context("Missing configuration parent")? + .join(&config.public_key); + Ok(config) + } +} +pub(super) fn string<'a>(value: &'a Value, key: &str) -> Result<&'a str> { + value[key] + .as_str() + .with_context(|| format!("Missing/invalid {key}")) +} +pub(super) fn entry_value(value: &Value) -> Result { + Ok(serde_json::from_value( + json!({"size":value["size"], "sha256":value["sha256"]}), + )?) +} + +/// Publishing has its own extraction budgets and does not depend on Vector's +/// runtime crates. Cap the decompressed stream before tar/XML parsing. +#[allow(clippy::disallowed_types)] +pub(super) fn gzip(source: impl Read, limit: u64) -> impl Read { + flate2::read::GzDecoder::new(source).take(limit) +} diff --git a/vdev/src/commands/release/repositories/publisher/repository.rs b/vdev/src/commands/release/repositories/publisher/repository.rs new file mode 100644 index 0000000000000..3cfc82279eef5 --- /dev/null +++ b/vdev/src/commands/release/repositories/publisher/repository.rs @@ -0,0 +1,987 @@ +//! Native package validation, `OpenPGP` signing and incremental repository metadata. +use super::{ + inputs::{DEB_ARCHES, RPM_ARCHES, verify_input}, + model::{ + Config, Entry, Snapshot, copy, digest, entry_value, files, mutable, safe_relative, string, + write, + }, +}; +use anyhow::{Context, Result, ensure}; +use chrono::{DateTime, Timelike, Utc}; +use flate2::{Compression, GzBuilder}; +use roxmltree::{Document, Node}; +use serde_json::Value; +use sha2::{Digest, Sha512}; +use std::fmt::Write as _; +use std::{ + collections::{BTreeMap, BTreeSet}, + ffi::OsStr, + fs, + io::{Read, Write}, + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; + +pub(super) fn command(program: &str) -> Command { + let mut command = Command::new(program); + // Never leak signing material or cloud credentials to packaging subprocesses. + for name in [ + "PACKAGE_SIGNING_SUBKEY", + "PACKAGE_SIGNING_PASSPHRASE", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "GH_TOKEN", + ] { + command.env_remove(name); + } + command +} +pub(super) fn run(command: &mut Command, input: Option<&[u8]>) -> Result> { + command + .stdin(if input.is_some() { + Stdio::piped() + } else { + Stdio::null() + }) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let mut child = command + .spawn() + .with_context(|| format!("Cannot start {}", command.get_program().to_string_lossy()))?; + if let Some(input) = input { + child + .stdin + .take() + .context("Missing command stdin")? + .write_all(input)?; + } + let output = child.wait_with_output()?; + ensure!( + output.status.success(), + "{} failed: {}", + command.get_program().to_string_lossy(), + String::from_utf8_lossy(&output.stderr) + .chars() + .rev() + .take(4000) + .collect::() + .chars() + .rev() + .collect::() + ); + Ok(output.stdout) +} +fn text(command: &mut Command) -> Result { + Ok(String::from_utf8(run(command, None)?)?) +} +pub(super) fn fields(paragraph: &str) -> Result> { + let mut result = BTreeMap::new(); + for line in paragraph.lines() { + if line.starts_with([' ', '\t']) { + continue; + } + let (key, value) = line.split_once(':').context("Malformed package metadata")?; + ensure!( + result + .insert(key.to_owned(), value.trim().to_owned()) + .is_none(), + "Duplicate metadata field" + ); + } + Ok(result) +} +pub(super) fn merge_packages(previous: &[u8], incoming: &[u8]) -> Result> { + let mut packages: BTreeMap<(String, String, String), &str> = BTreeMap::new(); + for data in [previous, incoming] { + for paragraph in std::str::from_utf8(data)? + .trim() + .split("\n\n") + .filter(|p| !p.is_empty()) + { + let item = fields(paragraph)?; + ensure!( + item.get("Package").map(String::as_str) == Some("vector"), + "Foreign Debian package" + ); + let key = ( + item.get("Package").context("Missing name")?.clone(), + item.get("Version").context("Missing version")?.clone(), + item.get("Architecture") + .context("Missing architecture")? + .clone(), + ); + if let Some(old) = packages.get(&key) { + ensure!(fields(old)? == item, "Conflicting Debian identity"); + } + packages.insert(key, paragraph); + } + } + let mut result = Vec::new(); + for paragraph in packages.values() { + result.extend_from_slice(paragraph.as_bytes()); + result.extend_from_slice(b"\n\n"); + } + Ok(result) +} +pub(super) fn lint_package(path: &Path, identity: &Value) -> Result<()> { + if identity["format"] == "deb" { + for (field, expected) in [ + ("Package", "vector"), + ("Version", string(identity, "version")?), + ("Architecture", string(identity, "arch")?), + ] { + ensure!( + text(command("dpkg-deb").arg("-f").arg(path).arg(field))?.trim() == expected, + "Debian native identity mismatch" + ); + } + let mut child = command("dpkg-deb") + .arg("--fsys-tarfile") + .arg(path) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn()?; + let result = (|| -> Result<()> { + let mut archive = + tar::Archive::new(child.stdout.take().context("Missing dpkg stdout")?); + let mut size: u64 = 0; + for member in archive.entries()? { + let member = member?; + size = size + .checked_add(member.size()) + .context("Package size overflow")?; + ensure!(size < 8 * 1024 * 1024 * 1024, "Unreasonably large payload"); + let kind = member.header().entry_type(); + ensure!( + kind.is_symlink() + || kind.is_hard_link() + || member.header().mode()? & 0o002 == 0, + "World-writable Debian entry" + ); + } + Ok(()) + })(); + if result.is_err() { + drop(child.kill()); + } + let status = child.wait()?; + result?; + ensure!(status.success(), "Cannot inspect Debian payload"); + } else { + ensure!(identity["format"] == "rpm", "Unsupported package format"); + let actual = text( + command("rpm") + .args(["-qp", "--qf", "%{NAME} %{VERSION}-%{RELEASE} %{ARCH}"]) + .arg(path), + )?; + ensure!( + actual + == format!( + "vector {} {}", + string(identity, "version")?, + string(identity, "arch")? + ), + "RPM native identity mismatch" + ); + for line in text( + command("rpm") + .args(["-qp", "--qf", "[%{FILEMODES:perms} %{FILENAMES}\\n]"]) + .arg(path), + )? + .lines() + { + ensure!( + line.len() >= 10 && (line.starts_with('l') || line.as_bytes()[8] != b'w'), + "Invalid or world-writable RPM entry" + ); + } + } + Ok(()) +} +fn fingerprints(data: &[u8], primary: &str, subkey: &str) -> Result, String)>> { + let mut pending = None; + let mut result = Vec::new(); + for line in std::str::from_utf8(data)?.lines() { + let parts: Vec<_> = line.split(':').map(str::to_owned).collect(); + if parts[0] == primary || parts[0] == subkey { + pending = Some(parts); + } else if parts[0] == "fpr" + && let Some(key) = pending.take() + { + result.push((key, parts.get(9).context("Missing fingerprint")?.clone())); + } + } + Ok(result) +} +pub(super) struct PublicKey<'a> { + pub config: &'a Config, + pub home: PathBuf, +} +impl<'a> PublicKey<'a> { + pub fn new(config: &'a Config, name: &str) -> Result { + use std::os::unix::fs::DirBuilderExt as _; + let home = Path::new("/run/keys").join(name); + fs::DirBuilder::new().mode(0o700).create(&home)?; + let public = Self { config, home }; + let keys = fingerprints( + &public.gpg( + [ + OsStr::new("--with-colons"), + OsStr::new("--show-keys"), + config.key_path.as_os_str(), + ], + None, + )?, + "pub", + "sub", + )?; + ensure!( + keys.iter() + .filter(|(p, _)| p[0] == "pub") + .map(|(_, f)| f.as_str()) + .collect::>() + == [config.primary_fingerprint.as_str()], + "Wrong public primary key" + ); + ensure!( + keys.iter() + .any(|(p, f)| p[0] == "sub" && *f == config.signing_fingerprint), + "Wrong signing subkey" + ); + public.gpg([OsStr::new("--import"), config.key_path.as_os_str()], None)?; + Ok(public) + } + fn gpg_command(&self) -> Command { + let mut cmd = command("gpg"); + cmd.args(["--batch", "--no-tty"]) + .env("GNUPGHOME", &self.home); + cmd + } + pub fn gpg( + &self, + args: impl IntoIterator>, + input: Option<&[u8]>, + ) -> Result> { + run(self.gpg_command().args(args), input) + } + pub fn verify(&self, signature: &Path, data: Option<&Path>) -> Result<()> { + let mut cmd = self.gpg_command(); + cmd.args(["--status-fd", "1", "--verify"]).arg(signature); + if let Some(data) = data { + cmd.arg(data); + } + let status = text(&mut cmd)?; + ensure!( + status.lines().any(|l| l.starts_with(&format!( + "[GNUPG:] VALIDSIG {} ", + self.config.signing_fingerprint + ))), + "Signature did not use pinned subkey" + ); + Ok(()) + } + pub fn verify_apt_release(&self, suite: &Path) -> Result> { + self.verify(&suite.join("InRelease"), None)?; + self.verify(&suite.join("Release.gpg"), Some(&suite.join("Release")))?; + let release = fs::read(suite.join("Release"))?; + ensure!( + self.gpg( + [OsStr::new("--decrypt"), suite.join("InRelease").as_os_str()], + None + )? == release, + "Inline and detached roots disagree" + ); + Ok(release) + } + pub fn rpmdb(&self) -> Result { + let path = self.home.join("rpmdb"); + run( + command("rpm") + .arg("--dbpath") + .arg(&path) + .arg("--import") + .arg(&self.config.key_path), + None, + )?; + Ok(path) + } +} +impl Drop for PublicKey<'_> { + fn drop(&mut self) { + drop( + command("gpgconf") + .args(["--kill", "all"]) + .env("GNUPGHOME", &self.home) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(), + ); + drop(fs::remove_dir_all(&self.home)); + } +} +pub(super) struct Signer<'a> { + pub public: PublicKey<'a>, + password: PathBuf, + pub rpmdb: PathBuf, +} +impl<'a> Signer<'a> { + pub fn new(config: &'a Config, name: &str) -> Result { + use std::os::unix::fs::OpenOptionsExt as _; + let private = + std::env::var("PACKAGE_SIGNING_SUBKEY").context("Missing encrypted signing subkey")?; + let passphrase = + std::env::var("PACKAGE_SIGNING_PASSPHRASE").context("Missing signing passphrase")?; + ensure!( + !private.is_empty() && !passphrase.is_empty() && !passphrase.contains(['\r', '\n']), + "Expected encrypted subkey and single-line passphrase" + ); + let public = PublicKey::new(config, name)?; + let password = public.home.join("passphrase"); + let mut output = fs::OpenOptions::new() + .create_new(true) + .write(true) + .mode(0o600) + .open(&password)?; + writeln!(output, "{passphrase}")?; + drop(output); + public.gpg(["--import"], Some(private.as_bytes()))?; + let keys = fingerprints( + &public.gpg(["--with-colons", "--list-secret-keys"], None)?, + "sec", + "ssb", + )?; + ensure!( + keys.len() == 2, + "Expected exactly one primary stub and one signing subkey" + ); + let (primary, pf) = &keys[0]; + let (subkey, sf) = &keys[1]; + ensure!( + *pf == config.primary_fingerprint + && primary[0] == "sec" + && primary.get(14).map(String::as_str) == Some("#"), + "Usable primary secret key is forbidden in CI" + ); + ensure!( + *sf == config.signing_fingerprint + && subkey[0] == "ssb" + && subkey.get(14).map(String::as_str) == Some("+") + && subkey.get(11).is_some_and(|s| s.contains('s')), + "Wrong or unavailable signing subkey" + ); + for (parts, _) in &keys { + ensure!( + !matches!(parts.get(1).map(String::as_str), Some("r" | "e" | "d")) + && parts.get(6).context("Missing key expiry")?.parse::()? + > Utc::now().timestamp(), + "Expired or revoked signing identity" + ); + } + let challenge = public.home.join("challenge"); + fs::write(&challenge, b"Vector repository signing capability check\n")?; + run( + command("gpgconf") + .args(["--kill", "gpg-agent"]) + .env("GNUPGHOME", &public.home), + None, + )?; + let wrong = run( + public + .gpg_command() + .args([ + "--pinentry-mode", + "loopback", + "--passphrase-fd", + "0", + "--local-user", + ]) + .arg(format!("{}!", config.signing_fingerprint)) + .arg("--output") + .arg(public.home.join("wrong-passphrase.asc")) + .arg("--detach-sign") + .arg(&challenge), + Some( + format!( + "{}{}\n", + uuid::Uuid::new_v4().simple(), + uuid::Uuid::new_v4().simple() + ) + .as_bytes(), + ), + ); + ensure!( + wrong.is_err(), + "Signing subkey must be passphrase-protected" + ); + let rpmdb = public.rpmdb()?; + let signer = Self { + public, + password, + rpmdb, + }; + signer.sign(&challenge, false, &challenge.with_extension("asc"))?; + signer + .public + .verify(&challenge.with_extension("asc"), Some(&challenge))?; + Ok(signer) + } + pub fn sign(&self, source: &Path, clear: bool, destination: &Path) -> Result<()> { + run( + self.public + .gpg_command() + .args(["--yes", "--pinentry-mode", "loopback", "--passphrase-file"]) + .arg(&self.password) + .arg("--local-user") + .arg(format!("{}!", self.public.config.signing_fingerprint)) + .args(["--digest-algo", "SHA256", "--armor", "--output"]) + .arg(destination) + .arg(if clear { + "--clearsign" + } else { + "--detach-sign" + }) + .arg(source), + None, + )?; + Ok(()) + } + pub fn rpm(&self, package: &Path) -> Result<()> { + run( + command("rpmsign") + .arg("--define") + .arg(format!( + "_gpg_name {}!", + self.public.config.signing_fingerprint + )) + .arg("--define") + .arg(format!("_gpg_path {}", self.public.home.display())) + .args([ + "--define", + "__gpg /usr/bin/gpg", + "--define", + "_gpg_digest_algo sha256", + "--define", + ]) + .arg(format!( + "_gpg_sign_cmd_extra_args --pinentry-mode loopback --passphrase-file {}", + self.password.display() + )) + .arg("--addsign") + .arg(package) + .env("GNUPGHOME", &self.public.home), + None, + )?; + verify_rpm(&self.rpmdb, package) + } +} +pub(super) fn verify_rpm(database: &Path, package: &Path) -> Result<()> { + ensure!( + text( + command("rpm") + .arg("--dbpath") + .arg(database) + .arg("--checksig") + .arg(package) + )? + .contains("digests signatures OK"), + "RPM signature verification failed" + ); + Ok(()) +} +fn child<'a>(node: Node<'a, 'a>, tag: &str) -> Result> { + node.children() + .find(|n| n.is_element() && n.tag_name().name() == tag) + .with_context(|| format!("Missing XML {tag}")) +} +fn location<'a>(node: Node<'a, 'a>) -> Result<&'a str> { + let location = child(node, "location")?; + ensure!( + location.attributes().len() == 1, + "RPM locations must be relative" + ); + let name = location.attribute("href").context("Missing href")?; + safe_relative(&format!("yum/metadata/{name}"))?; + Ok(name) +} +pub(super) fn repomd_locations(root: &Path) -> Result> { + let data = fs::read_to_string(root.join("repodata/repomd.xml"))?; + let document = Document::parse(&data)?; + let mut result = Vec::new(); + for entry in document + .root_element() + .children() + .filter(|n| n.has_tag_name(("http://linux.duke.edu/metadata/repo", "data"))) + { + let name = location(entry)?; + ensure!( + name.starts_with("repodata/") && name.split('/').count() == 2, + "Invalid RPM metadata location" + ); + result.push(name.to_owned()); + } + ensure!(!result.is_empty(), "Empty RPM root metadata"); + Ok(result) +} +pub(super) fn check_apt_index(data: &[u8], snapshot: &Snapshot) -> Result<()> { + for paragraph in std::str::from_utf8(data)? + .trim() + .split("\n\n") + .filter(|p| !p.is_empty()) + { + let item = fields(paragraph)?; + ensure!( + item.get("Package").map(String::as_str) == Some("vector"), + "Foreign package in apt history" + ); + let name = format!( + "apt/{}", + item.get("Filename").context("Missing package filename")? + ); + safe_relative(&name)?; + let entry = Entry { + sha256: item + .get("SHA256") + .context("Missing package digest")? + .clone(), + size: item.get("Size").context("Missing package size")?.parse()?, + }; + ensure!( + snapshot.entries.get(&name) == Some(&entry), + "Debian history differs from manifest" + ); + } + Ok(()) +} +#[allow(clippy::too_many_lines)] // Keep the signed-root/index/payload checks together. +pub(super) fn verify_metadata( + snapshot: &Snapshot, + root: &Path, + public: &PublicKey<'_>, +) -> Result<()> { + for name in snapshot.entries.keys() { + if name.ends_with("/InRelease") { + let suite = root + .join(name) + .parent() + .context("Missing suite")? + .to_owned(); + let release = public.verify_apt_release(&suite)?; + let mut sha256 = false; + let mut checked = BTreeSet::new(); + for line in std::str::from_utf8(&release)?.lines() { + if !line.starts_with(' ') { + sha256 = line == "SHA256:"; + } else if sha256 { + let pieces: Vec<_> = line.split_whitespace().collect(); + ensure!(pieces.len() == 3, "Malformed Release checksum"); + let relative = pieces[2]; + safe_relative(&format!("apt/index/{relative}"))?; + let entry = Entry { + sha256: pieces[0].to_owned(), + size: pieces[1].parse()?, + }; + let path = suite.join(relative); + let filename = path + .strip_prefix(root)? + .to_str() + .context("Invalid index path")?; + ensure!( + snapshot.entries.get(filename) == Some(&entry), + "Apt index differs from signed root" + ); + if relative.ends_with("/Packages") { + let data = fs::read(path)?; + ensure!(Entry::bytes(&data) == entry, "Apt checksum mismatch"); + check_apt_index(&data, snapshot)?; + checked.insert(relative); + } + } + } + ensure!(!checked.is_empty(), "Apt root has no SHA-256 indexes"); + } else if name.ends_with("/repomd.xml") { + let path = root.join(name); + let package_root = path + .parent() + .and_then(Path::parent) + .context("Invalid RPM root")?; + public.verify( + &PathBuf::from(format!("{}.asc", path.display())), + Some(&path), + )?; + let data = fs::read_to_string(&path)?; + let document = Document::parse(&data)?; + for item in document + .root_element() + .children() + .filter(|n| n.is_element() && n.tag_name().name() == "data") + { + let path = package_root.join(location(item)?); + let checksum = child(item, "checksum")?; + let data = fs::read(&path)?; + let algorithm = checksum.attribute("type").context("Missing hash type")?; + ensure!( + matches!(algorithm, "sha256" | "sha512"), + "RPM metadata needs SHA-2" + ); + ensure!( + snapshot + .entries + .get(path.strip_prefix(root)?.to_str().context("Invalid path")?) + == Some(&Entry::bytes(&data)) + && hash(&data, algorithm) == checksum.text().context("Missing hash")?, + "RPM metadata differs from signed root" + ); + if item.attribute("type") == Some("primary") { + ensure!( + path.extension() == Some(OsStr::new("gz")), + "Expected gzip primary metadata" + ); + let mut expanded = String::new(); + super::model::gzip(data.as_slice(), 512 * 1024 * 1024 + 1) + .read_to_string(&mut expanded)?; + ensure!( + expanded.len() <= 512 * 1024 * 1024, + "RPM primary metadata exceeds extraction budget" + ); + let primary = Document::parse(&expanded)?; + for package in primary + .root_element() + .children() + .filter(|n| n.is_element() && n.tag_name().name() == "package") + { + ensure!( + child(package, "name")?.text() == Some("vector"), + "Foreign RPM package" + ); + let path = package_root.join(location(package)?); + let checksum = child(package, "checksum")?; + ensure!( + checksum.attribute("type") == Some("sha256"), + "RPM package needs SHA-256" + ); + let entry = Entry { + sha256: checksum.text().context("Missing checksum")?.to_owned(), + size: child(package, "size")? + .attribute("package") + .context("Missing package size")? + .parse()?, + }; + ensure!( + snapshot + .entries + .get(path.strip_prefix(root)?.to_str().context("Invalid path")?) + == Some(&entry), + "RPM package differs from signed index" + ); + } + } + } + } + } + Ok(()) +} +fn hash(data: &[u8], algorithm: &str) -> String { + if algorithm == "sha256" { + digest(data) + } else { + hex::encode(Sha512::digest(data)) + } +} +pub(super) fn release_date(data: &[u8]) -> Result> { + let metadata = fields(std::str::from_utf8(data)?)?; + ensure!( + !metadata.contains_key("Valid-Until"), + "Repository expiry requires a separate policy" + ); + Ok( + DateTime::parse_from_rfc2822(metadata.get("Date").context("Missing repository date")?)? + .with_timezone(&Utc), + ) +} +pub(super) fn next_release_date(previous: &[Vec], now: DateTime) -> Result { + let now = now.with_nanosecond(0).context("Invalid clock")?; + let mut date = now; + for data in previous { + date = date.max(release_date(data)? + chrono::Duration::seconds(1)); + } + ensure!( + date <= now + chrono::Duration::seconds(5), + "Repository date too far ahead; check clock and retry later" + ); + Ok(date.to_rfc2822()) +} +pub(super) fn dated_release(content: &[u8], date: &str) -> Result> { + release_date(content)?; + Ok(std::str::from_utf8(content)? + .split_inclusive('\n') + .map(|line| { + if line.starts_with("Date:") { + format!("Date: {date}\n") + } else { + line.to_owned() + } + }) + .collect::() + .into_bytes()) +} +fn apt_indexes( + root: &Path, + channel: &str, + incoming: &BTreeMap>, + previous: &Path, + signer: &Signer<'_>, +) -> Result<()> { + let suite = root.join("apt/dists").join(channel); + let architectures = ["amd64", "arm64", "armhf", "x86_64", "all", "i386"]; + debug_assert_eq!(&architectures[..3], &DEB_ARCHES); + for arch in architectures { + let relative = format!("apt/dists/{channel}/vector-0/binary-{arch}/Packages"); + let old = previous.join(&relative); + let data = merge_packages( + &if old.exists() { + fs::read(old)? + } else { + Vec::new() + }, + incoming + .get(if arch == "x86_64" { "amd64" } else { arch }) + .map_or(&[], Vec::as_slice), + )?; + let index = root.join(relative); + write(&index, &data)?; + let mut compressed = GzBuilder::new() + .mtime(0) + .write(Vec::new(), Compression::best()); + compressed.write_all(&data)?; + write(&index.with_extension("gz"), &compressed.finish()?)?; + for path in [&index, &index.with_extension("gz")] { + let content = fs::read(path)?; + for algorithm in ["sha256", "sha512"] { + write( + &path + .parent() + .context("Missing index parent")? + .join("by-hash") + .join(algorithm.to_uppercase()) + .join(hash(&content, algorithm)), + &content, + )?; + } + } + } + let prior = previous.join(format!("apt/dists/{channel}/Release")); + let timestamp = next_release_date( + &if prior.exists() { + vec![fs::read(prior)?] + } else { + Vec::new() + }, + Utc::now(), + )?; + // Keep the existing repository identity to avoid an additional apt approval. + let mut release = format!( + "Suite:\nCodename: {channel}\nDate: {timestamp}\nArchitectures: {}\nComponents: vector-0\nAcquire-By-Hash: yes\n", + architectures.join(" ") + ); + for algorithm in ["sha256", "sha512"] { + writeln!(release, "{}:", algorithm.to_uppercase())?; + for arch in architectures { + for filename in ["Packages", "Packages.gz"] { + let relative = format!("vector-0/binary-{arch}/{filename}"); + let data = fs::read(suite.join(&relative))?; + writeln!( + release, + " {} {} {relative}", + hash(&data, algorithm), + data.len() + )?; + } + } + } + write(&suite.join("Release"), release.as_bytes())?; + signer.sign(&suite.join("Release"), true, &suite.join("InRelease"))?; + signer.sign(&suite.join("Release"), false, &suite.join("Release.gpg"))?; + Ok(()) +} +#[allow(clippy::too_many_lines)] // Ordered admission, signing and final verification phase. +pub(super) fn prepare(config: &Config, work: &Path) -> Result { + let (manifest, input_id) = verify_input(work)?; + ensure!( + !work.join("result").exists(), + "Prepared output exists; reuse it or choose another workspace" + ); + let base_data = fs::read(work.join("base.json"))?; + let previous = if base_data == b"null\n" { + None + } else { + Some(Snapshot::decode(&base_data, None)?) + }; + let base_files = work.join("base"); + for (name, path) in files(&base_files)? { + ensure!( + previous.as_ref().and_then(|s| s.entries.get(&name)) == Some(&Entry::file(&path)?), + "Base metadata changed" + ); + } + let assets = manifest["assets"].as_object().context("Missing assets")?; + for (name, identity) in assets { + if identity.get("format").is_some() { + lint_package(&work.join("input").join(name), identity)?; + } + } + { + let public = PublicKey::new(config, "verify")?; + if let Some(previous) = &previous { + verify_metadata(previous, &base_files, &public)?; + } + } + let signer = Signer::new(config, "sign")?; + let result = work.join("result"); + let root = result.join("files"); + fs::create_dir_all(&root)?; + let mut entries = previous + .as_ref() + .map_or_else(BTreeMap::new, |s| s.entries.clone()); + for (name, source) in files(&base_files)? { + copy(&source, &root.join(name))?; + } + let channel = string(&manifest, "channel")?; + let mut incoming = BTreeMap::new(); + for (name, identity) in assets { + if identity.get("format").is_none() { + continue; + } + let is_deb = identity["format"] == "deb"; + let arch = string(identity, "arch")?; + let relative = if is_deb { + format!("apt/pool/v/ve/{name}") + } else { + format!("yum/{channel}/vector-0/{arch}/{name}") + }; + if let Some(entry) = entries.get(&relative) { + ensure!( + is_deb && *entry == entry_value(identity)?, + "Package already present; reuse signed publication" + ); + } + let destination = root.join(&relative); + copy(&work.join("input").join(name), &destination)?; + if is_deb { + incoming.insert( + arch.to_owned(), + run( + command("apt-ftparchive") + .arg("packages") + .arg( + relative + .strip_prefix("apt/") + .context("Invalid Debian path")?, + ) + .current_dir(root.join("apt")), + None, + )?, + ); + } else { + let cached = work.join("reuse").join(name); + if cached.exists() { + copy(&cached, &destination)?; + lint_package(&destination, identity)?; + verify_rpm(&signer.rpmdb, &destination)?; + } else { + signer.rpm(&destination)?; + } + } + } + let empty = BTreeMap::new(); + for selected in ["stable", "beta"] { + if previous.is_none() || selected == channel { + apt_indexes( + &root, + selected, + if selected == channel { + &incoming + } else { + &empty + }, + &base_files, + &signer, + )?; + for arch in RPM_ARCHES { + let relative = format!("yum/{selected}/vector-0/{arch}"); + let target = root.join(&relative); + fs::create_dir_all(&target)?; + if target.join("repodata").exists() { + fs::remove_dir_all(target.join("repodata"))?; + } + run( + command("createrepo_c") + .args([ + "--no-database", + "--checksum", + "sha256", + "--compress-type", + "gz", + ]) + .arg(&target), + None, + )?; + let old = base_files.join(relative); + if old.join("repodata/repomd.xml").exists() { + let temporary = tempfile::tempdir()?; + let uri = |path: &Path| -> Result { + Ok(reqwest::Url::from_file_path(path.canonicalize()?) + .map_err(|()| anyhow::anyhow!("Invalid file URI"))? + .to_string()) + }; + run( + command("mergerepo_c") + .args([ + "--all", + "--omit-baseurl", + "--no-database", + "--compress-type", + "gz", + "--repo", + ]) + .arg(uri(&old)?) + .arg("--repo") + .arg(uri(&target)?) + .arg("--outputdir") + .arg(temporary.path()), + None, + )?; + fs::remove_dir_all(target.join("repodata"))?; + for (name, path) in files(&temporary.path().join("repodata"))? { + copy(&path, &target.join("repodata").join(name))?; + } + } + repomd_locations(&target)?; + signer.sign( + &target.join("repodata/repomd.xml"), + false, + &target.join("repodata/repomd.xml.asc"), + )?; + } + } + } + for (name, path) in files(&root)? { + let entry = Entry::file(&path)?; + ensure!( + mutable(&name) || entries.get(&name).is_none_or(|old| *old == entry), + "Immutable object changed" + ); + entries.insert(name, entry); + } + let snapshot = Snapshot::new( + entries, + previous.map(|s| s.id), + Some(input_id), + root.clone(), + )?; + verify_metadata(&snapshot, &root, &signer.public)?; + write(&result.join("manifest.json"), &snapshot.data)?; + copy(&work.join("input.json"), &result.join("input.json"))?; + Snapshot::local(&root)?; + Ok(snapshot.id) +} diff --git a/vdev/src/commands/release/repositories/publisher/rollback.rs b/vdev/src/commands/release/repositories/publisher/rollback.rs new file mode 100644 index 0000000000000..efff6dfd417be --- /dev/null +++ b/vdev/src/commands/release/repositories/publisher/rollback.rs @@ -0,0 +1,326 @@ +//! Restore a historical selection with fresh, monotonically dated apt roots. +use super::{ + bundles::{checked_download, unpack_bundle}, + engine::{download_metadata, save_bundle}, + model::{Config, Entry, Snapshot, canonical, copy, digest, files, is_digest, mutable, write}, + repository::{ + PublicKey, Signer, dated_release, next_release_date, release_date, verify_metadata, + }, + storage::{Publisher, S3Store, Store}, +}; +use anyhow::{Context, Result, ensure}; +use chrono::Utc; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + path::{Path, PathBuf}, +}; +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Plan { + schema: u32, + operation: String, + pub target: String, + pub active: String, + pub pending: Option, +} +impl Plan { + pub fn decode(data: &[u8]) -> Result<(Self, String)> { + let plan: Self = serde_json::from_slice(data)?; + ensure!( + plan.schema == 1 && plan.operation == "rollback" && canonical(&plan)? == data, + "Invalid rollback request" + ); + ensure!( + is_digest(&plan.target) + && is_digest(&plan.active) + && plan.pending.as_ref().is_none_or(|s| is_digest(s)), + "Invalid rollback snapshot identity" + ); + ensure!( + plan.pending.is_none() || plan.target == plan.active, + "Abandoning pending publication must restore last completed selection" + ); + Ok((plan, digest(data))) + } + fn source_ids(&self) -> BTreeSet<&str> { + [ + Some(self.target.as_str()), + Some(self.active.as_str()), + self.pending.as_deref(), + ] + .into_iter() + .flatten() + .collect() + } +} +fn request_key(config: &Config, input: &str) -> String { + format!("{}rollback-requests/{input}.json", config.bundle_prefix) +} +fn bundle_key(config: &Config, input: &str) -> String { + format!("{}rollbacks/{input}.tar.gz", config.bundle_prefix) +} +pub(super) fn base(config: &Config, work: &Path, target: &str) -> Result<()> { + let store = S3Store::new(config)?; + let publisher = Publisher::new(&store, config); + let (state, _) = publisher.load_state()?; + ensure!(!state.held, "Rollback requires unlocked repository"); + let active = state + .active + .context("Rollback requires initialized repository")?; + let candidate = publisher.archived(state.pending.as_deref().unwrap_or(&active))?; + let mut saved = None; + if let Some(input) = &candidate.input { + let key = request_key(config, input); + if store.head(&key)?.is_some() { + let (data, _) = store.read(&key)?; + let (plan, identity) = Plan::decode(&data)?; + ensure!( + identity == *input, + "Saved rollback request checksum mismatch" + ); + if plan.target == target { + saved = Some(data); + } + } + } + let retry = saved.is_some(); + let data = saved.unwrap_or(canonical(&Plan { + schema: 1, + operation: "rollback".into(), + target: target.into(), + active, + pending: state.pending, + })?); + let (plan, input) = Plan::decode(&data)?; + write(&work.join("input.json"), &data)?; + for identity in plan.source_ids() { + let snapshot = publisher.archived(identity)?; + let directory = work.join("sources").join(identity); + ensure!(!directory.exists(), "Rollback source already exists"); + write(&directory.join("manifest.json"), &snapshot.data)?; + download_metadata( + &publisher, + &snapshot, + &directory.join("files"), + identity == plan.active || identity == plan.target, + )?; + } + let key = bundle_key(config, &input); + if store.head(&key)?.is_some() { + let archive = work.join("cached.tar.gz"); + checked_download(&store, &key, &archive, None)?; + let snapshot = unpack_bundle(&archive, &work.join("result"))?; + ensure!( + snapshot.input.as_deref() == Some(&input) + && snapshot.base.as_deref() == Some(&plan.active) + && fs::read(work.join("result/input.json"))? == data, + "Saved rollback belongs to another request" + ); + ensure!( + !retry || snapshot.id == candidate.id, + "Saved rollback differs from pending/completed generation" + ); + fs::remove_file(archive)?; + println!("Restored saved signed rollback; signing skipped"); + } else { + ensure!( + candidate.input.as_deref() != Some(&input), + "Required rollback retry bundle missing" + ); + println!("Downloaded archived selections and potentially exposed dates"); + } + Ok(()) +} +fn check_sources( + config: &Config, + work: &Path, +) -> Result<(Plan, String, BTreeMap)> { + let (plan, input) = Plan::decode(&fs::read(work.join("input.json"))?)?; + let mut sources = BTreeMap::new(); + let public = PublicKey::new(config, "rollback-sources")?; + for identity in plan.source_ids() { + let directory = work.join("sources").join(identity); + let mut snapshot = + Snapshot::decode(&fs::read(directory.join("manifest.json"))?, Some(identity))?; + let root = directory.join("files"); + let present = files(&root)?; + for (name, path) in &present { + ensure!( + !name.ends_with(".deb") + && !name.ends_with(".rpm") + && snapshot.entries.get(name) == Some(&Entry::file(path)?), + "Rollback source differs from metadata manifest" + ); + } + ensure!( + snapshot + .entries + .keys() + .filter(|n| mutable(n)) + .all(|n| present.contains_key(n)), + "Missing mutable rollback metadata" + ); + if identity == plan.active || identity == plan.target { + verify_metadata(&snapshot, &root, &public)?; + } else { + for (name, path) in &present { + if name.ends_with("/Release") { + let suite = path.parent().context("Missing suite")?; + public.verify_apt_release(suite)?; + } else if name.ends_with("/repomd.xml") { + public.verify( + &PathBuf::from(format!("{}.asc", path.display())), + Some(path), + )?; + } + } + } + snapshot.source = root; + sources.insert(identity.to_owned(), snapshot); + } + let selected = &sources[&plan.target]; + let names: BTreeSet<_> = selected.entries.keys().filter(|n| mutable(n)).collect(); + for source in sources.values() { + ensure!( + source + .entries + .keys() + .filter(|n| mutable(n)) + .collect::>() + == names, + "Rollback across layout changes requires a separate migration" + ); + } + let active = &sources[&plan.active]; + ensure!( + selected + .entries + .iter() + .filter(|(n, _)| !mutable(n)) + .all(|(n, e)| active.entries.get(n) == Some(e)), + "Selection absent from active immutable history" + ); + Ok((plan, input, sources)) +} +fn expected_entries(plan: &Plan, sources: &BTreeMap) -> BTreeMap { + let mut entries = sources[&plan.target].entries.clone(); + entries.extend( + sources[&plan.active] + .entries + .iter() + .filter(|(n, _)| !mutable(n)) + .map(|(n, e)| (n.clone(), e.clone())), + ); + entries +} +pub(super) fn prepare(config: &Config, work: &Path) -> Result { + let (plan, input, sources) = check_sources(config, work)?; + let selected = &sources[&plan.target]; + let root = work.join("result/files"); + ensure!( + !work.join("result").exists(), + "Prepared rollback already exists" + ); + fs::create_dir_all(&root)?; + for (name, path) in files(&selected.source)? { + copy(&path, &root.join(name))?; + } + let mut entries = expected_entries(&plan, &sources); + { + let signer = Signer::new(config, "rollback-signing")?; + for name in selected.entries.keys().filter(|n| n.ends_with("/Release")) { + let previous = sources + .values() + .map(|s| fs::read(s.source.join(name))) + .collect::>>()?; + let date = next_release_date(&previous, Utc::now())?; + let release = root.join(name); + let suite = release.parent().context("Missing suite")?; + fs::write(&release, dated_release(&fs::read(&release)?, &date)?)?; + signer.sign(&release, true, &suite.join("InRelease"))?; + signer.sign(&release, false, &suite.join("Release.gpg"))?; + } + } + for (name, path) in files(&root)? { + entries.insert(name, Entry::file(&path)?); + } + let snapshot = Snapshot::new(entries, Some(plan.active), Some(input), root)?; + write(&work.join("result/manifest.json"), &snapshot.data)?; + copy(&work.join("input.json"), &work.join("result/input.json"))?; + check_result(config, work)?; + Ok(snapshot.id) +} +pub(super) fn check_result(config: &Config, work: &Path) -> Result<(Plan, Snapshot)> { + let (plan, input, sources) = check_sources(config, work)?; + let snapshot = Snapshot::local(&work.join("result/files"))?; + ensure!( + snapshot.input.as_deref() == Some(&input) + && snapshot.base.as_deref() == Some(&plan.active) + && fs::read(work.join("result/input.json"))? == fs::read(work.join("input.json"))?, + "Signed rollback belongs to another request" + ); + let expected = expected_entries(&plan, &sources); + ensure!( + snapshot.entries.keys().eq(expected.keys()), + "Rollback changed file set" + ); + for (name, entry) in expected { + if !["/Release", "/InRelease", "/Release.gpg"] + .iter() + .any(|suffix| name.ends_with(suffix)) + { + ensure!( + snapshot.entries[&name] == entry, + "Rollback changed package bytes or indexes" + ); + } + if name.ends_with("/Release") { + let data = fs::read(snapshot.source.join(&name))?; + let timestamp = release_date(&data)?; + for source in sources.values() { + ensure!( + timestamp > release_date(&fs::read(source.source.join(&name))?)?, + "Rollback date must exceed every exposed root" + ); + } + ensure!( + timestamp <= Utc::now() + chrono::Duration::seconds(5), + "Rollback dated too far ahead" + ); + ensure!( + dated_release(&data, "DATE")? + == dated_release(&fs::read(sources[&plan.target].source.join(&name))?, "DATE")?, + "Rollback changed apt identity or indexes" + ); + } + } + let public = PublicKey::new(config, "rollback-result")?; + verify_metadata(&snapshot, &snapshot.source, &public)?; + Ok((plan, snapshot)) +} +pub(super) fn publish(config: &Config, work: &Path) -> Result<()> { + let (plan, snapshot) = check_result(config, work)?; + let store = S3Store::new(config)?; + let mut publisher = Publisher::new(&store, config); + for id in plan.source_ids() { + ensure!( + publisher.archived(id)?.data + == fs::read(work.join("sources").join(id).join("manifest.json"))?, + "Rollback source differs from saved archive" + ); + } + let input = snapshot + .input + .as_deref() + .context("Missing rollback identity")?; + save_bundle(&publisher, work, &bundle_key(config, input))?; + publisher.ensure_bytes( + &request_key(config, input), + &fs::read(work.join("input.json"))?, + )?; + publisher.publish(&snapshot, Some((&plan.active, plan.pending.as_deref())))?; + println!("Published fresh rollback snapshot {}", snapshot.id); + Ok(()) +} diff --git a/vdev/src/commands/release/repositories/publisher/storage.rs b/vdev/src/commands/release/repositories/publisher/storage.rs new file mode 100644 index 0000000000000..bf58a42150bd6 --- /dev/null +++ b/vdev/src/commands/release/repositories/publisher/storage.rs @@ -0,0 +1,692 @@ +//! Conditional S3 publication. Ambiguous writes deliberately retain the lock. +use super::model::{Config, Entry, Snapshot, canonical, is_digest, mutable, root_order}; +use anyhow::{Context, Result, ensure}; +use aws_sdk_s3::{ + Client, + config::{Credentials, Region, retry::RetryConfig, timeout::TimeoutConfig}, + primitives::ByteStream, + types::{ChecksumMode, ServerSideEncryption}, +}; +use aws_smithy_http_client::{ + Builder, + tls::{Provider, rustls_provider::CryptoMode}, +}; +use aws_smithy_runtime_api::client::{ + interceptors::{Intercept, context::BeforeDeserializationInterceptorContextRef}, + runtime_components::RuntimeComponents, +}; +use aws_smithy_types::config_bag::ConfigBag; +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + collections::BTreeMap, + fs, + io::Write, + path::Path, + sync::{Arc, Mutex}, + time::Duration, +}; + +#[derive(Debug, Clone)] +struct ServerClock(Arc>>>); +impl Intercept for ServerClock { + fn name(&self) -> &'static str { + "PublicationServerClock" + } + fn read_before_deserialization( + &self, + context: &BeforeDeserializationInterceptorContextRef<'_>, + _: &RuntimeComponents, + _: &mut ConfigBag, + ) -> Result<(), aws_smithy_runtime_api::box_error::BoxError> { + let date = context + .response() + .headers() + .get("date") + .ok_or("AWS response has no server date")?; + *self.0.lock().map_err(|_| "Server clock poisoned")? = + Some(DateTime::parse_from_rfc2822(date)?.with_timezone(&Utc)); + Ok(()) + } +} +#[derive(Clone, Debug)] +pub(super) struct Head { + pub entry: Entry, + pub etag: String, + pub version: String, +} +pub(super) enum Body<'a> { + Bytes(&'a [u8]), + File(&'a Path), +} +pub(super) trait Store { + fn head(&self, key: &str) -> Result>; + fn read(&self, key: &str) -> Result<(Vec, Head)>; + fn heads(&self, keys: &[String]) -> Result>> { + keys.iter().map(|key| self.head(key)).collect() + } + fn put( + &self, + key: &str, + body: Body<'_>, + entry: &Entry, + previous: Option<&Head>, + immutable: bool, + ) -> Result; + fn download(&self, key: &str, path: &Path, limit: u64) -> Result; + fn server_time(&self) -> Result>; + fn owner(&self) -> &str; +} +pub(super) struct S3Store { + runtime: tokio::runtime::Runtime, + client: Client, + bucket: String, + live: String, + pub owner: String, + clock: ServerClock, +} +fn checksum(value: Option<&str>, size: Option) -> Result { + let hash = STANDARD.decode(value.context("Missing stored SHA-256 checksum")?)?; + ensure!(hash.len() == 32, "Invalid stored SHA-256 checksum"); + Ok(Entry { + sha256: hex::encode(hash), + size: u64::try_from(size.context("Missing object size")?)?, + }) +} +impl S3Store { + pub fn new(config: &Config) -> Result { + let role_arn = std::env::var("PACKAGE_REPOSITORIES_ROLE_ARN") + .context("Set the publishing role ARN")?; + ensure!( + regex::Regex::new(&format!( + r"^arn:aws:iam::{}:role/[A-Za-z0-9_+=,.@-]+$", + config.account + ))? + .is_match(&role_arn), + "Wrong publishing role ARN" + ); + // Snapshot one temporary session, with no credential-provider renewal. + let required = |name| -> Result { + let value = std::env::var(name).context("A fixed temporary AWS session is required")?; + ensure!(!value.is_empty(), "Empty AWS credential"); + Ok(value) + }; + let credentials = Credentials::new( + required("AWS_ACCESS_KEY_ID")?, + required("AWS_SECRET_ACCESS_KEY")?, + Some(required("AWS_SESSION_TOKEN")?), + None, + "fixed-publisher-session", + ); + let runtime = tokio::runtime::Runtime::new()?; + let http_client = Builder::new() + .tls_provider(Provider::Rustls(CryptoMode::Ring)) + .build_https(); + let sdk = runtime.block_on( + aws_config::defaults(aws_config::BehaviorVersion::latest()) + .http_client(http_client) + .region(Region::new(config.region.clone())) + .credentials_provider(credentials) + .retry_config(RetryConfig::disabled()) + .timeout_config( + TimeoutConfig::builder() + .connect_timeout(Duration::from_secs(10)) + .read_timeout(Duration::from_secs(90)) + .operation_timeout(Duration::from_mins(5)) + .build(), + ) + .load(), + ); + let identity = + runtime.block_on(aws_sdk_sts::Client::new(&sdk).get_caller_identity().send())?; + let role = role_arn.rsplit('/').next().context("Invalid role")?; + let owner = identity.arn().context("Missing caller ARN")?.to_owned(); + ensure!( + identity.account() == Some(config.account.as_str()) + && owner.starts_with(&format!( + "arn:aws:sts::{}:assumed-role/{role}/", + config.account + )), + "Wrong AWS account or role" + ); + let policy = runtime.block_on( + aws_sdk_iam::Client::new(&sdk) + .get_role() + .role_name(role) + .send(), + )?; + ensure!( + policy + .role() + .context("Missing role")? + .max_session_duration() + .is_some_and(|seconds| seconds <= 3600), + "Role exceeds the one-hour recovery bound" + ); + let clock = ServerClock(Arc::new(Mutex::new(None))); + let client = Client::from_conf( + aws_sdk_s3::config::Builder::from(&sdk) + .interceptor(clock.clone()) + .build(), + ); + Ok(Self { + runtime, + client, + bucket: config.bucket.clone(), + live: config.live_prefix.clone(), + owner, + clock, + }) + } +} +impl Store for S3Store { + fn heads(&self, keys: &[String]) -> Result>> { + if keys.is_empty() { + return Ok(Vec::new()); + } + // Join every request before returning a conflict and releasing the lock. + std::thread::scope(|scope| { + let workers: Vec<_> = keys + .chunks(keys.len().div_ceil(16)) + .map(|chunk| { + scope.spawn(move || { + chunk + .iter() + .map(|key| self.head(key)) + .collect::>>() + }) + }) + .collect(); + let completed: Vec<_> = workers + .into_iter() + .map(|worker| { + worker + .join() + .map_err(|_| anyhow::anyhow!("S3 inspection worker panicked"))? + }) + .collect(); + Ok(completed + .into_iter() + .collect::>>()? + .into_iter() + .flatten() + .collect()) + }) + } + fn owner(&self) -> &str { + &self.owner + } + fn server_time(&self) -> Result> { + self.clock + .0 + .lock() + .map_err(|_| anyhow::anyhow!("Server clock poisoned"))? + .context("No AWS server time") + } + fn head(&self, key: &str) -> Result> { + let result = self.runtime.block_on( + self.client + .head_object() + .bucket(&self.bucket) + .key(key) + .checksum_mode(ChecksumMode::Enabled) + .send(), + ); + let output = match result { + Ok(value) => value, + Err(error) => { + let status = error.raw_response().map(|r| r.status().as_u16()); + if status == Some(404) { + return Ok(None); + } + if status == Some(403) { + let listed = self.runtime.block_on( + self.client + .list_objects_v2() + .bucket(&self.bucket) + .prefix(key) + .max_keys(1) + .send(), + )?; + if listed.contents().iter().all(|o| o.key() != Some(key)) { + return Ok(None); + } + } + return Err(error.into()); + } + }; + Ok(Some(Head { + entry: checksum(output.checksum_sha256(), output.content_length())?, + etag: output.e_tag().context("Missing ETag")?.to_owned(), + version: output + .version_id() + .context("Bucket versioning is required")? + .to_owned(), + })) + } + fn read(&self, key: &str) -> Result<(Vec, Head)> { + let output = self.runtime.block_on( + self.client + .get_object() + .bucket(&self.bucket) + .key(key) + .checksum_mode(ChecksumMode::Enabled) + .send(), + )?; + let head = Head { + entry: checksum(output.checksum_sha256(), output.content_length())?, + etag: output.e_tag().context("Missing ETag")?.to_owned(), + version: output + .version_id() + .context("Bucket versioning is required")? + .to_owned(), + }; + ensure!(head.entry.size < super::model::LIMIT, "Oversized S3 object"); + let data = self + .runtime + .block_on(output.body.collect())? + .into_bytes() + .to_vec(); + ensure!( + Entry::bytes(&data) == head.entry, + "S3 read failed checksum verification" + ); + Ok((data, head)) + } + fn download(&self, key: &str, destination: &Path, limit: u64) -> Result { + ensure!(!destination.is_symlink(), "Invalid download destination"); + let output = self.runtime.block_on( + self.client + .get_object() + .bucket(&self.bucket) + .key(key) + .checksum_mode(ChecksumMode::Enabled) + .send(), + )?; + let expected = checksum(output.checksum_sha256(), output.content_length())?; + ensure!(expected.size <= limit, "S3 object grew before download"); + let mut target = fs::File::create(destination)?; + let mut digest = Sha256::new(); + let mut size = 0; + self.runtime.block_on(async { + let mut body = output.body; + while let Some(chunk) = body.try_next().await? { + size += chunk.len() as u64; + ensure!(size <= limit, "Oversized download"); + digest.update(&chunk); + target.write_all(&chunk)?; + } + Ok::<_, anyhow::Error>(()) + })?; + let actual = Entry { + sha256: hex::encode(digest.finalize()), + size, + }; + ensure!(actual == expected, "S3 body failed checksum verification"); + Ok(actual) + } + fn put( + &self, + key: &str, + body: Body<'_>, + entry: &Entry, + previous: Option<&Head>, + immutable: bool, + ) -> Result { + let body = match body { + Body::Bytes(bytes) => ByteStream::from(bytes.to_owned()), + Body::File(path) => self.runtime.block_on(ByteStream::from_path(path))?, + }; + let mut request = self + .client + .put_object() + .bucket(&self.bucket) + .key(key) + .body(body) + .content_length(i64::try_from(entry.size)?) + .checksum_sha256(STANDARD.encode(hex::decode(&entry.sha256)?)) + .server_side_encryption(ServerSideEncryption::Aes256) + .content_type(content_type(key)) + .cache_control(format!( + "{},{}", + if key.starts_with(&self.live) { + "public" + } else { + "private" + }, + if immutable { + "max-age=31536000,immutable" + } else { + "max-age=0,no-cache,must-revalidate" + } + )); + request = if let Some(head) = previous { + request.if_match(&head.etag) + } else { + request.if_none_match("*") + }; + let output = self.runtime.block_on(request.send())?; + Ok(Head { + entry: entry.clone(), + etag: output.e_tag().context("Missing ETag")?.to_owned(), + version: output + .version_id() + .context("Bucket versioning is required")? + .to_owned(), + }) + } +} +fn content_type(key: &str) -> &'static str { + if key.ends_with(".deb") { + "application/vnd.debian.binary-package" + } else if key.ends_with(".rpm") { + "application/x-rpm" + } else if key.ends_with(".json") { + "application/json" + } else if key.ends_with(".gz") { + "application/gzip" + } else if key.ends_with(".xml") { + "application/xml" + } else { + "application/octet-stream" + } +} +#[derive(Debug)] +struct Conflict(String); +impl std::fmt::Display for Conflict { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} +impl std::error::Error for Conflict {} +fn conflict(message: impl Into) -> anyhow::Error { + Conflict(message.into()).into() +} +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(super) struct State { + schema: u32, + pub held: bool, + pub active: Option, + pub pending: Option, + #[serde(skip_serializing_if = "Option::is_none", default)] + owner: Option, + #[serde(skip_serializing_if = "Option::is_none", default)] + recover_after: Option>, + #[serde(skip_serializing_if = "Option::is_none", default)] + nonce: Option, +} +pub(super) struct Publisher<'a> { + pub store: &'a dyn Store, + pub config: &'a Config, + state: Option, + head: Option, +} +impl<'a> Publisher<'a> { + pub fn new(store: &'a dyn Store, config: &'a Config) -> Self { + Self { + store, + config, + state: None, + head: None, + } + } + pub fn load_state(&self) -> Result<(State, Option)> { + if self.store.head(&self.config.control_key)?.is_none() { + return Ok(( + State { + schema: 1, + held: false, + active: None, + pending: None, + owner: None, + recover_after: None, + nonce: None, + }, + None, + )); + } + let (data, head) = self.store.read(&self.config.control_key)?; + let state: State = serde_json::from_slice(&data)?; + ensure!( + state.schema == 1 + && [&state.active, &state.pending] + .into_iter() + .flatten() + .all(|s| is_digest(s)), + "Invalid publication state" + ); + ensure!( + !state.held || (state.recover_after.is_some() && state.owner.is_some()), + "Incomplete held lock" + ); + Ok((state, Some(head))) + } + fn write_state(&mut self) -> Result<()> { + let state = self.state.as_mut().context("Lock not initialized")?; + state.nonce = Some(uuid::Uuid::new_v4().simple().to_string()); + let data = canonical(state)?; + self.head = Some(self.store.put( + &self.config.control_key, + Body::Bytes(&data), + &Entry::bytes(&data), + self.head.as_ref(), + false, + )?); + Ok(()) + } + fn acquire(&mut self) -> Result<()> { + let (mut state, head) = self.load_state()?; + ensure!( + !state.held, + "Publisher busy; locks are never stolen on timeout" + ); + state.held = true; + state.owner = Some(self.store.owner().to_owned()); + state.recover_after = Some(self.store.server_time()? + chrono::Duration::minutes(65)); + self.state = Some(state); + self.head = head; + self.write_state() + } + fn release(&mut self) -> Result<()> { + self.state.as_mut().context("No lock")?.held = false; + self.write_state() + } + pub fn recover_lock(&mut self, version: &str, owner_stopped: bool) -> Result<()> { + let (state, head) = self.load_state()?; + ensure!( + owner_stopped, + "Confirm that the previous workflow/process has stopped" + ); + ensure!( + state.held && head.as_ref().is_some_and(|h| h.version == version), + "Lock version changed or is released" + ); + ensure!( + self.store.server_time()? >= state.recover_after.context("Missing recovery time")?, + "Recovery waits 65 minutes for the previous fixed STS session to expire" + ); + self.state = Some(state); + self.head = head; + self.release() + } + pub fn ensure_bytes(&self, key: &str, data: &[u8]) -> Result<()> { + self.ensure_immutable(key, Body::Bytes(data), &Entry::bytes(data)) + } + pub fn ensure_file(&self, key: &str, path: &Path, entry: &Entry) -> Result<()> { + self.ensure_immutable(key, Body::File(path), entry) + } + fn ensure_immutable(&self, key: &str, body: Body<'_>, entry: &Entry) -> Result<()> { + if let Some(head) = self.store.head(key)? { + if head.entry != *entry { + return Err(conflict(format!("Immutable object conflict: {key}"))); + } + return Ok(()); + } + self.store.put(key, body, entry, None, true)?; + Ok(()) + } + pub fn archived(&self, id: &str) -> Result { + ensure!(is_digest(id), "Invalid snapshot ID"); + Snapshot::decode( + &self + .store + .read(&format!("{}{id}/manifest.json", self.config.archive_prefix))? + .0, + Some(id), + ) + } + fn preflight(&self, snapshot: &Snapshot) -> Result<()> { + let active = &self.state.as_ref().context("No lock")?.active; + if active.as_deref() != Some(&snapshot.id) { + if *active != snapshot.base { + return Err(conflict( + "Repository advanced after preparation; regenerate against current base", + )); + } + if let Some(id) = active { + for (name, entry) in self.archived(id)?.entries { + if !mutable(&name) && snapshot.entries.get(&name) != Some(&entry) { + return Err(conflict(format!( + "Publication would remove historical data: {name}" + ))); + } + } + } + } + let keys: Vec<_> = snapshot + .entries + .keys() + .map(|name| format!("{}{name}", self.config.live_prefix)) + .collect(); + let heads = self.store.heads(&keys)?; + for ((name, entry), old) in snapshot.entries.iter().zip(heads) { + if !mutable(name) { + if old.as_ref().is_some_and(|h| h.entry != *entry) { + return Err(conflict(format!("Immutable object conflict: {name}"))); + } + if old.is_none() && !snapshot.source.join(name).is_file() { + return Err(conflict(format!("Inherited object is missing: {name}"))); + } + } + } + Ok(()) + } + fn publish_locked( + &mut self, + snapshot: &Snapshot, + recovery: Option<(&str, Option<&str>)>, + ) -> Result<()> { + let state = self.state.as_ref().context("No lock")?; + if let Some((active, pending)) = recovery { + let pair = (state.active.as_deref(), state.pending.as_deref()); + if snapshot.base.as_deref() != Some(active) + || ![ + (Some(active), pending), + (Some(active), Some(snapshot.id.as_str())), + (Some(snapshot.id.as_str()), None), + ] + .contains(&pair) + { + return Err(conflict("Repository changed after rollback preparation")); + } + } else if state.pending.as_ref().is_some_and(|p| p != &snapshot.id) { + return Err(conflict("Recover pending publication or roll back")); + } + self.preflight(snapshot)?; + let archive = format!("{}{}/", self.config.archive_prefix, snapshot.id); + for (name, entry) in &snapshot.entries { + if mutable(name) { + self.ensure_file( + &format!("{archive}files/{name}"), + &snapshot.source.join(name), + entry, + )?; + } + } + self.ensure_bytes(&format!("{archive}manifest.json"), &snapshot.data)?; + let mut metadata = BTreeMap::new(); + for (name, entry) in &snapshot.entries { + if mutable(name) { + let data = self.store.read(&format!("{archive}files/{name}"))?.0; + if Entry::bytes(&data) != *entry { + return Err(conflict(format!( + "Archived metadata checksum mismatch: {name}" + ))); + } + metadata.insert(name, data); + } + } + self.state.as_mut().context("No lock")?.pending = Some(snapshot.id.clone()); + self.write_state()?; + for (name, entry) in &snapshot.entries { + if !mutable(name) && snapshot.source.join(name).exists() { + self.ensure_file( + &format!("{}{name}", self.config.live_prefix), + &snapshot.source.join(name), + entry, + )?; + } + } + let mut names: Vec<_> = metadata.keys().copied().collect(); + names.sort_by_key(|name| { + ( + if name.ends_with("/Packages") || name.ends_with("/Packages.gz") { + 0 + } else { + 1 + root_order(name) + }, + *name, + ) + }); + for name in names { + let key = format!("{}{name}", self.config.live_prefix); + let entry = &snapshot.entries[name]; + let previous = self.store.head(&key)?; + if previous.as_ref().is_some_and(|h| h.entry == *entry) { + continue; + } + self.store.put( + &key, + Body::Bytes(&metadata[name]), + entry, + previous.as_ref(), + false, + )?; + } + let keys: Vec<_> = snapshot + .entries + .keys() + .map(|name| format!("{}{name}", self.config.live_prefix)) + .collect(); + for ((name, entry), head) in snapshot.entries.iter().zip(self.store.heads(&keys)?) { + ensure!( + head.is_some_and(|h| h.entry == *entry), + "Published object failed final verification: {name}" + ); + } + let state = self.state.as_mut().context("No lock")?; + state.active = Some(snapshot.id.clone()); + state.pending = None; + self.write_state() + } + pub fn publish( + &mut self, + snapshot: &Snapshot, + recovery: Option<(&str, Option<&str>)>, + ) -> Result<()> { + self.acquire()?; + match self.publish_locked(snapshot, recovery) { + Ok(()) => self.release(), + Err(error) if error.is::() => { + self.release()?; + Err(error) + } + Err(error) => { + Err(error.context("Publication stopped; lock remains held for operator recovery")) + } + } + } +} From 15f76d6338eecb0320eb806766da0fe532f772c2 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 26 Oct 2026 13:45:36 -0400 Subject: [PATCH 18/23] feat(vdev): make package repository commands opt in --- .github/workflows/test.yml | 8 +++++++- vdev/Cargo.toml | 5 +++-- vdev/README.md | 8 ++++++++ vdev/src/commands/release/mod.rs | 3 +++ vdev/src/commands/release/repositories.rs | 4 ---- 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f959cee482e0f..35a9313ddd0bd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -81,8 +81,14 @@ jobs: persist-credentials: false # CLI and image checks require neither signing material nor AWS access. - run: | - cargo test --locked --package vdev commands::release::repositories:: cargo build --locked --package vdev + target/debug/vdev release --help + if target/debug/vdev release repositories --help; then + echo 'Package repository commands must be disabled by default.' >&2 + exit 1 + fi + cargo test --locked --package vdev --features package-repositories commands::release::repositories:: + cargo build --locked --package vdev --features package-repositories target/debug/vdev release repositories build-image docker run --rm vector-package-publisher:local --help diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index 7cc6c358abc7e..ffd089a76a41d 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -20,8 +20,9 @@ include = [ workspace = true [features] -# The ordinary developer CLI only launches the tools container. Compile the AWS -# clients and publisher implementation inside that container. +default = [] +# Opt in to the entire package-repository command and its publisher dependencies. +# Ordinary developer binaries omit it; the dedicated tools image enables it. package-repositories = [ "dep:aws-config", "dep:aws-sdk-s3", "dep:aws-sdk-sts", "dep:aws-sdk-iam", "dep:aws-smithy-http-client", "dep:aws-smithy-runtime-api", "dep:aws-smithy-types", "dep:base64", "dep:flate2", "dep:reqwest", "dep:roxmltree", "dep:tar", diff --git a/vdev/README.md b/vdev/README.md index efa56082b515d..71e6d213d8eef 100644 --- a/vdev/README.md +++ b/vdev/README.md @@ -109,6 +109,14 @@ If you are actively developing vdev itself, install from your working tree so th cargo install -f --path vdev ``` +Package repository commands are disabled in ordinary vdev builds. To enable them locally: + +```sh +cargo run --package vdev --features package-repositories -- release repositories --help +``` + +The dedicated publisher image enables this feature explicitly; normal vdev releases omit it. + ## Releasing vdev To release vdev, open a PR that bumps the `version` field in `vdev/Cargo.toml`. diff --git a/vdev/src/commands/release/mod.rs b/vdev/src/commands/release/mod.rs index 5d8689bb88bd2..28181a8426cb4 100644 --- a/vdev/src/commands/release/mod.rs +++ b/vdev/src/commands/release/mod.rs @@ -2,6 +2,7 @@ mod channel; mod generate_cue; mod github; mod prepare; +#[cfg(feature = "package-repositories")] mod repositories; mod workflow; @@ -41,6 +42,7 @@ enum Commands { GenerateCue(generate_cue::Cli), Github(github::Cli), Prepare(prepare::Cli), + #[cfg(feature = "package-repositories")] Repositories(repositories::Cli), Workflow(workflow::Cli), /// Uploads archives and packages to AWS S3 @@ -55,6 +57,7 @@ impl Cli { Commands::GenerateCue(cli) => cli.exec(), Commands::Github(cli) => cli.exec(), Commands::Prepare(cli) => cli.exec(), + #[cfg(feature = "package-repositories")] Commands::Repositories(cli) => cli.exec(), Commands::Workflow(cli) => cli.exec(), Commands::S3(args) => args.exec("release-s3.sh"), diff --git a/vdev/src/commands/release/repositories.rs b/vdev/src/commands/release/repositories.rs index a51238de831d1..cdd2c45b532e5 100644 --- a/vdev/src/commands/release/repositories.rs +++ b/vdev/src/commands/release/repositories.rs @@ -1,4 +1,3 @@ -#[cfg(feature = "package-repositories")] mod publisher; use std::{path::PathBuf, process::Command}; @@ -89,10 +88,7 @@ enum Channel { impl Cli { pub fn exec(self) -> Result<()> { if self.in_container { - #[cfg(feature = "package-repositories")] return publisher::exec(&self.config, &self.work, self.command); - #[cfg(not(feature = "package-repositories"))] - anyhow::bail!("The publisher requires the package-repositories Cargo feature"); } app::set_repo_dir()?; if matches!(self.command, Commands::BuildImage) { From 885653ef3f9b45f3cb86799618da5b20682c9ad2 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 26 Oct 2026 13:46:02 -0400 Subject: [PATCH 19/23] fix(ci): load opt-in vdev from the publisher image --- .../package-repository-tools/action.yml | 22 ++++++++++++++++--- .../workflows/package-repositories-image.yml | 1 + .../package-repositories-maintenance.yml | 16 -------------- .github/workflows/package-repositories.yml | 20 ----------------- 4 files changed, 20 insertions(+), 39 deletions(-) diff --git a/.github/actions/package-repository-tools/action.yml b/.github/actions/package-repository-tools/action.yml index 5a4257bdcd30d..55ad1558d4314 100644 --- a/.github/actions/package-repository-tools/action.yml +++ b/.github/actions/package-repository-tools/action.yml @@ -1,5 +1,5 @@ -name: Pull package repository image -description: Pull the selected immutable publisher image from private GHCR storage. +name: Load package repository tools +description: Pull the immutable publisher image and use its opt-in vdev binary. inputs: image: description: Publisher image pinned by its full GHCR SHA-256 digest @@ -27,8 +27,24 @@ runs: # Keep registry credentials in memory and remove them before any signing/AWS step. export DOCKER_CONFIG DOCKER_CONFIG=$(mktemp -d /dev/shm/package-registry.XXXXXX) - trap 'rm -rf "$DOCKER_CONFIG"' EXIT + publisher_container= + cleanup() { + if [[ -n "$publisher_container" ]]; then + docker rm "$publisher_container" >/dev/null || true + fi + rm -rf "$DOCKER_CONFIG" + } + trap cleanup EXIT printf '%s' "$GHCR_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin docker pull "$PUBLISHER_IMAGE" docker tag "$PUBLISHER_IMAGE" vector-package-publisher:local + # Ordinary vdev releases omit this command. Reuse the feature-enabled binary + # from the same reviewed image, without compiling on publishing runners. + publisher_tools="$RUNNER_TEMP/package-repository-tools" + mkdir -p "$publisher_tools" + publisher_container=$(docker create "$PUBLISHER_IMAGE") + docker cp "$publisher_container:/usr/local/bin/vdev" "$publisher_tools/vdev" + chmod +x "$publisher_tools/vdev" + "$publisher_tools/vdev" release repositories --help >/dev/null + echo "VDEV=$publisher_tools/vdev" >> "$GITHUB_ENV" echo "image=$PUBLISHER_IMAGE" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/package-repositories-image.yml b/.github/workflows/package-repositories-image.yml index 8d80cb16c52c0..1207efc822a8b 100644 --- a/.github/workflows/package-repositories-image.yml +++ b/.github/workflows/package-repositories-image.yml @@ -82,6 +82,7 @@ jobs: PUBLISHER_IMAGE: ${{ needs.build.outputs.image }} run: | docker run --rm vector-package-publisher:local --help + "$VDEV" release repositories --help { echo 'Set the repository variable below to select this image for future publishing and maintenance runs.' echo '```sh' diff --git a/.github/workflows/package-repositories-maintenance.yml b/.github/workflows/package-repositories-maintenance.yml index 95c7ad308a863..9bd67461a7147 100644 --- a/.github/workflows/package-repositories-maintenance.yml +++ b/.github/workflows/package-repositories-maintenance.yml @@ -59,10 +59,6 @@ jobs: recover-lock) test -n "$EXPECTED_VERSION" && test "$OWNER_STOPPED" = true ;; *) exit 1 ;; esac - - uses: ./.github/actions/setup - with: - vdev: true - mold: false - uses: ./.github/actions/package-repository-tools id: image with: @@ -125,10 +121,6 @@ jobs: with: ref: ${{ github.sha }} persist-credentials: false - - uses: ./.github/actions/setup - with: - vdev: true - mold: false - uses: ./.github/actions/package-repository-tools with: image: ${{ needs.maintain.outputs.publisher_image }} @@ -172,10 +164,6 @@ jobs: with: ref: ${{ github.sha }} persist-credentials: false - - uses: ./.github/actions/setup - with: - vdev: true - mold: false - uses: ./.github/actions/package-repository-tools with: image: ${{ needs.maintain.outputs.publisher_image }} @@ -217,10 +205,6 @@ jobs: with: ref: ${{ github.sha }} persist-credentials: false - - uses: ./.github/actions/setup - with: - vdev: true - mold: false - uses: ./.github/actions/package-repository-tools with: image: ${{ needs.maintain.outputs.publisher_image }} diff --git a/.github/workflows/package-repositories.yml b/.github/workflows/package-repositories.yml index b31a36919bf5d..533807f2a0a26 100644 --- a/.github/workflows/package-repositories.yml +++ b/.github/workflows/package-repositories.yml @@ -60,10 +60,6 @@ jobs: persist-credentials: false - id: source run: echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - - uses: ./.github/actions/setup - with: - vdev: true - mold: false - uses: ./.github/actions/package-repository-tools id: image with: @@ -100,10 +96,6 @@ jobs: with: ref: ${{ needs.inputs.outputs.tooling_commit }} persist-credentials: false - - uses: ./.github/actions/setup - with: - vdev: true - mold: false - uses: ./.github/actions/package-repository-tools with: image: ${{ needs.inputs.outputs.publisher_image }} @@ -144,10 +136,6 @@ jobs: with: ref: ${{ needs.inputs.outputs.tooling_commit }} persist-credentials: false - - uses: ./.github/actions/setup - with: - vdev: true - mold: false - uses: ./.github/actions/package-repository-tools with: image: ${{ needs.inputs.outputs.publisher_image }} @@ -191,10 +179,6 @@ jobs: with: ref: ${{ needs.inputs.outputs.tooling_commit }} persist-credentials: false - - uses: ./.github/actions/setup - with: - vdev: true - mold: false - uses: ./.github/actions/package-repository-tools with: image: ${{ needs.inputs.outputs.publisher_image }} @@ -227,10 +211,6 @@ jobs: with: ref: ${{ needs.inputs.outputs.tooling_commit }} persist-credentials: false - - uses: ./.github/actions/setup - with: - vdev: true - mold: false - uses: ./.github/actions/package-repository-tools with: image: ${{ needs.inputs.outputs.publisher_image }} From d0bb35a235d8ecae99d3a0ae7cb1336b1f7cb324 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 26 Oct 2026 15:15:15 -0400 Subject: [PATCH 20/23] refactor(vdev): run package repository tooling natively --- .github/workflows/changes.yml | 1 + .github/workflows/test.yml | 6 +- .github/workflows/vdev_publish.yml | 17 +- distribution/repositories/Dockerfile | 19 -- vdev/Cargo.toml | 2 +- vdev/README.md | 6 +- vdev/src/commands/release/repositories.rs | 224 +----------------- .../release/repositories/publisher/mod.rs | 5 +- .../repositories/publisher/repository.rs | 20 +- 9 files changed, 49 insertions(+), 251 deletions(-) delete mode 100644 distribution/repositories/Dockerfile diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index 5425373ac512a..1418b3b3e2743 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -295,6 +295,7 @@ jobs: - "rust-toolchain.toml" - ".github/actions/package-repository-tools/**" - ".github/workflows/package-repositories*.yml" + - ".github/workflows/vdev_publish.yml" - ".github/workflows/changes.yml" k8s: - "src/sources/kubernetes_logs/**" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 35a9313ddd0bd..1d65b5a25456f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,7 +79,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - # CLI and image checks require neither signing material nor AWS access. + # Default-off and opt-in CLI checks require neither signing material nor AWS access. - run: | cargo build --locked --package vdev target/debug/vdev release --help @@ -87,10 +87,8 @@ jobs: echo 'Package repository commands must be disabled by default.' >&2 exit 1 fi - cargo test --locked --package vdev --features package-repositories commands::release::repositories:: cargo build --locked --package vdev --features package-repositories - target/debug/vdev release repositories build-image - docker run --rm vector-package-publisher:local --help + target/debug/vdev release repositories --help test: needs: changes diff --git a/.github/workflows/vdev_publish.yml b/.github/workflows/vdev_publish.yml index 28bf00b463492..4080e6afc7ca5 100644 --- a/.github/workflows/vdev_publish.yml +++ b/.github/workflows/vdev_publish.yml @@ -14,10 +14,21 @@ jobs: include: - os: macos-latest target: aarch64-apple-darwin + features: "" + suffix: "" - os: ubuntu-24.04 target: x86_64-unknown-linux-gnu + features: "" + suffix: "" - os: ubuntu-24.04-arm target: aarch64-unknown-linux-gnu + features: "" + suffix: "" + # Keep publishing commands out of ordinary developer binaries. + - os: ubuntu-24.04 + target: x86_64-unknown-linux-gnu + features: package-repositories + suffix: -package-repositories steps: - name: Checkout Vector @@ -30,13 +41,13 @@ jobs: - name: Build working-directory: vdev - run: cargo build --release --target ${{ matrix.target }} + run: cargo build --locked --release --target ${{ matrix.target }} --features "${{ matrix.features }}" - name: Package shell: bash run: | VERSION=${GITHUB_REF_NAME#vdev-v} - OUTDIR="vdev-${{ matrix.target }}-v${VERSION}" + OUTDIR="vdev${{ matrix.suffix }}-${{ matrix.target }}-v${VERSION}" mkdir -p "$OUTDIR" BIN_ROOT="${CARGO_TARGET_DIR:-target}" @@ -54,7 +65,7 @@ jobs: - name: Stage asset for release uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: vdev-${{ matrix.target }} + name: vdev${{ matrix.suffix }}-${{ matrix.target }} path: ${{ env.ASSET }} if-no-files-found: error diff --git a/distribution/repositories/Dockerfile b/distribution/repositories/Dockerfile deleted file mode 100644 index 012915d13ff72..0000000000000 --- a/distribution/repositories/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM rust:1.96.1-bookworm@sha256:a339861ae23e9abb272cea45dfafde21760d2ce6577a70f8a926153677902663 AS build -RUN apt-get update && apt-get install -y --no-install-recommends cmake libssl-dev pkg-config \ - && rm -rf /var/lib/apt/lists/* -WORKDIR /src -COPY . . -RUN --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/usr/local/cargo/git \ - cargo build --locked --package vdev --features package-repositories \ - && strip target/debug/vdev - -FROM ubuntu:24.04@sha256:68ee22004a9109b44973313b02ff7b1de250f1de0f5ecb628d73f579aaff33ac -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - apt-utils ca-certificates createrepo-c dpkg-dev gnupg rpm libssl3t64 \ - && rm -rf /var/lib/apt/lists/* -COPY --from=build /src/target/debug/vdev /usr/local/bin/vdev -# vdev normally locates its checkout before dispatching a command. -COPY --from=build /src/Cargo.toml /app/Cargo.toml -WORKDIR /app -ENTRYPOINT ["vdev", "release", "repositories"] diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index ffd089a76a41d..49757f65dcba9 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -22,7 +22,7 @@ workspace = true [features] default = [] # Opt in to the entire package-repository command and its publisher dependencies. -# Ordinary developer binaries omit it; the dedicated tools image enables it. +# Ordinary developer binaries omit it; the separate Linux publisher binary enables it. package-repositories = [ "dep:aws-config", "dep:aws-sdk-s3", "dep:aws-sdk-sts", "dep:aws-sdk-iam", "dep:aws-smithy-http-client", "dep:aws-smithy-runtime-api", "dep:aws-smithy-types", "dep:base64", "dep:flate2", "dep:reqwest", "dep:roxmltree", "dep:tar", diff --git a/vdev/README.md b/vdev/README.md index 71e6d213d8eef..4b710ed9c62f5 100644 --- a/vdev/README.md +++ b/vdev/README.md @@ -115,7 +115,11 @@ Package repository commands are disabled in ordinary vdev builds. To enable them cargo run --package vdev --features package-repositories -- release repositories --help ``` -The dedicated publisher image enables this feature explicitly; normal vdev releases omit it. +The separate `vdev-package-repositories-x86_64-unknown-linux-gnu` release archive enables +this feature; ordinary vdev binaries omit it. Repository commands run natively on Linux +and need `apt-utils`, `dpkg-dev`, `createrepo-c`, `rpm`, and `gnupg`. Key files use private, +automatically cleaned directories in `/dev/shm`. Until the production integration is +installed, pass `--config` with the repository configuration and its public key. ## Releasing vdev diff --git a/vdev/src/commands/release/repositories.rs b/vdev/src/commands/release/repositories.rs index cdd2c45b532e5..df4d17a8573b8 100644 --- a/vdev/src/commands/release/repositories.rs +++ b/vdev/src/commands/release/repositories.rs @@ -1,21 +1,15 @@ mod publisher; -use std::{path::PathBuf, process::Command}; +use std::path::PathBuf; -use anyhow::{Context as _, Result, ensure}; +use anyhow::{Context as _, Result}; -use crate::app::{self, CommandExt as _}; +use crate::app; -/// Build, sign, publish, and recover the apt/yum repositories. +/// Sign, publish, and recover the apt/yum repositories using native Linux tools. #[derive(clap::Args, Debug)] pub(super) struct Cli { - /// Container built with `build-image` from the reviewed publisher source. - #[arg(long, default_value = "vector-package-publisher:local", global = true)] - image: String, - /// Run the publisher inside the prepared tools image. - #[arg(long, hide = true, global = true)] - in_container: bool, - /// Configuration and public key directory; never contains a private key. + /// Publisher configuration; use --config until the production integration is installed. #[arg( long, default_value = "distribution/repositories/production.json", @@ -31,8 +25,6 @@ pub(super) struct Cli { #[derive(clap::Subcommand, Debug)] enum Commands { - /// Build the tools image locally; no registry publication is required. - BuildImage, /// Validate a completed Release Suite run and freeze its release assets. Fetch { #[arg(long)] @@ -87,209 +79,11 @@ enum Channel { impl Cli { pub fn exec(self) -> Result<()> { - if self.in_container { - return publisher::exec(&self.config, &self.work, self.command); - } app::set_repo_dir()?; - if matches!(self.command, Commands::BuildImage) { - return Command::new("docker") - .args([ - "build", - "--tag", - &self.image, - "--file", - "distribution/repositories/Dockerfile", - ".", - ]) - .check_run(); - } std::fs::create_dir_all(&self.work)?; - let work = self.work.canonicalize()?; - let config = self - .config - .canonicalize() - .context("Publisher configuration is missing")?; - let directory = config.parent().context("Configuration has no parent")?; - ensure!(config.is_file(), "Configuration must be a regular file"); - let mut command = Command::new("docker"); - command.args(["run", "--rm", "--init"]); - let key_mount = { - #[cfg(unix)] - { - use std::os::unix::fs::MetadataExt as _; - - let owner = work.metadata()?; - command.args(["--user", &format!("{}:{}", owner.uid(), owner.gid())]); - format!( - "/run/keys:rw,noexec,nosuid,size=16m,mode=0700,uid={},gid={}", - owner.uid(), - owner.gid() - ) - } - #[cfg(not(unix))] - { - "/run/keys:rw,noexec,nosuid,size=16m,mode=0700".to_owned() - } - }; - command.args([ - "--cap-drop=ALL", - "--security-opt=no-new-privileges", - "--tmpfs", - &key_mount, - "--env", - "HOME=/run/keys", - "--mount", - &format!("type=bind,source={},target=/work", work.display()), - "--mount", - &format!( - "type=bind,source={},target=/config,readonly", - directory.display() - ), - ]); - // Forward names, never values: command logging must not expose credentials. - for name in self.command.credentials() { - command.args(["--env", name]); - } - command.arg(&self.image).args([ - "--in-container", - "--work", - "/work", - "--config", - &format!( - "/config/{}", - config - .file_name() - .context("Missing configuration filename")? - .to_string_lossy() - ), - ]); - self.command.add_arguments(&mut command); - command.check_run() - } -} - -impl Commands { - fn credentials(&self) -> &'static [&'static str] { - match self { - Self::Fetch { .. } => &["GH_TOKEN"], - Self::Prepare - | Self::Rollback { - phase: Rollback::Prepare, - } => &["PACKAGE_SIGNING_SUBKEY", "PACKAGE_SIGNING_PASSPHRASE"], - Self::Base - | Self::Publish - | Self::Status - | Self::Rollback { - phase: Rollback::Base { .. } | Rollback::Publish, - } - | Self::RecoverLock { .. } => &[ - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", - "AWS_SESSION_TOKEN", - "PACKAGE_REPOSITORIES_ROLE_ARN", - ], - _ => &[], - } - } - - fn add_arguments(self, command: &mut Command) { - match self { - Commands::Fetch { run, tag, channel } => { - command.args([ - "fetch", - "--run", - &run.to_string(), - "--tag", - &tag, - "--channel", - match channel { - Channel::Stable => "stable", - Channel::Beta => "beta", - }, - ]); - } - Commands::Base => { - command.arg("base"); - } - Commands::Prepare => { - command.arg("prepare"); - } - Commands::Publish => { - command.arg("publish"); - } - Commands::Verify => { - command.arg("verify"); - } - Commands::Status => { - command.arg("status"); - } - Commands::Rollback { phase } => { - command.arg("rollback"); - match phase { - Rollback::Base { snapshot } => { - command.args(["base", &snapshot]); - } - Rollback::Prepare => { - command.arg("prepare"); - } - Rollback::Publish => { - command.arg("publish"); - } - Rollback::Verify => { - command.arg("verify"); - } - } - } - Commands::RecoverLock { - expected_version, - confirm_owner_stopped, - } => { - command.args(["recover-lock", "--expected-version", &expected_version]); - if confirm_owner_stopped { - command.arg("--confirm-owner-stopped"); - } - } - Commands::BuildImage => unreachable!(), - } - } -} - -#[cfg(test)] -mod tests { - use super::{Commands, Rollback}; - - #[test] - fn signing_and_publishing_receive_disjoint_credentials() { - let signing = Commands::Prepare.credentials(); - let publishing = Commands::Publish.credentials(); - assert_eq!( - signing, - ["PACKAGE_SIGNING_SUBKEY", "PACKAGE_SIGNING_PASSPHRASE"] - ); - assert!(publishing.contains(&"AWS_SESSION_TOKEN")); - assert!(signing.iter().all(|name| !publishing.contains(name))); - for phase in [ - Rollback::Base { - snapshot: "test".into(), - }, - Rollback::Publish, - ] { - assert_eq!(Commands::Rollback { phase }.credentials(), publishing); - } - assert_eq!( - Commands::Rollback { - phase: Rollback::Prepare - } - .credentials(), - signing - ); - assert!( - Commands::Rollback { - phase: Rollback::Verify - } - .credentials() - .is_empty() - ); - assert!(Commands::Verify.credentials().is_empty()); + let config = self.config.canonicalize().context( + "Publisher configuration is missing; pass --config with a repository configuration", + )?; + publisher::exec(&config, &self.work, self.command) } } diff --git a/vdev/src/commands/release/repositories/publisher/mod.rs b/vdev/src/commands/release/repositories/publisher/mod.rs index c3b0bdd72584b..14b1948de3d3b 100644 --- a/vdev/src/commands/release/repositories/publisher/mod.rs +++ b/vdev/src/commands/release/repositories/publisher/mod.rs @@ -1,4 +1,4 @@ -//! Container-side implementation of `vdev release repositories`. +//! Native implementation of `vdev release repositories`. // Repository object names are case-sensitive protocol identifiers. #![allow(clippy::case_sensitive_file_extension_comparisons)] mod bundles; @@ -9,7 +9,7 @@ mod repository; mod rollback; mod storage; use super::{Channel, Commands, Rollback}; -use anyhow::{Result, bail}; +use anyhow::Result; use model::Config; use std::path::Path; @@ -71,6 +71,5 @@ pub(super) fn exec(config: &Path, work: &Path, command: Commands) -> Result<()> } Ok(()) } - Commands::BuildImage => bail!("Build the image outside the container"), } } diff --git a/vdev/src/commands/release/repositories/publisher/repository.rs b/vdev/src/commands/release/repositories/publisher/repository.rs index 3cfc82279eef5..6ac9e27e69185 100644 --- a/vdev/src/commands/release/repositories/publisher/repository.rs +++ b/vdev/src/commands/release/repositories/publisher/repository.rs @@ -217,13 +217,24 @@ fn fingerprints(data: &[u8], primary: &str, subkey: &str) -> Result { pub config: &'a Config, pub home: PathBuf, + _directory: tempfile::TempDir, } impl<'a> PublicKey<'a> { pub fn new(config: &'a Config, name: &str) -> Result { - use std::os::unix::fs::DirBuilderExt as _; - let home = Path::new("/run/keys").join(name); - fs::DirBuilder::new().mode(0o700).create(&home)?; - let public = Self { config, home }; + use std::os::unix::fs::PermissionsExt as _; + + // Linux tmpfs keeps key files outside the checkout and artifact directories. + // TempDir creates a unique 0700 directory and removes it on success or error. + let directory = tempfile::Builder::new() + .prefix(&format!("vector-package-{name}-")) + .permissions(fs::Permissions::from_mode(0o700)) + .tempdir_in("/dev/shm") + .context("Package repository signing requires writable Linux tmpfs at /dev/shm")?; + let public = Self { + config, + home: directory.path().to_owned(), + _directory: directory, + }; let keys = fingerprints( &public.gpg( [ @@ -317,7 +328,6 @@ impl Drop for PublicKey<'_> { .stderr(Stdio::null()) .status(), ); - drop(fs::remove_dir_all(&self.home)); } } pub(super) struct Signer<'a> { From edcba5f4bb561cb42dcfc96f13b8a9372919ad86 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 26 Oct 2026 15:15:19 -0400 Subject: [PATCH 21/23] refactor(ci): install package publishing tools on the runner --- .../package-repository-tools/action.yml | 65 ++++--------- .../workflows/package-repositories-image.yml | 91 ------------------- .../package-repositories-maintenance.yml | 23 +---- .github/workflows/package-repositories.yml | 28 +----- 4 files changed, 24 insertions(+), 183 deletions(-) delete mode 100644 .github/workflows/package-repositories-image.yml diff --git a/.github/actions/package-repository-tools/action.yml b/.github/actions/package-repository-tools/action.yml index 55ad1558d4314..04a585fca3b38 100644 --- a/.github/actions/package-repository-tools/action.yml +++ b/.github/actions/package-repository-tools/action.yml @@ -1,50 +1,23 @@ -name: Load package repository tools -description: Pull the immutable publisher image and use its opt-in vdev binary. -inputs: - image: - description: Publisher image pinned by its full GHCR SHA-256 digest - required: true - token: - description: GitHub token with read access to the container package - required: true -outputs: - image: - description: Exact image reference used by this job - value: ${{ steps.pull.outputs.image }} +name: Install package repository tools +description: Install native packaging tools and the versioned opt-in vdev binary. runs: using: composite steps: - - id: pull - shell: bash - env: - PUBLISHER_IMAGE: ${{ inputs.image }} - GHCR_TOKEN: ${{ inputs.token }} + # Install tools before the job acquires signing or AWS credentials. + - shell: bash run: | - [[ "$PUBLISHER_IMAGE" =~ ^ghcr\.io/vectordotdev/[a-z0-9._/-]+@sha256:[0-9a-f]{64}$ ]] || { - echo 'Set PACKAGE_REPOSITORIES_IMAGE to the digest reported by Package Repository Image.' >&2 - exit 1 - } - # Keep registry credentials in memory and remove them before any signing/AWS step. - export DOCKER_CONFIG - DOCKER_CONFIG=$(mktemp -d /dev/shm/package-registry.XXXXXX) - publisher_container= - cleanup() { - if [[ -n "$publisher_container" ]]; then - docker rm "$publisher_container" >/dev/null || true - fi - rm -rf "$DOCKER_CONFIG" - } - trap cleanup EXIT - printf '%s' "$GHCR_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin - docker pull "$PUBLISHER_IMAGE" - docker tag "$PUBLISHER_IMAGE" vector-package-publisher:local - # Ordinary vdev releases omit this command. Reuse the feature-enabled binary - # from the same reviewed image, without compiling on publishing runners. - publisher_tools="$RUNNER_TEMP/package-repository-tools" - mkdir -p "$publisher_tools" - publisher_container=$(docker create "$PUBLISHER_IMAGE") - docker cp "$publisher_container:/usr/local/bin/vdev" "$publisher_tools/vdev" - chmod +x "$publisher_tools/vdev" - "$publisher_tools/vdev" release repositories --help >/dev/null - echo "VDEV=$publisher_tools/vdev" >> "$GITHUB_ENV" - echo "image=$PUBLISHER_IMAGE" >> "$GITHUB_OUTPUT" + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + apt-utils ca-certificates createrepo-c dpkg-dev gnupg rpm libssl3t64 + version=$(sed -n 's/^version = "\(.*\)"/\1/p' vdev/Cargo.toml) + [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] + archive="vdev-package-repositories-x86_64-unknown-linux-gnu-v${version}" + tools="$RUNNER_TEMP/package-repository-tools" + mkdir -p "$tools" + curl --fail --silent --show-error --location --retry 3 \ + "https://install.datadoghq.com/vector/vdev/${version}/${archive}.tgz" \ + --output "$tools/$archive.tgz" + tar -xzf "$tools/$archive.tgz" -C "$tools" --strip-components=1 "$archive/vdev" + test "$("$tools/vdev" --version)" = "vdev $version" + "$tools/vdev" release repositories --help >/dev/null + echo "VDEV=$tools/vdev" >> "$GITHUB_ENV" diff --git a/.github/workflows/package-repositories-image.yml b/.github/workflows/package-repositories-image.yml deleted file mode 100644 index 1207efc822a8b..0000000000000 --- a/.github/workflows/package-repositories-image.yml +++ /dev/null @@ -1,91 +0,0 @@ -name: Package Repository Image - -on: - workflow_dispatch: - -permissions: {} - -# Rebuild manually after publisher/dependency changes or to refresh OS packages. -# A successful build does not change the image selected for production jobs. -concurrency: - group: package-repositories-image - cancel-in-progress: false - -jobs: - # Build without package-signing or AWS credentials; publish only from master. - build: - if: github.repository_id == '146327667' && github.ref == 'refs/heads/master' - runs-on: ubuntu-24.04 - timeout-minutes: 45 - permissions: - contents: read - packages: write - env: - IMAGE_NAME: ghcr.io/vectordotdev/vector-package-publisher - outputs: - image: ${{ steps.publish.outputs.image }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.sha }} - persist-credentials: false - - name: Build, check and publish a private image - id: publish - env: - GH_TOKEN: ${{ github.token }} - run: | - package="orgs/vectordotdev/packages/container/${IMAGE_NAME##*/}" - metadata="$RUNNER_TEMP/package-image.json" - gh api "$package" > "$metadata" || jq -e '.status == "404"' "$metadata" >/dev/null - jq -e '.status == "404" or .visibility == "private"' "$metadata" >/dev/null - - revision=$(git rev-parse HEAD) - tag="$IMAGE_NAME:$revision-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" - docker build --pull --no-cache --tag "$tag" \ - --label "org.opencontainers.image.source=https://github.com/$GITHUB_REPOSITORY" \ - --label "org.opencontainers.image.revision=$revision" \ - --file distribution/repositories/Dockerfile . - docker run --rm "$tag" --help - docker run --rm --entrypoint sh "$tag" -ec \ - 'for tool in vdev apt-ftparchive dpkg-deb createrepo_c rpm rpmsign gpg; do command -v "$tool"; done' - - # Registry credentials stay in memory; the new GHCR package is private by default. - export DOCKER_CONFIG - DOCKER_CONFIG=$(mktemp -d /dev/shm/package-registry.XXXXXX) - trap 'rm -rf "$DOCKER_CONFIG"' EXIT - printf '%s' "$GH_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin - docker push "$tag" - test "$(gh api "$package" --jq .visibility)" = private - image=$(docker inspect --format '{{index .RepoDigests 0}}' "$tag") - [[ "$image" == "$IMAGE_NAME@sha256:"* ]] - echo "image=$image" >> "$GITHUB_OUTPUT" - - # Pull from GHCR on a fresh runner before showing the digest operators can select. - verify: - needs: build - runs-on: ubuntu-24.04 - timeout-minutes: 10 - permissions: - contents: read - packages: read - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.sha }} - persist-credentials: false - - uses: ./.github/actions/package-repository-tools - with: - image: ${{ needs.build.outputs.image }} - token: ${{ github.token }} - - name: Check the pulled image and report the manual selection command - env: - PUBLISHER_IMAGE: ${{ needs.build.outputs.image }} - run: | - docker run --rm vector-package-publisher:local --help - "$VDEV" release repositories --help - { - echo 'Set the repository variable below to select this image for future publishing and maintenance runs.' - echo '```sh' - printf 'gh variable set PACKAGE_REPOSITORIES_IMAGE --repo %s --body %s\n' "$GITHUB_REPOSITORY" "$PUBLISHER_IMAGE" - echo '```' - } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/package-repositories-maintenance.yml b/.github/workflows/package-repositories-maintenance.yml index 9bd67461a7147..7c694757775f5 100644 --- a/.github/workflows/package-repositories-maintenance.yml +++ b/.github/workflows/package-repositories-maintenance.yml @@ -27,19 +27,16 @@ concurrency: jobs: # Inspect S3 state, prepare a rollback, or recover a lock with the publishing role. - # Capture PACKAGE_REPOSITORIES_IMAGE once so every rollback job uses the same digest. + # Every job uses the vdev version pinned by the reviewed workflow checkout. # Available even when automated publication is disabled. Recovery requires the exact # control version, a stopped owner and an expired 65-minute recovery window. maintain: if: github.repository_id == '146327667' && github.ref == 'refs/heads/master' runs-on: ubuntu-24.04 timeout-minutes: 30 - outputs: - publisher_image: ${{ steps.image.outputs.image }} environment: package-repositories-publishing permissions: contents: read - packages: read id-token: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -60,10 +57,6 @@ jobs: *) exit 1 ;; esac - uses: ./.github/actions/package-repository-tools - id: image - with: - image: ${{ vars.PACKAGE_REPOSITORIES_IMAGE }} - token: ${{ github.token }} - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 with: role-to-assume: ${{ vars.PACKAGE_REPOSITORIES_ROLE_ARN }} @@ -115,16 +108,12 @@ jobs: timeout-minutes: 20 permissions: contents: read - packages: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.sha }} persist-credentials: false - uses: ./.github/actions/package-repository-tools - with: - image: ${{ needs.maintain.outputs.publisher_image }} - token: ${{ github.token }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: package-repository-rollback-base @@ -140,7 +129,7 @@ jobs: PACKAGE_SIGNING_SUBKEY: ${{ secrets.PACKAGE_SIGNING_SUBKEY }} PACKAGE_SIGNING_PASSPHRASE: ${{ secrets.PACKAGE_SIGNING_PASSPHRASE }} run: '"$VDEV" release repositories rollback prepare' - # Private material stays in the container's tmpfs, outside this artifact. + # GPG homes use private /dev/shm directories, outside this artifact. - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: package-repository-rollback-signed @@ -157,7 +146,6 @@ jobs: timeout-minutes: 20 permissions: contents: read - packages: read id-token: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -165,9 +153,6 @@ jobs: ref: ${{ github.sha }} persist-credentials: false - uses: ./.github/actions/package-repository-tools - with: - image: ${{ needs.maintain.outputs.publisher_image }} - token: ${{ github.token }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: package-repository-rollback-signed @@ -199,16 +184,12 @@ jobs: timeout-minutes: 20 permissions: contents: read - packages: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.sha }} persist-credentials: false - uses: ./.github/actions/package-repository-tools - with: - image: ${{ needs.maintain.outputs.publisher_image }} - token: ${{ github.token }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: package-repository-rollback-signed diff --git a/.github/workflows/package-repositories.yml b/.github/workflows/package-repositories.yml index 533807f2a0a26..3be2197627899 100644 --- a/.github/workflows/package-repositories.yml +++ b/.github/workflows/package-repositories.yml @@ -28,8 +28,8 @@ concurrency: cancel-in-progress: false jobs: - # Freeze verified release assets and the manually selected publisher image digest. - # Rebuild via Package Repository Image; select its digest with PACKAGE_REPOSITORIES_IMAGE. + # Freeze verified release assets and the reviewed tooling checkout. + # Jobs install the opt-in binary published with the vdev version in that checkout. # Trust authorized GitHub release publishers: run/tag and checksum checks do not # prove build provenance. Stronger provenance verification is deferred. # Enable only after historical backfill and the repository/key migration are ready. @@ -47,11 +47,9 @@ jobs: timeout-minutes: 30 permissions: contents: read - packages: read actions: read outputs: tooling_commit: ${{ steps.source.outputs.commit }} - publisher_image: ${{ steps.image.outputs.image }} steps: # workflow_run executes reviewed default-branch code, never release/tag code. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -61,10 +59,6 @@ jobs: - id: source run: echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - uses: ./.github/actions/package-repository-tools - id: image - with: - image: ${{ vars.PACKAGE_REPOSITORIES_IMAGE }} - token: ${{ github.token }} - name: Freeze and verify release inputs env: GH_TOKEN: ${{ github.token }} @@ -89,7 +83,6 @@ jobs: timeout-minutes: 20 permissions: contents: read - packages: read id-token: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -97,9 +90,6 @@ jobs: ref: ${{ needs.inputs.outputs.tooling_commit }} persist-credentials: false - uses: ./.github/actions/package-repository-tools - with: - image: ${{ needs.inputs.outputs.publisher_image }} - token: ${{ github.token }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: package-repository-inputs @@ -130,16 +120,12 @@ jobs: timeout-minutes: 20 permissions: contents: read - packages: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ needs.inputs.outputs.tooling_commit }} persist-credentials: false - uses: ./.github/actions/package-repository-tools - with: - image: ${{ needs.inputs.outputs.publisher_image }} - token: ${{ github.token }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: package-repository-base @@ -155,7 +141,7 @@ jobs: PACKAGE_SIGNING_SUBKEY: ${{ secrets.PACKAGE_SIGNING_SUBKEY }} PACKAGE_SIGNING_PASSPHRASE: ${{ secrets.PACKAGE_SIGNING_PASSPHRASE }} run: '"$VDEV" release repositories prepare' - # Private material only exists in the container's tmpfs, outside this path. + # GPG homes use private /dev/shm directories and are removed before this upload. - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: package-repository-signed @@ -172,7 +158,6 @@ jobs: timeout-minutes: 20 permissions: contents: read - packages: read id-token: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -180,9 +165,6 @@ jobs: ref: ${{ needs.inputs.outputs.tooling_commit }} persist-credentials: false - uses: ./.github/actions/package-repository-tools - with: - image: ${{ needs.inputs.outputs.publisher_image }} - token: ${{ github.token }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: package-repository-signed @@ -205,16 +187,12 @@ jobs: timeout-minutes: 20 permissions: contents: read - packages: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ needs.inputs.outputs.tooling_commit }} persist-credentials: false - uses: ./.github/actions/package-repository-tools - with: - image: ${{ needs.inputs.outputs.publisher_image }} - token: ${{ github.token }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: package-repository-signed From b11727b7acbd2681d439f404c5a3ab3b3721e75f Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 26 Oct 2026 15:59:34 -0400 Subject: [PATCH 22/23] docs(ci): document interrupted package upload recovery --- .../package-repositories-maintenance.yml | 25 ++++++++++++++++--- .github/workflows/package-repositories.yml | 2 ++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.github/workflows/package-repositories-maintenance.yml b/.github/workflows/package-repositories-maintenance.yml index 7c694757775f5..c5d8f4ce27765 100644 --- a/.github/workflows/package-repositories-maintenance.yml +++ b/.github/workflows/package-repositories-maintenance.yml @@ -12,7 +12,7 @@ on: description: Exact archived snapshot SHA-256 for rollback type: string expected_version: - description: Exact control-object S3 version reported by status, for lock recovery + description: Top-level version from the status summary (S3 VersionId, not snapshot SHA-256) type: string confirm_owner_stopped: description: The previous workflow/process and its in-flight writes have stopped @@ -28,8 +28,27 @@ concurrency: jobs: # Inspect S3 state, prepare a rollback, or recover a lock with the publishing role. # Every job uses the vdev version pinned by the reviewed workflow checkout. - # Available even when automated publication is disabled. Recovery requires the exact - # control version, a stopped owner and an expired 65-minute recovery window. + # Available even when automated publication is disabled. + # + # Recover after a workflow dies during upload (GitHub Actions -> this workflow): + # 1. Stop/cancel the old workflow and confirm its runner/process and in-flight + # writes have stopped. A failed workflow alone is not that confirmation. + # 2. Run on master with operation=status and obtain Vector team approval. In + # the maintain job summary, inspect state.held, state.owner, state.pending, + # state.recover_after and the top-level version. If held=false, skip unlocking. + # 3. Wait until state.recover_after (UTC, 65 minutes after lock acquisition). + # The lock never expires automatically; do not edit/delete the S3 control object. + # 4. Run on master with operation=recover-lock, expected_version set to that + # version, and confirm_owner_stopped=true; obtain team approval again. + # If the version changed, repeat status and reassess the owner before retrying. + # 5. Confirm state.held=false in the recovery summary. Unlocking preserves + # active/pending and does not repair partially uploaded repository metadata. + # 6. Resume by dispatching Package Repositories with the same Release Suite + # run_id, tag and channel; it restores the saved signed S3 bundle. For an + # interrupted rollback, dispatch operation=rollback with the same snapshot. + # To abandon an unfinished release, use operation=rollback with state.active + # as snapshot. Finish the resumed publication/rollback and its verification + # before publishing a different release. maintain: if: github.repository_id == '146327667' && github.ref == 'refs/heads/master' runs-on: ubuntu-24.04 diff --git a/.github/workflows/package-repositories.yml b/.github/workflows/package-repositories.yml index 3be2197627899..067ff5a7f583e 100644 --- a/.github/workflows/package-repositories.yml +++ b/.github/workflows/package-repositories.yml @@ -151,6 +151,8 @@ jobs: # Upload immutable packages/indexes before switching signed repository roots. # Serialize writes with the S3 control lock; this job has no signing key. + # If an upload fails with the lock held, follow the recovery procedure above + # the maintain job in package-repositories-maintenance.yml before retrying. publish: needs: [inputs, sign] runs-on: ubuntu-24.04 From 0c28075e8292bcf732c325dbe17374eeb3b59230 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 28 Oct 2026 15:53:21 -0400 Subject: [PATCH 23/23] fix(ci): pin publisher tools and preserve queued releases Preserve curl-minimal during RPM prerequisite repair and verify the published opt-in vdev archive before execution. AI assistance: GPT-6 (Codex). --- .github/actions/package-repository-tools/action.yml | 11 +++++++---- .../workflows/package-repositories-maintenance.yml | 3 ++- .github/workflows/package-repositories.yml | 3 ++- distribution/repositories/setup.sh | 4 +++- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/actions/package-repository-tools/action.yml b/.github/actions/package-repository-tools/action.yml index 04a585fca3b38..715bad04f7f27 100644 --- a/.github/actions/package-repository-tools/action.yml +++ b/.github/actions/package-repository-tools/action.yml @@ -1,5 +1,5 @@ name: Install package repository tools -description: Install native packaging tools and the versioned opt-in vdev binary. +description: Install native packaging tools and the checksum-pinned opt-in vdev binary. runs: using: composite steps: @@ -9,14 +9,17 @@ runs: sudo apt-get update sudo apt-get install -y --no-install-recommends \ apt-utils ca-certificates createrepo-c dpkg-dev gnupg rpm libssl3t64 - version=$(sed -n 's/^version = "\(.*\)"/\1/p' vdev/Cargo.toml) - [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] + # Pin independently of developer vdev; update version and archive SHA-256 together. + # Published by https://github.com/vectordotdev/vector/actions/runs/40552518149. + version=0.3.37 + sha256=43ec4b112782a51af7cefec6a28e0248dc8b8b6b8b51cca4e4d0a3c2e05e8990 archive="vdev-package-repositories-x86_64-unknown-linux-gnu-v${version}" tools="$RUNNER_TEMP/package-repository-tools" mkdir -p "$tools" - curl --fail --silent --show-error --location --retry 3 \ + curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --retry 3 \ "https://install.datadoghq.com/vector/vdev/${version}/${archive}.tgz" \ --output "$tools/$archive.tgz" + echo "$sha256 $tools/$archive.tgz" | sha256sum --check --strict tar -xzf "$tools/$archive.tgz" -C "$tools" --strip-components=1 "$archive/vdev" test "$("$tools/vdev" --version)" = "vdev $version" "$tools/vdev" release repositories --help >/dev/null diff --git a/.github/workflows/package-repositories-maintenance.yml b/.github/workflows/package-repositories-maintenance.yml index c5d8f4ce27765..5f032799f1697 100644 --- a/.github/workflows/package-repositories-maintenance.yml +++ b/.github/workflows/package-repositories-maintenance.yml @@ -23,11 +23,12 @@ permissions: {} concurrency: group: package-repositories + queue: max cancel-in-progress: false jobs: # Inspect S3 state, prepare a rollback, or recover a lock with the publishing role. - # Every job uses the vdev version pinned by the reviewed workflow checkout. + # Every job uses the publisher version and SHA-256 pinned by the reviewed checkout. # Available even when automated publication is disabled. # # Recover after a workflow dies during upload (GitHub Actions -> this workflow): diff --git a/.github/workflows/package-repositories.yml b/.github/workflows/package-repositories.yml index 067ff5a7f583e..d4233f2030574 100644 --- a/.github/workflows/package-repositories.yml +++ b/.github/workflows/package-repositories.yml @@ -25,11 +25,12 @@ permissions: {} # All channels share the apt pool and one S3 publication control object. concurrency: group: package-repositories + queue: max cancel-in-progress: false jobs: # Freeze verified release assets and the reviewed tooling checkout. - # Jobs install the opt-in binary published with the vdev version in that checkout. + # Jobs install the publisher version and SHA-256 pinned by that checkout. # Trust authorized GitHub release publishers: run/tag and checksum checks do not # prove build provenance. Stronger provenance verification is deferred. # Enable only after historical backfill and the repository/key migration are ready. diff --git a/distribution/repositories/setup.sh b/distribution/repositories/setup.sh index c0a2c81e65854..7815f14d33473 100644 --- a/distribution/repositories/setup.sh +++ b/distribution/repositories/setup.sh @@ -259,7 +259,9 @@ if ! command -v curl >/dev/null || ! command -v gpg >/dev/null || [[ ! -s /etc/s "${sudo_cmd[@]}" apt-get update || printf 'Some indexes could not refresh; attempting prerequisite installation.\n' >&2 "${sudo_cmd[@]}" env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates curl gnupg else - "${sudo_cmd[@]}" "$package_manager" -y --disablerepo=vector install ca-certificates curl gnupg2 + prerequisites=(ca-certificates gnupg2) + if ! command -v curl >/dev/null; then prerequisites+=(curl); fi + "${sudo_cmd[@]}" "$package_manager" -y --disablerepo=vector install "${prerequisites[@]}" fi fi