pront · Oct 19, 2026 · Oct 20, 2026 · Oct 20, 2026 · Oct 20, 2026 · Oct 20, 2026
@@ -0,0 +1,23 @@
+name: Install package repository tools
+description: Install native packaging tools and the versioned opt-in vdev binary.
+runs:
+  using: composite
+  steps:
+    # Install tools before the job acquires signing or AWS credentials.
+    - shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y --no-install-recommends \
+          apt-utils ca-certificates createrepo-c dpkg-dev gnupg rpm libssl3t64
+        version=$(sed -n 's/^version = "\(.*\)"/\1/p' vdev/Cargo.toml)
+        [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]
+        archive="vdev-package-repositories-x86_64-unknown-linux-gnu-v${version}"
+        tools="$RUNNER_TEMP/package-repository-tools"
+        mkdir -p "$tools"
+        curl --fail --silent --show-error --location --retry 3 \
+          "https://install.datadoghq.com/vector/vdev/${version}/${archive}.tgz" \
+          --output "$tools/$archive.tgz"
+        tar -xzf "$tools/$archive.tgz" -C "$tools" --strip-components=1 "$archive/vdev"
+        test "$("$tools/vdev" --version)" = "vdev $version"
+        "$tools/vdev" release repositories --help >/dev/null
+        echo "VDEV=$tools/vdev" >> "$GITHUB_ENV"
diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml
@@ -52,6 +52,8 @@ on:
         value: ${{ jobs.source.outputs.website_tests }}
       install:
         value: ${{ jobs.source.outputs.install }}
+      package_repositories:
+        value: ${{ jobs.source.outputs.package_repositories }}
       actionlint:
         value: ${{ jobs.source.outputs.actionlint }}
       prettier:
@@ -187,6 +189,7 @@ jobs:
       markdown: ${{ steps.filter.outputs.markdown }}
       scripts: ${{ steps.filter.outputs.scripts }}
       install: ${{ steps.filter.outputs.install }}
+      package_repositories: ${{ steps.filter.outputs.package_repositories }}
       k8s: ${{ steps.filter.outputs.k8s }}
       website_only: ${{ steps.filter.outputs.website == 'true' && steps.filter.outputs.not_website == 'false' }}
       website_tests: ${{ steps.website_filter.outputs.website_tests }}
@@ -284,6 +287,16 @@ jobs:
               - ".github/workflows/install-sh.yml"
               - "distribution/install.sh"
               - ".github/workflows/changes.yml"
+            package_repositories:
+              - "distribution/repositories/**"
+              - "vdev/**"
+              - "Cargo.toml"
+              - "Cargo.lock"
+              - "rust-toolchain.toml"
+              - ".github/actions/package-repository-tools/**"
+              - ".github/workflows/package-repositories*.yml"
+              - ".github/workflows/vdev_publish.yml"
+              - ".github/workflows/changes.yml"
             k8s:
               - "src/sources/kubernetes_logs/**"
               - "lib/k8s-e2e-tests/**"

@@ -0,0 +1,217 @@
+name: Package Repository Maintenance
+
+on:
+  workflow_dispatch:
+    inputs:
+      operation:
+        description: Inspect state, restore a package selection, or recover an abandoned lock
+        type: choice
+        options: [status, rollback, recover-lock]
+        default: status
+      snapshot:
+        description: Exact archived snapshot SHA-256 for rollback
+        type: string
+      expected_version:
+        description: Top-level version from the status summary (S3 VersionId, not snapshot SHA-256)
+        type: string
+      confirm_owner_stopped:
+        description: The previous workflow/process and its in-flight writes have stopped
+        type: boolean
+        default: false
+
+permissions: {}
+
+concurrency:
+  group: package-repositories
+  cancel-in-progress: false
+
+jobs:
+  # Inspect S3 state, prepare a rollback, or recover a lock with the publishing role.
+  # Every job uses the vdev version pinned by the reviewed workflow checkout.
+  # Available even when automated publication is disabled.
+  #
+  # Recover after a workflow dies during upload (GitHub Actions -> this workflow):
+  # 1. Stop/cancel the old workflow and confirm its runner/process and in-flight
+  #    writes have stopped. A failed workflow alone is not that confirmation.
+  # 2. Run on master with operation=status and obtain Vector team approval. In
+  #    the maintain job summary, inspect state.held, state.owner, state.pending,
+  #    state.recover_after and the top-level version. If held=false, skip unlocking.
+  # 3. Wait until state.recover_after (UTC, 65 minutes after lock acquisition).
+  #    The lock never expires automatically; do not edit/delete the S3 control object.
+  # 4. Run on master with operation=recover-lock, expected_version set to that
+  #    version, and confirm_owner_stopped=true; obtain team approval again.
+  #    If the version changed, repeat status and reassess the owner before retrying.
+  # 5. Confirm state.held=false in the recovery summary. Unlocking preserves
+  #    active/pending and does not repair partially uploaded repository metadata.
+  # 6. Resume by dispatching Package Repositories with the same Release Suite
+  #    run_id, tag and channel; it restores the saved signed S3 bundle. For an
+  #    interrupted rollback, dispatch operation=rollback with the same snapshot.
+  #    To abandon an unfinished release, use operation=rollback with state.active
+  #    as snapshot. Finish the resumed publication/rollback and its verification
+  #    before publishing a different release.
+  maintain:
+    if: github.repository_id == '146327667' && github.ref == 'refs/heads/master'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    environment: package-repositories-publishing
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+      - name: Validate the requested maintenance operation
+        env:
+          OPERATION: ${{ inputs.operation }}
+          SNAPSHOT: ${{ inputs.snapshot }}
+          EXPECTED_VERSION: ${{ inputs.expected_version }}
+          OWNER_STOPPED: ${{ inputs.confirm_owner_stopped }}
+        run: |
+          case "$OPERATION" in
+            status) ;;
+            rollback) [[ "$SNAPSHOT" =~ ^[0-9a-f]{64}$ ]] ;;
+            recover-lock) test -n "$EXPECTED_VERSION" && test "$OWNER_STOPPED" = true ;;
+            *) exit 1 ;;
+          esac
+      - uses: ./.github/actions/package-repository-tools
+      - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0
+        with:
+          role-to-assume: ${{ vars.PACKAGE_REPOSITORIES_ROLE_ARN }}
+          aws-region: us-east-1
+          role-duration-seconds: 900
+          role-session-name: package-maintenance-${{ github.run_id }}
+          allowed-account-ids: "486234852809"
+          unset-current-credentials: true
+      - name: Prepare rollback or run maintenance, and record the current state
+        env:
+          PACKAGE_REPOSITORIES_ROLE_ARN: ${{ vars.PACKAGE_REPOSITORIES_ROLE_ARN }}
+          OPERATION: ${{ inputs.operation }}
+          SNAPSHOT: ${{ inputs.snapshot }}
+          EXPECTED_VERSION: ${{ inputs.expected_version }}
+        run: |
+          case "$OPERATION" in
+            status) ;;
+            rollback)
+              "$VDEV" release repositories rollback base "$SNAPSHOT"
+              ;;
+            recover-lock)
+              "$VDEV" release repositories recover-lock \
+                --expected-version "$EXPECTED_VERSION" --confirm-owner-stopped
+              ;;
+          esac
+          mkdir -p target
+          "$VDEV" release repositories status > target/package-repository-status.json
+          cat target/package-repository-status.json
+          {
+            echo '```json'
+            cat target/package-repository-status.json
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        if: inputs.operation == 'rollback'
+        with:
+          name: package-repository-rollback-base
+          path: target/package-repositories/
+          retention-days: 1
+          if-no-files-found: error
+
+  # Refresh apt dates/signatures so cached clients accept the archived package selection.
+  # Reuse saved rollback output on retries; the signing job has no AWS credentials.
+  sign:
+    needs: maintain
+    if: inputs.operation == 'rollback'
+    runs-on: ubuntu-24.04
+    environment: package-repositories-signing
+    timeout-minutes: 20
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+      - uses: ./.github/actions/package-repository-tools
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          name: package-repository-rollback-base
+          path: target/package-repositories
+      - id: cached
+        run: |
+          if test -f target/package-repositories/result/manifest.json; then
+            echo 'ready=true' >> "$GITHUB_OUTPUT"
+          fi
+      - name: Sign fresh apt roots for the selected archived indexes
+        if: steps.cached.outputs.ready != 'true'
+        env:
+          PACKAGE_SIGNING_SUBKEY: ${{ secrets.PACKAGE_SIGNING_SUBKEY }}
+          PACKAGE_SIGNING_PASSPHRASE: ${{ secrets.PACKAGE_SIGNING_PASSPHRASE }}
+        run: '"$VDEV" release repositories rollback prepare'
+      # GPG homes use private /dev/shm directories, outside this artifact.
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: package-repository-rollback-signed
+          path: target/package-repositories/
+          retention-days: 1
+          if-no-files-found: error
+
+  # Restore the selected repository state under the shared S3 lock, rejecting stale plans.
+  # Uses AWS OIDC without the signing key; installed clients are not downgraded.
+  publish:
+    needs: [maintain, sign]
+    runs-on: ubuntu-24.04
+    environment: package-repositories-publishing
+    timeout-minutes: 20
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+      - uses: ./.github/actions/package-repository-tools
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          name: package-repository-rollback-signed
+          path: target/package-repositories
+      - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0
+        with:
+          role-to-assume: ${{ vars.PACKAGE_REPOSITORIES_ROLE_ARN }}
+          aws-region: us-east-1
+          role-duration-seconds: 900
+          allowed-account-ids: "486234852809"
+          unset-current-credentials: true
+      - name: Publish the fresh rollback against its expected repository state
+        env:
+          PACKAGE_REPOSITORIES_ROLE_ARN: ${{ vars.PACKAGE_REPOSITORIES_ROLE_ARN }}
+        run: |
+          "$VDEV" release repositories rollback publish
+          "$VDEV" release repositories status > target/package-repository-status.json
+          cat target/package-repository-status.json
+          {
+            echo '```json'
+            cat target/package-repository-status.json
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+  # Verify the restored indexes and fresh signed roots through public HTTPS.
+  endpoints:
+    needs: [maintain, publish]
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+      - uses: ./.github/actions/package-repository-tools
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          name: package-repository-rollback-signed
+          path: target/package-repositories
+      - name: Verify the selected indexes and fresh roots through public HTTPS
+        run: '"$VDEV" release repositories rollback verify'