Fix the self-hosted APT repo publish workflow

.github/workflows/platform-apt-repo-publish.yml

@@ -12,9 +12,7 @@ on:
         required: true
         type: string
     secrets:
-      GPG_SIGNING_KEY:
-        required: true
-      GPG_PASSPHRASE:
+      DEBIAN_REPO_SIGNING_KEY:
         required: true
       GH_UPLOADER_GCP_SA_CREDENTIALS:
         required: true
@@ -49,30 +47,29 @@ jobs:
       - name: Install reprepro and Debian packaging tools
         run: |
           sudo apt-get update
-          sudo apt-get install -y reprepro dpkg-dev debhelper
-      - name: Import the GPG signing key and preset its passphrase
+          sudo apt-get install -y reprepro dpkg-dev debhelper build-essential
+      - name: Import the Debian repo signing key
         id: gpg
-        env:
-          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
         run: |
-          echo -n "${{ secrets.GPG_SIGNING_KEY }}" | base64 --decode | gpg --import --no-tty --batch --yes
-          # Derive the signing key id from the imported key (no separate secret),
-          # mirroring the release workflow.
-          GPG_KEY_ID=$(gpg --list-secret-keys --keyid-format long --with-colons | grep ^sec | head -1 | cut -d: -f5)
+          # Same key, keyring layout and loopback pinentry as the release_ppa
+          # workflow in learningequality/kolibri-installer-debian, which publishes
+          # the Pages repos this one replaces.
+          GNUPGHOME=$(mktemp -d)
+          export GNUPGHOME
+          echo "GNUPGHOME=$GNUPGHOME" >> "$GITHUB_ENV"
+          echo "pinentry-mode loopback" > "$GNUPGHOME/gpg.conf"
+          echo "allow-loopback-pinentry" > "$GNUPGHOME/gpg-agent.conf"
+          echo "${{ secrets.DEBIAN_REPO_SIGNING_KEY }}" | gpg --batch --import
+          # The repo must be signed by the key kolibri-archive-keyring ships, or no
+          # client trusting that keyring can verify it. Take the key id from the
+          # committed key and require the imported secret key to match.
+          GPG_KEY_ID=$(gpg --show-keys --with-colons platforms/apt-repo/keyring/kolibri-archive-keyring.asc | awk -F: '/^fpr:/ {print $10; exit}')
+          gpg --list-secret-keys "$GPG_KEY_ID" >/dev/null 2>&1 || {
+            echo "::error::DEBIAN_REPO_SIGNING_KEY does not hold $GPG_KEY_ID, the key shipped in kolibri-archive-keyring.asc"
+            exit 1
+          }
+          echo "$GPG_KEY_ID:6:" | gpg --batch --import-ownertrust
           echo "key-id=$GPG_KEY_ID" >> "$GITHUB_OUTPUT"
-          # reprepro signs Release via gpgme/gpg-agent (no CLI passphrase flag),
-          # so preset the passphrase or signing hangs on the pinentry prompt.
-          echo "allow-preset-passphrase" >> ~/.gnupg/gpg-agent.conf
-          gpg-connect-agent reloadagent /bye
-          # gpg-preset-passphrase ships in a version-specific libexec dir, not on PATH.
-          for candidate in /usr/lib/gnupg*/gpg-preset-passphrase; do
-            [ -x "$candidate" ] && PRESET="$candidate" && break
-          done
-          KEYGRIP=$(gpg --with-keygrip --list-secret-keys | grep Keygrip | head -1 | awk '{print $3}')
-          # gpg-preset-passphrase reads the passphrase from stdin only, so pipe
-          # it in: printf is a shell builtin, so the secret never reaches argv
-          # or the log, and no copy is written to disk.
-          printf '%s' "$GPG_PASSPHRASE" | "$PRESET" --preset "$KEYGRIP"
       - uses: "google-github-actions/auth@v3"
         with:
           credentials_json: "${{ secrets.GH_UPLOADER_GCP_SA_CREDENTIALS }}"
@@ -86,7 +83,7 @@ jobs:
         run: dpkg-buildpackage -b -us -uc
       - name: Publish the .deb into the reprepro repo
         env:
-          KOLIBRI_APT_BUCKET: "${{ secrets.KOLIBRI_PUBLIC_RELEASE_GCS_BUCKET }}/apt"
+          KOLIBRI_APT_BUCKET: ${{ secrets.KOLIBRI_PUBLIC_RELEASE_GCS_BUCKET }}
           REPREPRO_SIGN_KEY: ${{ steps.gpg.outputs.key-id }}
         run: |
           KOLIBRI_KEYRING_DEB=$(ls "$GITHUB_WORKSPACE"/platforms/apt-repo/kolibri-archive-keyring_*.deb)
@@ -96,4 +93,4 @@ jobs:
         if: always()
         run: |
           gpgconf --kill gpg-agent || true
-          rm -rf ~/.gnupg
+          rm -rf "$GNUPGHOME"

.github/workflows/platform-debian-server-release.yml

@@ -14,6 +14,8 @@ on:
         required: true
       GPG_PASSPHRASE:
         required: true
+      DEBIAN_REPO_SIGNING_KEY:
+        required: true
       GH_UPLOADER_GCP_SA_CREDENTIALS:
         required: true
       KOLIBRI_PUBLIC_RELEASE_GCS_BUCKET:
@@ -266,7 +268,6 @@ jobs:
     with:
       deb-artifact-name: ${{ inputs.deb-artifact-name }}
     secrets:
-      GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
-      GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
+      DEBIAN_REPO_SIGNING_KEY: ${{ secrets.DEBIAN_REPO_SIGNING_KEY }}
       GH_UPLOADER_GCP_SA_CREDENTIALS: ${{ secrets.GH_UPLOADER_GCP_SA_CREDENTIALS }}
       KOLIBRI_PUBLIC_RELEASE_GCS_BUCKET: ${{ secrets.KOLIBRI_PUBLIC_RELEASE_GCS_BUCKET }}

.github/workflows/release_kolibri.yml

@@ -249,6 +249,7 @@ jobs:
       LP_CREDENTIALS: ${{ secrets.LP_CREDENTIALS }}
       GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
       GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
+      DEBIAN_REPO_SIGNING_KEY: ${{ secrets.DEBIAN_REPO_SIGNING_KEY }}
       GH_UPLOADER_GCP_SA_CREDENTIALS: ${{ secrets.GH_UPLOADER_GCP_SA_CREDENTIALS }}
       KOLIBRI_PUBLIC_RELEASE_GCS_BUCKET: ${{ secrets.KOLIBRI_PUBLIC_RELEASE_GCS_BUCKET }}
   container_image_publish:

platforms/apt-repo/README.md

@@ -0,0 +1,58 @@
+# Kolibri self-hosted APT repository
+
+Publishing infrastructure for `https://apt.learningequality.org/` (suite `stable`, component `main`), replacing the `learningequality.github.io/kolibri-server/` and `…/kolibri-installer-debian/` Pages repos.
+
+## Publishing model
+
+A [reprepro](https://salsa.debian.org/debian/reprepro) tree on the release GCS bucket under `downloads/kolibri/apt`. Each release read-modify-writes it, so prior packages and versions persist:
+
+1. `gcloud storage rsync` the tree **down**.
+2. `reprepro includedeb stable` the new `.deb`(s), skipping any version already published — reprepro rejects same-version bytes that differ.
+3. Export `pubkey.asc` from the signing key.
+4. Sync **up** one prefix at a time:
+   - `pool` first, so no index is published naming a file that is not there yet.
+   - `--checksums-only`, because reprepro rewrites `db/*.db` in place without changing size or mtime.
+   - `Cache-Control` per prefix: pool objects never change once published, while a cached index served against a newer pool is a client-side hash mismatch.
+
+`publish.sh` implements this; its header documents the env contract. `conf/distributions.in` is the suite template, with `SignWith` rendered in at runtime.
+
+`.github/workflows/platform-apt-repo-publish.yml` runs it in CI, serialized by a static `concurrency` group so two releases cannot clobber the shared state mid-write.
+
+The workflow never builds the `.deb` it publishes: `release_kolibri.yml` builds `kolibri-server` and passes the artifact name down through `platform-debian-server-release.yml`. Dispatch it by hand with `deb-url` to publish an already-released `.deb`.
+
+## New-user install
+
+`kolibri-archive-keyring` (`keyring/`) ships the apt source file (`/etc/apt/sources.list.d/kolibri.sources`) and the signing key (`/usr/share/keyrings/kolibri-archive-keyring.asc`). The workflow serves it at the repo root, so a fresh host can bootstrap before it has apt configured:
+
+```sh
+curl -fsSLO https://apt.learningequality.org/kolibri-archive-keyring.deb
+sudo dpkg -i kolibri-archive-keyring.deb
+sudo apt update && sudo apt install kolibri
+```
+
+## Self-migration of the installed base
+
+`migrate-apt-source.sh` defines `migrate_kolibri_apt_source()`, which rewrites any existing `github.io` Kolibri source under `/etc/apt/sources.list.d/` to `apt.learningequality.org`. It is idempotent, and a no-op when no such source is present (e.g. a Launchpad-PPA install). `kolibri-server`'s `postinst` calls it on `configure`, so existing installs migrate on their next `apt upgrade` with no user action.
+
+## seed_old_pages.sh — run once
+
+> **Warning:** a one-shot script for the cutover release only — the first release carrying the `postinst` migration snippet. It is not part of the ongoing release process.
+
+It pushes the cutover `.deb` into the old, soon-to-be-archived Pages repo(s), so stragglers still resolving the old URL receive it on their next upgrade and self-migrate. Each old repo is read-modify-written against its own committed `conf/`, and its `Release` re-signed with the key that repo's `SignWith` names, which must already be in the gpg keyring.
+
+```sh
+./seed_old_pages.sh --deb path/to/kolibri-server_<ver>_all.deb --repo kolibri-server
+./seed_old_pages.sh --deb path/to/kolibri_<ver>_all.deb        --repo kolibri-installer-debian
+```
+
+## Verification
+
+Each script in `tests/` covers one acceptance criterion and names it in its header. CI runs them with `APT_REPO_TESTS_STRICT=1`, so a missing tool fails rather than skips; standalone on a dev box, each skips cleanly when its tooling is absent.
+
+`e2e_cutover.sh` is the full containerized cutover: an old-source client is auto-rewritten and fetches its next update from the new host.
+
+## Ops prerequisites (#13720)
+
+- The uploader service account needs `storage.objects.delete` and `storage.objects.update` under `downloads/kolibri/apt` — `roles/storage.objectCreator`, which the release uploads run on, is not enough. A GCS overwrite is a delete plus a create, and every publish after the first rewrites `dists/`, `db/` and the root files. Reads come from the bucket's public `allUsers` grant.
+- `DEBIAN_REPO_SIGNING_KEY` must hold the private half of `platforms/raspberry-pi/files/learningequality.asc`, the key `keyring/kolibri-archive-keyring.asc` ships and the old Pages repos sign with. The workflow takes the key id from that committed key and aborts if the secret does not hold it.
+- Standing up the subdomain (DNS + Cloudflare in front of the bucket), running `seed_old_pages.sh` for the cutover release, the readthedocs user-manual update, and archiving the old Pages repos.

platforms/apt-repo/conf/distributions.in

@@ -2,7 +2,7 @@ Origin: Learning Equality
 Label: Kolibri
 Codename: stable
 Suite: stable
-Architectures: amd64 arm64
+Architectures: amd64 i386 arm64 armhf
 Components: main
 Description: Kolibri self-hosted APT repository
 SignWith: __REPREPRO_SIGN_KEY__