Add a separate action for removing old wheels

.github/workflows/remove-wheels.yml

@@ -3,7 +3,7 @@ name: Remove old wheels
 on:
   # Run daily at 1:23 UTC
   schedule:
-  - cron:  '23 1 * * *'
+    - cron: '23 1 * * *'
   workflow_dispatch:
 
 concurrency:
@@ -30,16 +30,7 @@ jobs:
       name: remove-old-wheels
 
     steps:
-      - name: Install micromamba and anaconda-client
-        uses: mamba-org/setup-micromamba@f8b8a1e23a26f60a44c853292711bacfd3eac822  # v1.9.0
-        with:
-          environment-name: remove-wheels
-          create-args: >-
-            python=3.12
-            anaconda-client=1.12.3
-            curl
-            jq
-
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  # v4.1.7
       - name: Show environment
         run: env
 
@@ -49,64 +40,9 @@ jobs:
           echo ""
           anaconda remove --help
 
-      - name: Query package index for packages
-        run: |
-          curl https://raw.githubusercontent.com/scientific-python/upload-nightly-action/main/packages-ignore-from-cleanup.txt --output packages-ignore-from-cleanup.txt
-          anaconda show "${ANACONDA_USER}" &> >(grep "${ANACONDA_USER}/") | \
-              awk '{print $1}' | \
-              sed 's|.*/||g' | \
-              grep -vf packages-ignore-from-cleanup.txt > package-names.txt
-
-      - name: Remove old uploads to save space
-        run: |
-          # Remove all _but_ the last ${N_LATEST_UPLOADS} package versions and
-          # remove all package versions older than 30 days.
-
-          if [ -s package-names.txt ]; then
-              threshold_date="$(date +%F -d '30 days ago')"
-
-              # Remember can't quote subshell as need to split on (space seperated) token
-              for package_name in $(cat package-names.txt); do
-
-                  echo -e "\n# package: ${package_name}"
-
-                  curl --silent https://api.anaconda.org/package/"${ANACONDA_USER}/${package_name}" | \
-                      jq -r '.releases[].version' > package-versions.txt
-                  head --lines "-${N_LATEST_UPLOADS}" package-versions.txt > remove-package-versions.txt
-
-                  for package_version in $(cat package-versions.txt); do
-                    # c.f. https://github.com/Anaconda-Platform/anaconda-client/issues/682#issuecomment-1677283067
-                    upload_date=$(curl --silent https://api.anaconda.org/release/"${ANACONDA_USER}/${package_name}/${package_version}" | \
-                        jq -r '.distributions[].upload_time' | \
-                        sort | \
-                        tail --lines 1 | \
-                        awk '{print $1}')
-
-                    # check upload_date is YYYY-MM-DD formatted
-                    # c.f. https://github.com/scientific-python/upload-nightly-action/issues/73
-                    if [[ "${upload_date}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then
-                        if [[ "${upload_date}" < "${threshold_date}" ]]; then
-                            echo "# ${ANACONDA_USER}/${package_name}/${package_version} last uploaded on ${upload_date}"
-                            echo "${package_version}" >> remove-package-versions.txt
-                        fi
-                    else
-                        echo "# ERROR: ${ANACONDA_USER}/${package_name}/${package_version} upload date ${upload_date} is not YYYY-MM-DD."
-                    fi
-
-                  done
-
-                  if [ -s remove-package-versions.txt ]; then
-                      # Guard against duplicate entries from packages over
-                      # count and time thresholds
-                      sort --output remove-package-versions.txt --unique remove-package-versions.txt
-
-                      for package_version in $(cat remove-package-versions.txt); do
-                          echo "# Removing ${ANACONDA_USER}/${package_name}/${package_version}"
-                          anaconda --token ${{ secrets.ANACONDA_TOKEN }} remove \
-                            --force \
-                            "${ANACONDA_USER}/${package_name}/${package_version}"
-                      done
-                  fi
-
-              done
-          fi
+      - name: Remove old wheels
+        uses: ./remove-wheels
+        with:
+          n_latest_uploads: ${{ env.N_LATEST_UPLOADS }}
+          anaconda_nightly_upload_organization: ${{ env.ANACONDA_USER }}
+          anaconda_token: ${{ secrets.ANACONDA_TOKEN }}

README.md

@@ -1,7 +1,8 @@
 # Nightly upload
 
-This is a GitHub Action that uploads nightly builds to the [scientific-python nightly channel][],
-as recommended in [SPEC4 — Using and Creating Nightly Wheels][].
+This is a GitHub Action that uploads (and helps remove) nightly builds to
+the [scientific-python nightly channel][], as recommended in
+[SPEC4 — Using and Creating Nightly Wheels][].
 
 In a GitHub Actions workflow (`.github/workflows/*.yaml`), use the
 following snippet on a Linux or macOS runner to upload built wheels to the
@@ -18,11 +19,45 @@ jobs:
         anaconda_nightly_upload_token: ${{secrets.UPLOAD_TOKEN}}
 ```
 
-Note that we recommend pinning the action against a specific SHA
+> [!IMPORTANT]
+> Note that we recommend pinning the action against a specific SHA
 (rather than a tag), to guard against the unlikely event of upstream
 being compromised.
 
-## Updating the action
+# Removing old nightly builds
+
+This repository also ships with an action to ease removals of older nightly wheels from a channel.
+
+To use this functionality, add the following snippet to your workflow:
+
+```yml
+jobs:
+  steps:
+    ...
+    - name: Remove old wheels
+      uses: scientific-python/upload-nightly-action/remove-wheels@cantknowhashyet # 0.6.0
+      with:
+        n_latest_uploads: ${{ env.N_LATEST_UPLOADS }}
+        anaconda_nightly_upload_token: ${{secrets.UPLOAD_TOKEN}}
+```
+
+Which will remove all but the `n_latest_uploads` latest uploads from the channel. This is useful
+to avoid hosting outdated development versions, as well as to clean up space.
+
+The channel to remove wheels from is set to the ``scientific-python-nightly-wheels`` channel
+by default. If you are using this channel, please note that this repository will automatically
+clean up old artifacts for you.
+
+If you do not wish to have this automated cleanup, please open an issue on this repository
+to be added to the list of packages exempt from it. The current ones are named in 
+`packages-ignore-from-cleanup.txt`.
+
+Please refer to the [artifact cleanup policy][] for more information.
+
+To remove wheels from a different channel, set the ``anaconda_nightly_upload_organization``
+variable to the desired organization.
+
+## Updating the actions
 
 You can [use Dependabot to keep the GitHub Action up to date][],
 with a `.github/dependabot.yml` config file similar to:
@@ -45,7 +80,7 @@ then generate a token at `https://anaconda.org/<anaconda cloud user name>/settin
 with permissions to _Allow write access to the API site_ and _Allow uploads to Standard Python repositories_,
 and add the token as a secret to your GitHub repository.
 
-## Using a different channel
+## Using a different channel other than ``scientific-python-nightly-wheels``
 
 This Github Action can upload your nightly builds to a different channel. To do so,
 define the `anaconda_nightly_upload_organization` variable. Furthermore,
@@ -112,3 +147,4 @@ dependencies:
 [PyPI]: https://pypi.org/
 [scientific-python nightly channel]: https://anaconda.org/scientific-python-nightly-wheels
 [SPEC4 — Using and Creating Nightly Wheels]: https://scientific-python.org/specs/spec-0004/
+[artifact cleanup policy]: #artifact-cleanup-policy-at-the-scientific-python-nightly-wheels-channel

action.yml

@@ -1,4 +1,4 @@
-name: Upload Nightly
+name: Scientific Python / Upload Nightly Wheels
 description: A GitHub Action to upload artifacts nightly
 permissions:
   actions: read

remove-wheels/action.yml

@@ -0,0 +1,50 @@
+name: Scientific Python / Remove Old Wheels
+description: A GitHub Action to remove old wheels
+permissions:
+  actions: read
+  contents: read
+  metadata: read
+author: "Scientific-Python"
+# TODO: have to think about versioning; whether to version separately, or
+# for it to be in sync with the version for the upload action
+version: "0.1.0"
+
+inputs:
+  n_latest_uploads:
+    description: 'The number of previous wheel uploads to keep'
+    required: false
+    default: '5'
+  anaconda_nightly_upload_organization:
+    description: 'Anaconda Cloud organisation name to remove the wheels from'
+    required: false
+    default: scientific-python-nightly-wheels
+  anaconda_token:
+    description: 'Anaconda Cloud API token'
+    required: true
+
+# TODO: Linux only for now, need to see how to add macOS support
+runs:
+  using: "composite"
+  steps:
+    - name: Set up pixi
+      uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659  # v0.8.1
+      with:
+        locked: true
+        cache: true
+        cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
+        # Avoid post cleanup errors if action run multiple times
+        post-cleanup: false
+
+    - name: Install dependencies
+      shell: bash
+      run: |
+        sudo apt-get update && sudo apt-get install -y curl jq
+
+    - name: Remove old wheels
+      shell: bash
+      env:
+        INPUT_N_LATEST_UPLOADS: ${{ inputs.n_latest_uploads }}
+        INPUT_ANACONDA_USER: ${{ inputs.anaconda_user }}
+        INPUT_ANACONDA_TOKEN: ${{ inputs.anaconda_token }}
+      run: |
+        pixi run remove_old_wheels.sh

remove_wheels.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+# fail on undefined variables
+set -u
+# Prevent pipe errors to be silenced
+set -o pipefail
+# Exit if any command exit as non-zero
+set -e
+# enable trace mode (print what it does)
+set -x
+
+# get the anaconda token from the github secrets
+#
+# this is to prevent accidental removals
+echo "Getting anaconda token from github secrets..."
+
+ANACONDA_USER="${INPUT_ANACONDA_USER}"
+ANACONDA_TOKEN="${INPUT_ANACONDA_TOKEN}"
+N_LATEST_UPLOADS="${INPUT_N_LATEST_UPLOADS}"
+
+
+# if the ANACONDA_TOKEN is empty, exit with status -1
+# this is to prevent accidental removals
+if [ -z "${ANACONDA_TOKEN}" ]; then
+  echo "ANACONDA_TOKEN is empty, exiting..."
+  exit -1
+fi
+
+# if the N_LATEST_UPLOADS is empty, exit with status -1
+# as this should be set in by the user and it is better
+# to fail on this to signal a problem. i.e.,
+# explicit is better than implicit.
+if [ -z "${N_LATEST_UPLOADS}" ]; then
+  echo "N_LATEST_UPLOADS is empty, exiting..."
+  exit -1
+fi
+
+
+# Query the package index for packages
+#
+# TODO: should be possible to alter this, since separating the workflow
+# into two steps, one for uploading and one for cleanup, should make it
+# possible for users to manually trigger the cleanup step before/after the
+# upload step has completed in their own repos instead of us having to do it.
+#
+# TODO: raises questions on how to moderate cleanups among multiple users
+# operating on the same channel, but that might be a different issue.
+curl https://raw.githubusercontent.com/scientific-python/upload-nightly-action/main/packages-ignore-from-cleanup.txt --output packages-ignore-from-cleanup.txt
+anaconda show "${ANACONDA_USER}" &> >(grep "${ANACONDA_USER}/") | \
+    awk '{print $1}' | \
+    sed 's|.*/||g' | \
+    grep -vf packages-ignore-from-cleanup.txt > package-names.txt
+
+# Remove old uploads to save space
+# Remove all _but_ the last ${N_LATEST_UPLOADS} package versions and
+# remove all package versions older than 30 days.
+if [ -s package-names.txt ]; then
+    threshold_date="$(date +%F -d '30 days ago')"
+
+    # Remember can't quote subshell as need to split on (space separated) token
+    for package_name in $(cat package-names.txt); do
+    # TODO: this outer loop can be removed when ready since there will be
+    # just one package to remove when the action is triggered manually from
+    # a user's (different) repo.
+
+        echo -e "\n# package: ${package_name}"
+
+        curl --silent https://api.anaconda.org/package/"${ANACONDA_USER}/${package_name}" | \
+            jq -r '.releases[].version' > package-versions.txt
+        head --lines "-${N_LATEST_UPLOADS}" package-versions.txt > remove-package-versions.txt
+
+        for package_version in $(cat package-versions.txt); do
+          # c.f. https://github.com/Anaconda-Platform/anaconda-client/issues/682#issuecomment-1677283067
+          upload_date=$(curl --silent https://api.anaconda.org/release/"${ANACONDA_USER}/${package_name}/${package_version}" | \
+              jq -r '.distributions[].upload_time' | \
+              sort | \
+              tail --lines 1 | \
+              awk '{print $1}')
+
+          # check upload_date is YYYY-MM-DD formatted
+          # c.f. https://github.com/scientific-python/upload-nightly-action/issues/73
+          if [[ "${upload_date}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then
+              if [[ "${upload_date}" < "${threshold_date}" ]]; then
+                  echo "# ${ANACONDA_USER}/${package_name}/${package_version} last uploaded on ${upload_date}"
+                  echo "${package_version}" >> remove-package-versions.txt
+              fi
+          else
+              echo "# ERROR: ${ANACONDA_USER}/${package_name}/${package_version} upload date ${upload_date} is not YYYY-MM-DD."
+          fi
+
+        done
+
+        if [ -s remove-package-versions.txt ]; then
+            # Guard against duplicate entries from packages over
+            # count and time thresholds
+            sort --output remove-package-versions.txt --unique remove-package-versions.txt
+
+            for package_version in $(cat remove-package-versions.txt); do
+                echo "# Removing ${ANACONDA_USER}/${package_name}/${package_version}"
+                anaconda --token "${ANACONDA_TOKEN}" remove \
+                  --force \
+                  "${ANACONDA_USER}/${package_name}/${package_version}"
+            done
+        fi
+
+    done
+fi
+
+echo "Finished removing old wheels except the last ${N_LATEST_UPLOADS} uploads from the ${ANACONDA_USER} channel."