From 23bfdb663542c15f24a125dafdafadc3a5648f57 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Sat, 8 Aug 2026 22:29:05 +0000
Subject: [PATCH 1/4] Apply remaining changes

Co-authored-by: wdconinc <4656391+wdconinc@users.noreply.github.com>
---
 .github/workflows/build-push.yml | 137 +++++++++++++++++++++++++++++++
 containers/eic/Dockerfile        |  22 +++++
 2 files changed, 159 insertions(+)

diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml
index aa6cd35c1..1a83e7393 100644
--- a/.github/workflows/build-push.yml
+++ b/.github/workflows/build-push.yml
@@ -481,6 +481,58 @@ jobs:
           name: eic-${{ matrix.ENV }}-${{ matrix.arch }}-nightly-digest
           path: /tmp/digests/${{ matrix.arch }}-nightly.digest
           retention-days: 1
+      - name: Export spack.lock files
+        run: |
+          set -e
+          slugify() { echo "$1" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-+//; s/-+$//' | cut -c1-63; }
+          ARCH=$(echo "${{ matrix.PLATFORM }}" | sed 's|linux/||; s|/|_|g')
+          CI_COMMIT_REF_SLUG="$(slugify "${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}")"
+          CI_DEFAULT_BRANCH_SLUG="$(slugify "${GITHUB_BASE_REF:-${{ github.event.repository.default_branch }}}")"
+          CACHE_BASE="${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/buildcache:${{ matrix.BUILD_IMAGE }}${{ matrix.ENV }}"
+
+          ## Common build args: same as the main build (needed for BuildKit registry cache to match).
+          COMMON_ARGS=(
+            --file containers/eic/Dockerfile
+            --platform "${{ matrix.PLATFORM }}"
+            --build-arg "DOCKER_REGISTRY=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/"
+            --build-arg "INTERNAL_TAG=${{ env.INTERNAL_TAG }}"
+            --build-arg "BUILDER_IMAGE=${{ matrix.BUILDER_IMAGE }}"
+            --build-arg "RUNTIME_IMAGE=${{ matrix.RUNTIME_IMAGE }}"
+            --build-arg "ENV=${{ matrix.ENV }}"
+            --cache-from "type=registry,ref=${CACHE_BASE}-default-${CI_COMMIT_REF_SLUG}-${ARCH}"
+            --cache-from "type=registry,ref=${CACHE_BASE}-default-${CI_DEFAULT_BRANCH_SLUG}-${ARCH}"
+            --cache-from "type=registry,ref=${CACHE_BASE}-nightly-${CI_COMMIT_REF_SLUG}-${ARCH}"
+            --cache-from "type=registry,ref=${CACHE_BASE}-nightly-${CI_DEFAULT_BRANCH_SLUG}-${ARCH}"
+            --build-context spack-environment=spack-environment
+          )
+
+          mkdir -p /tmp/spack-locks
+
+          ## Export default spack.lock (available for all environments and targets).
+          ## This is a fast export build that hits registry cache from the main build.
+          docker buildx build "${COMMON_ARGS[@]}" \
+            --target spack_lock_default_export \
+            --output "type=local,dest=/tmp/spack-lock-default-export" \
+            containers/eic
+          mv /tmp/spack-lock-default-export/spack.lock /tmp/spack-locks/default.spack.lock 2>/dev/null || true
+
+          ## Export custom (epic) spack.lock (only present for full builds).
+          if [ "${{ matrix.target }}" = "final" ]; then
+            docker buildx build "${COMMON_ARGS[@]}" \
+              --target spack_lock_custom_export \
+              --output "type=local,dest=/tmp/spack-lock-custom-export" \
+              containers/eic || true
+            mv /tmp/spack-lock-custom-export/spack.lock /tmp/spack-locks/custom.spack.lock 2>/dev/null || true
+          fi
+
+          ls -la /tmp/spack-locks/
+      - name: Upload spack.lock files as artifacts
+        uses: actions/upload-artifact@v7
+        with:
+          name: spack-lock-${{ matrix.ENV }}-${{ matrix.arch }}
+          path: /tmp/spack-locks/*.spack.lock
+          if-no-files-found: warn
+          retention-days: 90
 
   eic-manifest:
     name: Push ${{ matrix.BUILD_IMAGE }}${{ matrix.ENV }} ${{ matrix.BUILD_TYPE }} manifest
@@ -575,6 +627,91 @@ jobs:
             $TAG_ARGS \
             ${{ steps.digests.outputs.all }}
 
+  spack-diff:
+    name: Spack diff (${{ matrix.ENV }}, ${{ matrix.arch }})
+    if: github.event_name == 'pull_request' || github.event_name == 'merge_group'
+    needs: eic-manifest
+    runs-on: ${{ matrix.runner }}
+    container:
+      image: ghcr.io/eic/eic_ci:pipeline-${{ github.run_id }}-default
+      credentials:
+        username: ${{ secrets.GHCR_REGISTRY_USER }}
+        password: ${{ secrets.GHCR_REGISTRY_TOKEN }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - ENV: ci
+          arch: amd64
+          runner: ubuntu-latest
+        - ENV: xl
+          arch: amd64
+          runner: ubuntu-latest
+    steps:
+    - name: Download new spack.lock artifact
+      uses: actions/download-artifact@v8
+      with:
+        name: spack-lock-${{ matrix.ENV }}-${{ matrix.arch }}
+        path: /tmp/spack-locks-new/
+    - name: Download previous spack.lock artifact
+      id: download_ref
+      uses: dawidd6/action-download-artifact@v21
+      with:
+        branch: ${{ github.base_ref || github.event.merge_group.base_ref }}
+        path: /tmp/spack-locks-ref/
+        name: spack-lock-${{ matrix.ENV }}-${{ matrix.arch }}
+        workflow: ".github/workflows/build-push.yml"
+        workflow_conclusion: "success"
+        if_no_artifact_found: warn
+    - name: Show spack environment diff
+      if: steps.download_ref.outputs.found_artifact == 'true'
+      shell: bash --login -eo pipefail {0}
+      continue-on-error: true
+      run: |
+        cat > /tmp/compare_spack_locks.py << 'PYEOF'
+        import json, subprocess, sys, os, glob
+        old_dir, new_dir = sys.argv[1], sys.argv[2]
+        def load_lock(path):
+            try:
+                with open(path) as f:
+                    data = json.load(f)
+                return {v['name']: {'version': v.get('version', '?'), 'hash': k}
+                        for k, v in data.get('concrete_specs', {}).items()}
+            except (FileNotFoundError, KeyError, json.JSONDecodeError):
+                return None
+        for lock_file in sorted(glob.glob(os.path.join(new_dir, '*.spack.lock'))):
+            lock_name = os.path.basename(lock_file)[:-len('.spack.lock')]
+            old = load_lock(os.path.join(old_dir, os.path.basename(lock_file)))
+            new_lock = load_lock(lock_file)
+            if old is None or new_lock is None:
+                print(f'Skipping {lock_name}: no reference lock file found')
+                continue
+            print(f'\n=== Spack environment diff: {lock_name} ===')
+            all_names = sorted(set(old) | set(new_lock))
+            changed, any_change = [], False
+            for name in all_names:
+                if name not in old:
+                    print(f'  + {name}@{new_lock[name]["version"]}'); any_change = True
+                elif name not in new_lock:
+                    print(f'  - {name}@{old[name]["version"]}'); any_change = True
+                elif old[name]['hash'] != new_lock[name]['hash']:
+                    o, n = old[name]['version'], new_lock[name]['version']
+                    print(f'  ~ {name}: {o} -> {n}'); any_change = True
+                    if o != n:
+                        changed.append((name, o, n))
+            if not any_change:
+                print('  (no changes)')
+            for name, o, n in changed[:3]:
+                print(f'\n  --- spack diff {name}@{o} {name}@{n} ---')
+                try:
+                    r = subprocess.run(['spack', 'diff', f'{name}@{o}', f'{name}@{n}'],
+                        capture_output=True, text=True, timeout=60)
+                    print('\n'.join(f'  {ln}' for ln in (r.stdout or r.stderr or '(no output)').splitlines()))
+                except Exception as e:
+                    print(f'  ({e})')
+        PYEOF
+        python3 /tmp/compare_spack_locks.py /tmp/spack-locks-ref /tmp/spack-locks-new
+
   npsim-gun:
     name: npsim (gun, ${{ matrix.particle }}, ${{ matrix.detector_config }}, ${{ matrix.arch }})
     runs-on: ${{ matrix.runner }}
diff --git a/containers/eic/Dockerfile b/containers/eic/Dockerfile
index 1d212b74a..addcf4382 100644
--- a/containers/eic/Dockerfile
+++ b/containers/eic/Dockerfile
@@ -444,3 +444,25 @@ RUN echo -e "\n - eic_container: ${CI_COMMIT_SHA}" | tee -a /etc/eic_info /etc/j
 
 ## Hotfix for misbehaving OSG nodes
 RUN mkdir /hadoop /localscratch
+
+
+## ========================================================================================
+## spack_lock_default_export
+## - scratch-based export stage for CI artifact extraction of the default spack.lock
+## ========================================================================================
+FROM scratch AS spack_lock_default_export
+ARG ENV=xl
+COPY --from=builder_concretization_default \
+  /opt/spack-environment/${ENV}/spack.lock \
+  /
+
+
+## ========================================================================================
+## spack_lock_custom_export
+## - scratch-based export stage for CI artifact extraction of the custom spack.lock
+## ========================================================================================
+FROM scratch AS spack_lock_custom_export
+ARG ENV=xl
+COPY --from=builder_concretization_custom \
+  /opt/spack-environment/${ENV}/epic/spack.lock \
+  /

From f2775bfeb5afe498339e5b3887b079b3700794e8 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Sat, 8 Aug 2026 23:11:16 +0000
Subject: [PATCH 2/4] Apply remaining changes

Co-authored-by: wdconinc <4656391+wdconinc@users.noreply.github.com>
---
 .github/workflows/build-push.yml | 58 +++++++++-----------------------
 1 file changed, 15 insertions(+), 43 deletions(-)

diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml
index 1a83e7393..cf402a5ad 100644
--- a/.github/workflows/build-push.yml
+++ b/.github/workflows/build-push.yml
@@ -668,49 +668,21 @@ jobs:
       shell: bash --login -eo pipefail {0}
       continue-on-error: true
       run: |
-        cat > /tmp/compare_spack_locks.py << 'PYEOF'
-        import json, subprocess, sys, os, glob
-        old_dir, new_dir = sys.argv[1], sys.argv[2]
-        def load_lock(path):
-            try:
-                with open(path) as f:
-                    data = json.load(f)
-                return {v['name']: {'version': v.get('version', '?'), 'hash': k}
-                        for k, v in data.get('concrete_specs', {}).items()}
-            except (FileNotFoundError, KeyError, json.JSONDecodeError):
-                return None
-        for lock_file in sorted(glob.glob(os.path.join(new_dir, '*.spack.lock'))):
-            lock_name = os.path.basename(lock_file)[:-len('.spack.lock')]
-            old = load_lock(os.path.join(old_dir, os.path.basename(lock_file)))
-            new_lock = load_lock(lock_file)
-            if old is None or new_lock is None:
-                print(f'Skipping {lock_name}: no reference lock file found')
-                continue
-            print(f'\n=== Spack environment diff: {lock_name} ===')
-            all_names = sorted(set(old) | set(new_lock))
-            changed, any_change = [], False
-            for name in all_names:
-                if name not in old:
-                    print(f'  + {name}@{new_lock[name]["version"]}'); any_change = True
-                elif name not in new_lock:
-                    print(f'  - {name}@{old[name]["version"]}'); any_change = True
-                elif old[name]['hash'] != new_lock[name]['hash']:
-                    o, n = old[name]['version'], new_lock[name]['version']
-                    print(f'  ~ {name}: {o} -> {n}'); any_change = True
-                    if o != n:
-                        changed.append((name, o, n))
-            if not any_change:
-                print('  (no changes)')
-            for name, o, n in changed[:3]:
-                print(f'\n  --- spack diff {name}@{o} {name}@{n} ---')
-                try:
-                    r = subprocess.run(['spack', 'diff', f'{name}@{o}', f'{name}@{n}'],
-                        capture_output=True, text=True, timeout=60)
-                    print('\n'.join(f'  {ln}' for ln in (r.stdout or r.stderr or '(no output)').splitlines()))
-                except Exception as e:
-                    print(f'  ({e})')
-        PYEOF
-        python3 /tmp/compare_spack_locks.py /tmp/spack-locks-ref /tmp/spack-locks-new
+        for new_lock in /tmp/spack-locks-new/*.spack.lock; do
+          lock_name=$(basename "${new_lock}" .spack.lock)
+          ref_lock=/tmp/spack-locks-ref/${lock_name}.spack.lock
+          echo "=== Spack environment diff: ${lock_name} ==="
+          cat "${new_lock}" > /tmp/${lock_name}.spack.lock
+          if [ -f "${ref_lock}" ]; then
+            jq -r '.concrete_specs | to_entries[] | "\(.value.name) \(.key) \(.value.version)"' "${ref_lock}" | sort > /tmp/ref_specs.txt
+            jq -r '.concrete_specs | to_entries[] | "\(.value.name) \(.key) \(.value.version)"' "${new_lock}" | sort > /tmp/new_specs.txt
+            join /tmp/ref_specs.txt /tmp/new_specs.txt | while read -r name ref_hash new_hash ref_ver new_ver; do
+              if [ "${ref_hash}" != "${new_hash}" ] && [ "${ref_ver}" != "${new_ver}" ]; then
+                spack diff "${name}@${ref_ver}" "${name}@${new_ver}" || true
+              fi
+            done
+          fi
+        done
 
   npsim-gun:
     name: npsim (gun, ${{ matrix.particle }}, ${{ matrix.detector_config }}, ${{ matrix.arch }})

From 88803004d683927406752f044aebe3f8f4f54fda Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Sun, 9 Aug 2026 15:05:59 +0000
Subject: [PATCH 3/4] use docker cp instead of second docker buildx build to
 extract spack.lock files

Co-authored-by: wdconinc <4656391+wdconinc@users.noreply.github.com>
---
 .github/workflows/build-push.yml | 46 +++++++-------------------------
 containers/eic/Dockerfile        | 21 ---------------
 2 files changed, 10 insertions(+), 57 deletions(-)

diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml
index cf402a5ad..f43528301 100644
--- a/.github/workflows/build-push.yml
+++ b/.github/workflows/build-push.yml
@@ -484,46 +484,20 @@ jobs:
       - name: Export spack.lock files
         run: |
           set -e
-          slugify() { echo "$1" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-+//; s/-+$//' | cut -c1-63; }
-          ARCH=$(echo "${{ matrix.PLATFORM }}" | sed 's|linux/||; s|/|_|g')
-          CI_COMMIT_REF_SLUG="$(slugify "${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}")"
-          CI_DEFAULT_BRANCH_SLUG="$(slugify "${GITHUB_BASE_REF:-${{ github.event.repository.default_branch }}}")"
-          CACHE_BASE="${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/buildcache:${{ matrix.BUILD_IMAGE }}${{ matrix.ENV }}"
-
-          ## Common build args: same as the main build (needed for BuildKit registry cache to match).
-          COMMON_ARGS=(
-            --file containers/eic/Dockerfile
-            --platform "${{ matrix.PLATFORM }}"
-            --build-arg "DOCKER_REGISTRY=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/"
-            --build-arg "INTERNAL_TAG=${{ env.INTERNAL_TAG }}"
-            --build-arg "BUILDER_IMAGE=${{ matrix.BUILDER_IMAGE }}"
-            --build-arg "RUNTIME_IMAGE=${{ matrix.RUNTIME_IMAGE }}"
-            --build-arg "ENV=${{ matrix.ENV }}"
-            --cache-from "type=registry,ref=${CACHE_BASE}-default-${CI_COMMIT_REF_SLUG}-${ARCH}"
-            --cache-from "type=registry,ref=${CACHE_BASE}-default-${CI_DEFAULT_BRANCH_SLUG}-${ARCH}"
-            --cache-from "type=registry,ref=${CACHE_BASE}-nightly-${CI_COMMIT_REF_SLUG}-${ARCH}"
-            --cache-from "type=registry,ref=${CACHE_BASE}-nightly-${CI_DEFAULT_BRANCH_SLUG}-${ARCH}"
-            --build-context spack-environment=spack-environment
-          )
-
+          IMAGE_REPO="${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/${{ matrix.BUILD_IMAGE }}${{ matrix.ENV }}"
+          DIGEST=$(jq -r '."containerimage.digest"' /tmp/build-metadata-default.json)
           mkdir -p /tmp/spack-locks
 
-          ## Export default spack.lock (available for all environments and targets).
-          ## This is a fast export build that hits registry cache from the main build.
-          docker buildx build "${COMMON_ARGS[@]}" \
-            --target spack_lock_default_export \
-            --output "type=local,dest=/tmp/spack-lock-default-export" \
-            containers/eic
-          mv /tmp/spack-lock-default-export/spack.lock /tmp/spack-locks/default.spack.lock 2>/dev/null || true
-
-          ## Export custom (epic) spack.lock (only present for full builds).
+          ## Create a temporary container from the built image (already pushed by digest) and
+          ## docker cp the spack.lock files out of it — no second build needed.
+          CONTAINER=$(docker create "${IMAGE_REPO}@${DIGEST}")
+          docker cp "${CONTAINER}:/opt/spack-environment/${{ matrix.ENV }}/spack.lock" \
+            /tmp/spack-locks/default.spack.lock 2>/dev/null || true
           if [ "${{ matrix.target }}" = "final" ]; then
-            docker buildx build "${COMMON_ARGS[@]}" \
-              --target spack_lock_custom_export \
-              --output "type=local,dest=/tmp/spack-lock-custom-export" \
-              containers/eic || true
-            mv /tmp/spack-lock-custom-export/spack.lock /tmp/spack-locks/custom.spack.lock 2>/dev/null || true
+            docker cp "${CONTAINER}:/opt/spack-environment/${{ matrix.ENV }}/epic/spack.lock" \
+              /tmp/spack-locks/custom.spack.lock 2>/dev/null || true
           fi
+          docker rm "${CONTAINER}"
 
           ls -la /tmp/spack-locks/
       - name: Upload spack.lock files as artifacts
diff --git a/containers/eic/Dockerfile b/containers/eic/Dockerfile
index addcf4382..4dc0b23d7 100644
--- a/containers/eic/Dockerfile
+++ b/containers/eic/Dockerfile
@@ -445,24 +445,3 @@ RUN echo -e "\n - eic_container: ${CI_COMMIT_SHA}" | tee -a /etc/eic_info /etc/j
 ## Hotfix for misbehaving OSG nodes
 RUN mkdir /hadoop /localscratch
 
-
-## ========================================================================================
-## spack_lock_default_export
-## - scratch-based export stage for CI artifact extraction of the default spack.lock
-## ========================================================================================
-FROM scratch AS spack_lock_default_export
-ARG ENV=xl
-COPY --from=builder_concretization_default \
-  /opt/spack-environment/${ENV}/spack.lock \
-  /
-
-
-## ========================================================================================
-## spack_lock_custom_export
-## - scratch-based export stage for CI artifact extraction of the custom spack.lock
-## ========================================================================================
-FROM scratch AS spack_lock_custom_export
-ARG ENV=xl
-COPY --from=builder_concretization_custom \
-  /opt/spack-environment/${ENV}/epic/spack.lock \
-  /

From fa757e0ab9f40b210cdbe29235d04bd082afd8c5 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Sun, 9 Aug 2026 17:11:05 +0000
Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 containers/eic/Dockerfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/containers/eic/Dockerfile b/containers/eic/Dockerfile
index 4dc0b23d7..1d212b74a 100644
--- a/containers/eic/Dockerfile
+++ b/containers/eic/Dockerfile
@@ -444,4 +444,3 @@ RUN echo -e "\n - eic_container: ${CI_COMMIT_SHA}" | tee -a /etc/eic_info /etc/j
 
 ## Hotfix for misbehaving OSG nodes
 RUN mkdir /hadoop /localscratch
-