From 3d0d242fb1b4b8c3b8a674701de6419e082602d7 Mon Sep 17 00:00:00 2001 From: kylelutze <4561737+kylelutze@users.noreply.github.com> Date: Sun, 6 Sep 2026 19:09:04 -0400 Subject: [PATCH 1/2] Cache compilation in CI with ccache CI rebuilds every object from scratch on each of the four builds a run performs, across two container images. Wrapping the compilers in ccache lets the compiles hit a content-addressed store instead. Only ccache's store is cached. The build/ tree and .sconsign.dblite stay out of it: those are incremental state whose correctness depends on scons's dependency scanner having recorded every input, and it does not record all of them -- DET_INIT is read straight from the environment, and the Configure results in config.h and options_cache.py are not tracked either. Persisting that across runs is how a stale object survives a source change. ccache re-hashes the source, every header it includes by content, the compiler binary and the full command line on every compile, so scons still rebuilds its dependency graph from a fresh checkout each run and only the individual compiles are reused. CCACHE_SLOPPINESS is left unset for the same reason: include_file_mtime and include_file_ctime would trade content hashing for timestamps, and time_macros would let ccache serve a stale __DATE__/__TIME__ banner for GlobalContainerArgs.cpp. CCACHE_COMPILERCHECK=content covers a g++ point release landing inside a base image. Two tripwires: the per-run statistics are printed, so a change to a widely included header that still shows near-total direct hits is visible, and the existing harnesses already run against the produced binaries. Only master writes the cache, and it writes whenever the compiles ran, including runs a later test step failed -- the entries are content-addressed, so a failing test says nothing about whether the objects are the right objects. Enabled by CCACHE=1 in the environment rather than a scons option, so it cannot stick in options_cache.py. Windows is not covered. --- .github/workflows/build.yml | 42 ++++++++++++++++++++++++++++++++++++- CLAUDE.md | 9 ++++++++ SConstruct | 7 +++++++ scons/ccache.py | 35 +++++++++++++++++++++++++++++++ test/SConstruct | 9 ++++++++ 5 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 scons/ccache.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aea425a85..ff430b7d1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,12 +4,24 @@ on: push: branches: [master] pull_request: + workflow_dispatch: jobs: linux: name: linux (${{ matrix.image }}) runs-on: ubuntu-latest container: ${{ matrix.image }} + env: + CCACHE: 1 + CCACHE_DIR: /ccache + # Hash the compiler binary instead of trusting its mtime, so a g++ point + # release landing inside the base image cannot reuse the old one's + # entries. CCACHE_SLOPPINESS is deliberately left unset: include_file_mtime + # and include_file_ctime would make ccache trust timestamps over content, + # and time_macros would let it cache the __DATE__/__TIME__ build banner in + # GlobalContainerArgs.cpp. + CCACHE_COMPILERCHECK: content + CCACHE_MAXSIZE: 1G strategy: fail-fast: false matrix: @@ -25,7 +37,7 @@ jobs: run: | apt-get update -qq apt-get install -y --no-install-recommends \ - ca-certificates git g++ python3 scons pkg-config \ + ca-certificates git g++ python3 scons pkg-config ccache \ libsdl2-dev libsdl2-image-dev libsdl2-net-dev libsdl2-ttf-dev \ libvorbis-dev libogg-dev libspeex-dev \ libboost-date-time-dev libboost-thread-dev libboost-system-dev \ @@ -35,6 +47,16 @@ jobs: - uses: actions/checkout@v4 + - name: Restore the compiler cache + uses: actions/cache/restore@v4 + with: + path: /ccache + key: ccache-${{ matrix.image }}-${{ github.run_id }} + restore-keys: ccache-${{ matrix.image }}- + + - name: Reset the cache statistics + run: ccache -z + - name: Build glob2 run: scons -j$(nproc) release=1 @@ -96,6 +118,24 @@ jobs: scons -j$(nproc) release=1 server=0 speed-tests xvfb-run -a python3 test/run-game-speed-tests.py + # A run that touched a widely included header and still shows near-total + # direct hits means the cache is not seeing the change. + - name: Report the cache statistics + if: always() + run: ccache -s -v + + # Only master writes. PR branches cannot read each other's caches anyway, + # and letting every PR save would churn the repository's cache quota and + # evict the entries PRs restore from. Saved even when a step failed: the + # entries are content-addressed, so objects compiled during a run whose + # tests failed are still the right objects. + - name: Save the compiler cache + if: ${{ !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/master' }} + uses: actions/cache/save@v4 + with: + path: /ccache + key: ccache-${{ matrix.image }}-${{ github.run_id }} + windows: name: windows (mingw-w64) runs-on: windows-latest diff --git a/CLAUDE.md b/CLAUDE.md index 177f52329..cfb5510c1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -23,6 +23,15 @@ Options are cached in `options_cache.py`, so `release=1` and `server=1` stick un **`DET_INIT` — hunting uninitialized-read nondeterminism.** Valgrind/MSan don't work on modern macOS. Instead build twice, `DET_INIT=zero scons` and `DET_INIT=pattern scons`, and hammer one seed serially per build (add `MallocPreScribble=1 MallocScribble=1` to scribble the heap). If each build is internally stable but the two disagree, an uninitialized stack read is confirmed; if a scribbled build is still unstable run-to-run, the cause is not uninitialized memory. The env var is invisible to scons's dependency tracking, so rebuild affected objects when toggling. +**`CCACHE=1`** wraps the C/C++ compilers in ccache (`scons/ccache.py`, used by +both this SConstruct and `test/SConstruct`). Opt-in via the environment rather +than a scons option so it never sticks in `options_cache.py`. Leave it unset +when regenerating `compile_commands.json`, which would otherwise record the +wrapped command line. Do not set `CCACHE_SLOPPINESS`: `include_file_mtime` and +`include_file_ctime` make ccache trust timestamps over content, and +`time_macros` lets it cache the `__DATE__`/`__TIME__` banner in +`GlobalContainerArgs.cpp`. + **Dependencies:** SDL2, SDL2_net, SDL2_ttf, SDL2_image, libvorbis, libogg, speex, OpenGL, GLU, libepoxy, Boost date_time, zlib, fribidi, pcre. Optional: portaudio. See `vcpkg.json`. ## Tests diff --git a/SConstruct b/SConstruct index 262eb1e36..da3d498da 100644 --- a/SConstruct +++ b/SConstruct @@ -4,6 +4,7 @@ import os import glob sys.path.append( os.path.abspath("scons") ) import bundle +import ccache import dmg import nsis @@ -271,6 +272,12 @@ def main(): env['CXX'] = 'x86_64-w64-mingw32-g++' env['AR'] = 'x86_64-w64-mingw32-ar' env['RANLIB'] = 'x86_64-w64-mingw32-ranlib' + + # Compiler cache. Done here so it wraps whichever compiler the mingw + # branches above settled on, and before configure() runs its CheckCC / + # CheckCXX probes against the same command line. + if ccache.enabled(): + ccache.enable(env) diff --git a/scons/ccache.py b/scons/ccache.py new file mode 100644 index 000000000..9548ed60c --- /dev/null +++ b/scons/ccache.py @@ -0,0 +1,35 @@ +"""Optional ccache wrapper for the C/C++ compilers. + +Enabled by setting CCACHE=1 in the environment. An env var rather than a scons +option so it never lands in options_cache.py and silently sticks across later +builds. Note that compile_commands.json records the wrapped command line, so +leave CCACHE unset when regenerating it for tools/remove-unused-includes.py. +""" + +import os +import shutil + +# ccache settings forwarded into the environment scons scrubs for build +# commands. Deliberately absent: CCACHE_SLOPPINESS. Marking include_file_mtime +# or include_file_ctime sloppy makes ccache trust timestamps over content, and +# time_macros would let it cache the __DATE__/__TIME__ build banner in +# GlobalContainerArgs.cpp. Both trade correctness for hit rate. +_FORWARDED = ('CCACHE_DIR', 'CCACHE_COMPILERCHECK', 'CCACHE_MAXSIZE', + 'CCACHE_DISABLE', 'CCACHE_LOGFILE', 'HOME') + + +def enabled(): + return bool(os.environ.get('CCACHE')) + + +def enable(env): + """Prefix env's compilers with ccache. Call once CC and CXX are final.""" + binary = shutil.which('ccache') + if not binary: + raise SystemExit("CCACHE is set but ccache was not found on PATH") + for var in ('CC', 'CXX'): + if env.get(var) and not env[var].startswith(binary): + env[var] = binary + ' ' + env[var] + for var in _FORWARDED: + if var in os.environ: + env['ENV'][var] = os.environ[var] diff --git a/test/SConstruct b/test/SConstruct index 444b876e8..dbfd21094 100644 --- a/test/SConstruct +++ b/test/SConstruct @@ -1,7 +1,16 @@ +import os import sys +sys.path.append( os.path.abspath("../scons") ) +import ccache + env = Environment() +# Compiler cache, shared with the top-level build. Set before the Clone()s +# below so every harness environment inherits it. +if ccache.enabled(): + ccache.enable(env) + # Shared include paths so tests can compile against game headers. # echo/Echo.h transitively pulls in Map.h / Player.h / Team.h / etc., # which need libgag and SDL2 headers visible at parse time even when the From 2d1779fd7878d2aecb8a59656bb559a33b622d72 Mon Sep 17 00:00:00 2001 From: Bradley Arsenault Date: Mon, 7 Sep 2026 22:00:37 -0400 Subject: [PATCH 2/2] Handle cache reruns and manual warming; test compiler integration --- .github/workflows/build.yml | 12 ++++-- CLAUDE.md | 2 +- SConstruct | 5 +-- scons/ccache.py | 18 +++++++-- test/test-ccache.py | 75 +++++++++++++++++++++++++++++++++++++ 5 files changed, 100 insertions(+), 12 deletions(-) create mode 100644 test/test-ccache.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 12e8024cb..f334da3a0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,14 +47,18 @@ jobs: - uses: actions/checkout@v4 + - name: Test compiler cache integration + run: python3 test/test-ccache.py + - name: Restore the compiler cache uses: actions/cache/restore@v4 with: path: /ccache - key: ccache-${{ matrix.image }}-${{ github.run_id }} + key: ccache-${{ matrix.image }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: ccache-${{ matrix.image }}- - name: Reset the cache statistics + id: cache_ready run: ccache -z - name: Build glob2 @@ -126,7 +130,7 @@ jobs: # A run that touched a widely included header and still shows near-total # direct hits means the cache is not seeing the change. - name: Report the cache statistics - if: always() + if: ${{ !cancelled() && steps.cache_ready.outcome == 'success' }} run: ccache -s -v # Only master writes. PR branches cannot read each other's caches anyway, @@ -135,11 +139,11 @@ jobs: # entries are content-addressed, so objects compiled during a run whose # tests failed are still the right objects. - name: Save the compiler cache - if: ${{ !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/master' }} + if: ${{ !cancelled() && steps.cache_ready.outcome == 'success' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/master' }} uses: actions/cache/save@v4 with: path: /ccache - key: ccache-${{ matrix.image }}-${{ github.run_id }} + key: ccache-${{ matrix.image }}-${{ github.run_id }}-${{ github.run_attempt }} windows: name: windows (mingw-w64) diff --git a/CLAUDE.md b/CLAUDE.md index cfb5510c1..7d529b872 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -23,7 +23,7 @@ Options are cached in `options_cache.py`, so `release=1` and `server=1` stick un **`DET_INIT` — hunting uninitialized-read nondeterminism.** Valgrind/MSan don't work on modern macOS. Instead build twice, `DET_INIT=zero scons` and `DET_INIT=pattern scons`, and hammer one seed serially per build (add `MallocPreScribble=1 MallocScribble=1` to scribble the heap). If each build is internally stable but the two disagree, an uninitialized stack read is confirmed; if a scribbled build is still unstable run-to-run, the cause is not uninitialized memory. The env var is invisible to scons's dependency tracking, so rebuild affected objects when toggling. -**`CCACHE=1`** wraps the C/C++ compilers in ccache (`scons/ccache.py`, used by +**`CCACHE=1`** wraps C/C++ compilation commands in ccache (`scons/ccache.py`, used by both this SConstruct and `test/SConstruct`). Opt-in via the environment rather than a scons option so it never sticks in `options_cache.py`. Leave it unset when regenerating `compile_commands.json`, which would otherwise record the diff --git a/SConstruct b/SConstruct index da3d498da..88bc62cab 100644 --- a/SConstruct +++ b/SConstruct @@ -273,9 +273,8 @@ def main(): env['AR'] = 'x86_64-w64-mingw32-ar' env['RANLIB'] = 'x86_64-w64-mingw32-ranlib' - # Compiler cache. Done here so it wraps whichever compiler the mingw - # branches above settled on, and before configure() runs its CheckCC / - # CheckCXX probes against the same command line. + # Cache compilation after compiler selection and before configure probes. + # Link commands continue to use the original compiler driver. if ccache.enabled(): ccache.enable(env) diff --git a/scons/ccache.py b/scons/ccache.py index 9548ed60c..ffb67093d 100644 --- a/scons/ccache.py +++ b/scons/ccache.py @@ -8,6 +8,7 @@ import os import shutil +import shlex # ccache settings forwarded into the environment scons scrubs for build # commands. Deliberately absent: CCACHE_SLOPPINESS. Marking include_file_mtime @@ -23,13 +24,22 @@ def enabled(): def enable(env): - """Prefix env's compilers with ccache. Call once CC and CXX are final.""" + """Prefix compilation commands with ccache once CC and CXX are final.""" binary = shutil.which('ccache') if not binary: raise SystemExit("CCACHE is set but ccache was not found on PATH") - for var in ('CC', 'CXX'): - if env.get(var) and not env[var].startswith(binary): - env[var] = binary + ' ' + env[var] + # Wrap compilation commands, not CC/CXX: SMARTLINK and configure link + # probes must keep the original compiler driver without a ccache process. + # Respect callers who already supplied a ccache-prefixed compiler. + for compiler, commands in (('CC', ('CCCOM', 'SHCCCOM')), + ('CXX', ('CXXCOM', 'SHCXXCOM'))): + words = shlex.split(str(env.get(compiler, ''))) + if words and os.path.basename(words[0]) == 'ccache': + continue + for command in commands: + value = env.get(command) + if value and not str(value).startswith(binary + ' '): + env[command] = binary + ' ' + str(value) for var in _FORWARDED: if var in os.environ: env['ENV'][var] = os.environ[var] diff --git a/test/test-ccache.py b/test/test-ccache.py new file mode 100644 index 000000000..af12e6187 --- /dev/null +++ b/test/test-ccache.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Exercise the SCons wrapper with a real cold cache, warm cache and header edit.""" +import importlib.util +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +from unittest.mock import patch + +root = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location('glob2_ccache', root / 'scons/ccache.py') +wrapper = importlib.util.module_from_spec(spec) +spec.loader.exec_module(wrapper) + +# Respect an explicitly wrapped compiler and never double-wrap commands. +env = {'CC': 'cc', 'CXX': 'ccache c++', 'CCCOM': '$CC -c $SOURCE', + 'CXXCOM': '$CXX -c $SOURCE', 'LINKCOM': '$CXX -o $TARGET $SOURCES', 'ENV': {}} +with patch.object(wrapper.shutil, 'which', return_value='/cache/ccache'): + wrapper.enable(env) + wrapper.enable(env) +assert env['CCCOM'] == '/cache/ccache $CC -c $SOURCE' +assert env['CXXCOM'] == '$CXX -c $SOURCE' +assert env['LINKCOM'] == '$CXX -o $TARGET $SOURCES' +with patch.object(wrapper.shutil, 'which', return_value=None): + try: + wrapper.enable(env) + except SystemExit as error: + assert 'not found' in str(error) + else: + raise AssertionError('Missing ccache did not fail') + +ccache = shutil.which('ccache') +assert ccache, 'Install ccache to run this integration test' +with tempfile.TemporaryDirectory(prefix='glob2-ccache-') as directory: + work = Path(directory) + child_env = dict(os.environ, CCACHE='1', CCACHE_DIR=str(work / 'cache'), + CCACHE_COMPILERCHECK='content') + # Isolate cache configuration from the developer's user configuration. + config = work / 'ccache.conf' + config.write_text('sloppiness =\n') + child_env['CCACHE_CONFIGPATH'] = str(config) + (work / 'SConstruct').write_text( + 'import os, sys\n' + f'sys.path.insert(0, {str(root / "scons")!r})\n' + 'import ccache\n' + 'env = Environment(ENV=dict(os.environ))\n' + 'ccache.enable(env)\n' + 'env.Program("probe", "probe.cpp")\n') + (work / 'probe.cpp').write_text('#include "value.h"\nint main() { return VALUE; }\n') + header = work / 'value.h' + header.write_text('#define VALUE 7\n') + + def build(expected): + result = subprocess.run(['scons', '-Q'], cwd=work, env=child_env, + text=True, capture_output=True, check=True) + lines = result.stdout.splitlines() + compile_lines = [line for line in lines if ' -c ' in line] + link_lines = [line for line in lines if ' -o probe ' in line] + assert compile_lines and all('ccache' in line for line in compile_lines), lines + assert link_lines and all('ccache' not in line for line in link_lines), lines + assert subprocess.run([str(work / 'probe')]).returncode == expected + stats = subprocess.check_output([ccache, '--print-stats'], env=child_env, text=True) + return {key: int(value) for key, value in (line.split() for line in stats.splitlines())} + + cold = build(7) + (work / 'probe.o').unlink() + (work / 'probe').unlink() + warm = build(7) + hits = lambda stats: stats.get('direct_cache_hit', 0) + stats.get('preprocessed_cache_hit', 0) + assert hits(warm) > hits(cold), (cold, warm) + header.write_text('#define VALUE 9\n') + edited = build(9) + assert edited['cache_miss'] > warm['cache_miss'], (warm, edited) +print('ccache: cold/warm builds, header invalidation, unwrapped linking and wrapper guards PASS')