diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42b2c70bd..cb6b0af5f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,6 +189,7 @@ jobs: "$BUSTER_CI_PYTHON" tools/coverage_manifest_test.py -v 2>&1 | tee "$RUNNER_TEMP/buster-ci/coverage-manifest.log" "$BUSTER_CI_PYTHON" tools/matrix_shard_test.py -v 2>&1 | tee "$RUNNER_TEMP/buster-ci/matrix-shard.log" "$BUSTER_CI_PYTHON" tools/differential_ci_policy_test.py -v 2>&1 | tee "$RUNNER_TEMP/buster-ci/differential-ci-policy.log" + "$BUSTER_CI_PYTHON" tools/native_producer_profile_test.py -v 2>&1 | tee "$RUNNER_TEMP/buster-ci/native-producer-profile.log" "$BUSTER_CI_PYTHON" tools/ci_configure_evidence_test.py -v 2>&1 | tee "$RUNNER_TEMP/buster-ci/configure-evidence-test.log" # Wrapper integration starts many real shells. It must not consume the @@ -478,7 +479,7 @@ jobs: # generate recreates build/, so the driver and logs must live outside it. driver="$RUNNER_TEMP/buster-build" clang -Isrc -Wall -Werror -Wno-unused-function -Wno-unused-variable -g build.c -o "$driver" - "$driver" generate --cc clang --config Release --linker DEFAULT + "$driver" generate --cc clang --config Release --linker DEFAULT -- -DBUSTER_DEBUG_INFO=OFF "$driver" test_mode_matrix --config Release - name: Native configuration differential matrix @@ -497,7 +498,7 @@ jobs: # The preceding mode step normally configured this tree. Recover a # missing graph too, so an earlier failure cannot skip this gate. if [[ ! -f build/CMakeCache.txt ]]; then - "$driver" generate --cc clang --config Release --linker DEFAULT + "$driver" generate --cc clang --config Release --linker DEFAULT -- -DBUSTER_DEBUG_INFO=OFF fi "$driver" build --config Release -t ide "$driver" test_differential --ide build/Release/ide --out "$RUNNER_TEMP/buster-ci/differential" --sanitize-oracle --jobs 4 diff --git a/docs/ci-suite-partition.md b/docs/ci-suite-partition.md index 8757fdfd2..26f3a62ce 100644 --- a/docs/ci-suite-partition.md +++ b/docs/ci-suite-partition.md @@ -91,13 +91,31 @@ export CFLAGS=-Wno-invalid-feature-combination driver="${RUNNER_TEMP:-/tmp}/buster-build" out="$(mktemp -d)/differential" clang -Isrc -Wall -Werror -Wno-unused-function -Wno-unused-variable -g build.c -o "$driver" -"$driver" generate --cc clang --config Release --linker DEFAULT +"$driver" generate --cc clang --config Release --linker DEFAULT -- -DBUSTER_DEBUG_INFO=OFF "$driver" test_mode_matrix --config Release "$driver" test_differential --self-test "$driver" build --config Release -t ide "$driver" test_differential --ide build/Release/ide --out "$out" --sanitize-oracle --jobs 4 ``` +The hosted native producer deliberately keeps the non-`--ci` Release policy: +`-O3`, tests enabled, one unity translation unit and object, and frame pointers +enabled. It opts out only of producer debug information. The normal mode setup +and missing-cache recovery pass the same explicit +`-DBUSTER_DEBUG_INFO=OFF` override. Local Release generation remains unchanged +and therefore keeps debug information and frame pointers for profiling. + +The profile qualification probe requires invalid input to fail nonzero with a +nonempty diagnostic; the production differential runner separately captures and +compares compiler diagnostics. The native lane keeps raw logs, but it neither +retains the producer executable or a core file nor symbolizes producer program +counters after a crash. On Linux the installed non-sanitized crash handler +reports the fatal signal, fault address and raw program counter before +re-raising; macOS has no corresponding custom handler. The hosted profile +therefore retains frame-pointer unwindability and symbols, while source-line +DWARF remains a local profiling facility rather than retained native-CI failure +evidence. + Do not generate concurrently with another build in the same tree. This local sequence is a reproduction of successful execution; the workflow additionally retains independent results after an earlier test failure. Combination and @@ -113,6 +131,9 @@ actual aggregate Bash body for all 625 combinations of four dependency groups across success, failure, cancellation, skipped and missing results. Timing tests reject absent, duplicate or unsuccessful jobs and absent native suite results. Workflow blob and runner labels continue to separate measurement cohorts. +`tools/native_producer_profile_test.py` additionally locks the same explicit +producer profile into normal setup and missing-cache recovery and rejects +`--ci` or disabled frame pointers in either path. Before claiming a latency win, obtain at least three complete successful first-attempt runs per variant with equivalent source, coverage and runner diff --git a/docs/performance-audits/2026-09-14T100625Z.md b/docs/performance-audits/2026-09-14T100625Z.md new file mode 100644 index 000000000..4048400cd --- /dev/null +++ b/docs/performance-audits/2026-09-14T100625Z.md @@ -0,0 +1,180 @@ +`2026-09-14T100625Z` (GitHub-hosted Ubuntu 26.04 and macOS 26, x86_64 and +AArch64; **Screened an explicit O3 native Release producer profile (#604)**). + +# Native Release producer profile screening for #604 + +## Result and production boundary + +Profile C, the existing native producer policy with only +`BUSTER_DEBUG_INFO=OFF`, is the supported production candidate. It keeps +`BUSTER_CI=OFF`, `-O3 -DNDEBUG`, frame pointers, tests, the unity translation +unit and single native object, the default linker, and every semantic flag. +Across all four hosted Unix architectures its median complete producer build +was 14.2% to 21.8% faster than the current profile A. It also reduced peak build +RSS on every platform. + +This screening does **not** declare a qualified winner. The retained result +records say `profile_winner=null` and +`full_native_job_timing_satisfied=false`. The compiler-throughput comparisons +were useful identical-output screens but deliberately used `--no-guard`; they +are not acceptance gates. A production head still requires guarded throughput +and at least three complete, successful exact-head native attempts, including +modes, differential, self-host/fixed-point, full job timing, evidence and +upload timing. + +The production change supported by this audit is narrowly attributable: pass +`-DBUSTER_DEBUG_INFO=OFF` to both ordinary native generation and missing-cache +recovery. It does not pass `--ci`, change O3, disable frame pointers, split the +unity source/object, alter compiler tests, cache a producer across runs, or +change local Release profiling defaults. + +## Exact cohort + +The cohort is [Actions run 34827295920][run], attempt 1, source commit +`f3b460be724a22eb51c2ff4590241e16738d156c`, tree +`0e918c44cd0808c7592937fa3e072f34f12b1415`, and qualification workflow blob +`447b06373977a09498860f04f8af4f761f02a1b9`. Each A-D profile used a fresh +directory on a warm filesystem in rotated orders `ABCD`, `BCDA`, and +`CDAB`. Profile E was a separate one-build characterization of the broader +canonical CI policy and is not mixed into the A/C comparison. + +| Runner | Toolchain / image | Job result | Artifact / ZIP SHA-256 | +|---|---|---|---| +| `ubuntu-26.04` x86_64 | Ubuntu Clang 21.1.8; `ubuntu26` `20260907.131.1` | success | `10342335719` / `7a929387aa8ea03506618dae4fd6dd910d472fcac9756737b11b2898a66ab95d` | +| `ubuntu-26.04-arm` AArch64 | Ubuntu Clang 21.1.8; `ubuntu26-arm64` `20260907.118.1` | success | `10341054816` / `595707905cbe551196b4d5499171044923f4e6854aec8293186f0d0ddb37903b` | +| `macos-26-intel` x86_64 | Apple Clang 21.0.0; `macos26` `20260824.0517.1` | success | `10342296705` / `715463bc5e2b059688157d7ea37aca30a64b49250e05d445dd587a7afd23339b` | +| `macos-26` AArch64 | Apple Clang 21.0.0; `macos26` `20260907.0351.1` | failed finalizer; complete A/C subcohort | partial `10341617416` / `6ffcfefb089de29acfeaf4a5099c38e55c4019cf91c55c08343aaab4d94d5972` | + +## Uninstrumented producer measurements + +Values are medians of three fresh-directory samples. Times are seconds; RSS +and executable size are MiB. Complete build includes the one unity compile and +one native link. + +| Platform | Metric | A: current | C: debug off, frame on | Change | +|---|---|---:|---:|---:| +| Linux x86_64 | unity compile | 152.115 | 119.399 | -21.507% | +| | link | 2.862 | 1.862 | -34.941% | +| | complete build | 154.984 | 121.263 | -21.758% | +| | generate | 0.550 | 0.552 | +0.405% | +| | mode matrix | 2.021 | 2.038 | +0.861% | +| | peak build RSS | 4664.840 | 1645.313 | -64.729% | +| | executable size | 38.696 | 25.211 | -34.849% | +| Linux AArch64 | unity compile | 117.398 | 94.690 | -19.343% | +| | link | 0.148 | 0.100 | -32.432% | +| | complete build | 117.551 | 94.796 | -19.358% | +| | generate | 0.546 | 0.477 | -12.562% | +| | mode matrix | 2.009 | 2.007 | -0.099% | +| | peak build RSS | 5355.211 | 1730.855 | -67.679% | +| | executable size | 36.337 | 23.011 | -36.675% | +| macOS x86_64 | unity compile | 162.443 | 139.097 | -14.372% | +| | link | 0.272 | 0.312 | +14.706% | +| | complete build | 162.825 | 139.663 | -14.226% | +| | generate | 4.098 | 3.026 | -26.163% | +| | mode matrix | 4.296 | 3.630 | -15.503% | +| | peak build RSS | 1698.469 | 1559.188 | -8.200% | +| | executable size | 26.240 | 25.693 | -2.086% | +| macOS AArch64 | unity compile | 92.047 | 72.687 | -21.033% | +| | link | 0.181 | 0.145 | -19.890% | +| | complete build | 92.265 | 72.840 | -21.054% | +| | generate | 2.016 | 1.688 | -16.280% | +| | mode matrix | 2.671 | 2.609 | -2.328% | +| | peak build RSS | 2768.828 | 2558.797 | -7.586% | +| | executable size | 23.252 | 23.008 | -1.053% | + +Intel macOS compile samples were noisy, but C was faster in all three pairs: +`162.443 -> 160.801`, `176.152 -> 121.011`, and +`144.925 -> 139.097` seconds. The median link increase was 0.040 seconds and +did not reverse the complete-build improvement. + +## Policy, recovery and generated-compiler checks + +After normalizing only isolated build-directory paths, every A/C compile argv +pair differed solely by A's `-g`. Both retained `-O3 -DNDEBUG`, +`-fno-omit-frame-pointer`, tests and unity enabled, CI/fuzz/sanitizer/LTO +disabled, and one `ide.c` compile edge plus one link edge. Link commands were +identical on each platform. + +Profile C's executable SHA-256 was stable across all three directories: + +| Platform | Profile C executable SHA-256 | +|---|---| +| Linux x86_64 | `e55918418d38b8b18257d9b87923745ca0eae48bb1d02da2b3c3fa3e842e1676` | +| Linux AArch64 | `fec8479f6f891f518f7976df570397a72cfb0b33f5c7a78e4d288acb239f115d` | +| macOS x86_64 | `c8846099e4294f04bc1952839a1497b4e09c53678a4058c397521ea7a6a00310` | +| macOS AArch64 | `44156c80515171026312abaf19d4ce030f54d67e09225d8e2043525b9422cf77` | + +For both A and C on every platform, initial missing-cache generation and +generation after deleting `CMakeCache.txt` succeeded. After path +normalization, normal and regenerated caches were byte-identical and C remained +exactly O3/debug-off/frame-on/tests-on/unity-on. Each profile also passed the +complete mode matrix. + +## Diagnostics and compiler throughput + +The diagnostic contract is the behavior the native lane can retain and use. +Invalid-input controls required both Clang and the produced Buster compiler to +fail nonzero with a nonempty diagnostic; A and C passed on every platform. +Linux C omits DWARF but retains `.eh_frame`, the symbol table and frame +pointers. Both Apple profiles retain symbol and unwind sections; the linked +section inventory did not expose usable line information for either profile. + +The current non-sanitized Linux crash handler prints the fatal signal, fault +address and raw program counter before re-raising; macOS has no corresponding +custom handler. Native artifacts retain logs, but not the producer executable +or a core file, and the workflow does not symbolize a producer program counter. +The production candidate therefore retains frame-pointer unwindability and +symbols while treating source-line DWARF as the unchanged local profiling +facility, not as retained hosted failure evidence. + +Each A/C throughput campaign covered 24 workload/mode combinations, two rounds, +four pairs, one warmup, fixed seed `20260907`, and +`--require-identical-output`. All twelve A/C campaigns completed with matching +outputs and reported zero confirmed regressions and zero inconclusive cases. +They used `--no-guard`, so every verdict is literally +`diagnostic (guard disabled)`; this result cannot substitute for the guarded +production-head throughput gate. + +## Failed and rejected evidence + +[Attempt 1, run 34822192017][attempt1] was rejected before measurement because +its script used an empty array under macOS Bash 3.2 and its first diagnostic +probe incorrectly required text/stream parity. It supplies no performance +sample. + +[Attempt 2, run 34823444517][attempt2] produced valid Linux cohorts but both +Apple jobs failed because the throughput command included Linux-only +`--cpu auto`. The corrected workflow removes that option and statically +rejects its reintroduction. Attempt 2 is not counted as a successful +cross-platform cohort. + +In the accepted screening run, macOS AArch64 completed all thirteen profile +rows and eight of nine throughput campaigns. One `r2-A-vs-B` +`many_functions/quality` timed candidate exited 125 before `exec`; the log +was empty and the finalizer correctly rejected 14 actual versus 13 required +rows. The preceding warmup and pair succeeded, all three A/C campaigns were +complete, and no producer profile command failed. This remains a failed job and +is not rerun or relabeled green. The missing child-side errno evidence was +routed to the existing throughput-harness reliability owner in #46. + +Profile B was rejected because removing frame pointers was unnecessary for the +build-time gain and weakens inexpensive unwindability. Profile D was rejected +because removing frame pointers alone did not provide a consistent benefit. +Profile E was rejected for this slice because `--ci` changes optimization and +other policy, preventing attribution to profiling overhead. + +## Remaining acceptance + +Before the production candidate is merge-ready: + +1. Run the repository's guarded compiler-throughput gate with identical-output + enforcement. +2. Obtain at least three successful exact-head full native attempts on all four + Unix lanes, including modes, full differential, self-host/fixed-point, + complete native-job time, peak RSS and evidence/upload durations. +3. Compare complete workflow critical path and aggregate runner seconds without + adding independent lane savings as if they were end-to-end savings. + +[run]: https://github.com/buster14a/buster/actions/runs/34827295920 +[attempt1]: https://github.com/buster14a/buster/actions/runs/34822192017 +[attempt2]: https://github.com/buster14a/buster/actions/runs/34823444517 diff --git a/tools/native_producer_profile_test.py b/tools/native_producer_profile_test.py new file mode 100644 index 000000000..b943310a2 --- /dev/null +++ b/tools/native_producer_profile_test.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Network-free regression for the native CI Release producer profile.""" + +from pathlib import Path +import textwrap +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ROOT / ".github/workflows/ci.yml" +GENERATE = ( + '"$driver" generate --cc clang --config Release --linker DEFAULT -- ' + '-DBUSTER_DEBUG_INFO=OFF' +) + + +def step_body(name): + workflow = WORKFLOW.read_text() + block = workflow.split(" - name: " + name + "\n", 1)[1] + return textwrap.dedent(block.split(" - name:", 1)[0]) + + +class NativeProducerProfileTests(unittest.TestCase): + def test_normal_generation_uses_explicit_profile_c(self): + block = step_body("Execution-mode matrix") + self.assertEqual(block.count(GENERATE), 1) + self.assertNotIn("--ci", block) + self.assertNotIn("BUSTER_FRAME_POINTERS=OFF", block) + + def test_missing_cache_recovery_uses_the_same_profile(self): + block = step_body("Native configuration differential matrix") + recovery = block.split("if [[ ! -f build/CMakeCache.txt ]]; then\n", 1)[1] + recovery = recovery.split("fi\n", 1)[0] + self.assertEqual(recovery.count(GENERATE), 1) + self.assertNotIn("--ci", recovery) + self.assertNotIn("BUSTER_FRAME_POINTERS=OFF", recovery) + + +if __name__ == "__main__": + unittest.main()