Repository navigation
[gpu][alpaka] Implementation of inference on heterogeneous architectures using alpaka #26
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| pull_request: | |
| issue_comment: | |
| types: [created] | |
| workflow_dispatch: | |
| inputs: | |
| warmup: | |
| description: "Warmup iterations" | |
| default: "5" | |
| iterations: | |
| description: "Timed iterations" | |
| default: "100" | |
| concurrency: | |
| group: benchmark-${{ github.event.issue.number || github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| issues: write | |
| env: | |
| CUDA_ARCH: "native" | |
| BUILD_TYPE: Release | |
| BENCH_WARMUP: ${{ github.event.inputs.warmup || '5' }} | |
| BENCH_ITERS: ${{ github.event.inputs.iterations || '100' }} | |
| DEPS_CACHE: /tmp/sofie-cmake-deps | |
| jobs: | |
| benchmark: | |
| name: Benchmark Comparison (L40S) | |
| runs-on: ml4ep-l40s | |
| container: registry.cern.ch/ngt/lxplus-like:9 | |
| timeout-minutes: 120 | |
| if: | | |
| github.event_name == 'workflow_dispatch' || | |
| github.event_name == 'pull_request' || | |
| ( | |
| github.event_name == 'issue_comment' && | |
| github.event.issue.pull_request != null && | |
| contains(fromJSON('["sanjibansg","lmoneta"]'), github.event.comment.user.login) && | |
| startsWith(github.event.comment.body, '/runbenchmark') | |
| ) | |
| environment: ${{ github.event_name == 'pull_request' && 'ci-benchmark-approval' || 'ci-open' }} | |
| steps: | |
| - name: React to trigger comment | |
| if: github.event_name == 'issue_comment' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| await github.rest.reactions.createForIssueComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: context.payload.comment.id, | |
| content: 'eyes' | |
| }); | |
| - name: GPU check | |
| run: nvidia-smi | |
| - name: Resolve PR info | |
| id: pr | |
| if: github.event_name == 'issue_comment' || github.event_name == 'pull_request' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const prNumber = context.eventName === 'issue_comment' | |
| ? context.payload.issue.number | |
| : context.payload.pull_request.number; | |
| const pr = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: prNumber | |
| }); | |
| core.setOutput('number', pr.data.number); | |
| core.setOutput('head_ref', `refs/pull/${pr.data.number}/head`); | |
| core.setOutput('base_ref', pr.data.base.ref); | |
| - name: Checkout PR branch | |
| uses: actions/checkout@v4 | |
| with: | |
| path: sofie-pr | |
| ref: ${{ (github.event_name == 'issue_comment' || github.event_name == 'pull_request') && steps.pr.outputs.head_ref || github.ref }} | |
| - name: Checkout base branch | |
| if: github.event_name == 'issue_comment' || github.event_name == 'pull_request' | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ steps.pr.outputs.base_ref }} | |
| path: sofie-main | |
| - name: Cache FetchContent dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.DEPS_CACHE }} | |
| key: cmake-deps-bench-${{ hashFiles('sofie-pr/benchmark/CMakeLists.txt') }} | |
| restore-keys: cmake-deps-bench- | |
| - name: Configure PR build | |
| run: | | |
| cmake -B sofie-pr/build -S sofie-pr \ | |
| -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ | |
| -DSOFIE_WITH_ROOT=OFF \ | |
| -DSOFIE_BENCHMARK=ON \ | |
| -DSOFIE_BENCHMARK_BACKEND=CUDA \ | |
| "-DSOFIE_BENCHMARK_CUDA_ARCH=${{ env.CUDA_ARCH }}" \ | |
| "-DCMAKE_CUDA_ARCHITECTURES=${{ env.CUDA_ARCH }}" \ | |
| "-DFETCHCONTENT_BASE_DIR=${{ env.DEPS_CACHE }}" | |
| - name: Build PR benchmark | |
| run: cmake --build sofie-pr/build --target sofie_benchmark -j10 | |
| - name: Run PR benchmark | |
| working-directory: sofie-pr/build/benchmark | |
| run: | | |
| ./sofie_benchmark \ | |
| -w ${{ env.BENCH_WARMUP }} \ | |
| -n ${{ env.BENCH_ITERS }} \ | |
| | tee benchmark_pr.txt | |
| - name: Check whether base branch supports the benchmark harness | |
| id: base_support | |
| if: github.event_name == 'issue_comment' || github.event_name == 'pull_request' | |
| run: | | |
| if grep -q "option(SOFIE_BENCHMARK " sofie-main/CMakeLists.txt 2>/dev/null; then | |
| echo "supported=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "supported=false" >> "$GITHUB_OUTPUT" | |
| echo "::warning::${{ steps.pr.outputs.base_ref }} does not define the SOFIE_BENCHMARK option yet — skipping the base-branch comparison." | |
| fi | |
| - name: Configure base build | |
| if: (github.event_name == 'issue_comment' || github.event_name == 'pull_request') && steps.base_support.outputs.supported == 'true' | |
| run: | | |
| cmake -B sofie-main/build -S sofie-main \ | |
| -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ | |
| -DSOFIE_WITH_ROOT=OFF \ | |
| -DSOFIE_BENCHMARK=ON \ | |
| -DSOFIE_BENCHMARK_BACKEND=CUDA \ | |
| "-DSOFIE_BENCHMARK_CUDA_ARCH=${{ env.CUDA_ARCH }}" \ | |
| "-DCMAKE_CUDA_ARCHITECTURES=${{ env.CUDA_ARCH }}" \ | |
| "-DFETCHCONTENT_BASE_DIR=${{ env.DEPS_CACHE }}" | |
| - name: Build base benchmark | |
| if: (github.event_name == 'issue_comment' || github.event_name == 'pull_request') && steps.base_support.outputs.supported == 'true' | |
| run: cmake --build sofie-main/build --target sofie_benchmark -j10 | |
| - name: Run base benchmark | |
| if: (github.event_name == 'issue_comment' || github.event_name == 'pull_request') && steps.base_support.outputs.supported == 'true' | |
| working-directory: sofie-main/build/benchmark | |
| run: | | |
| ./sofie_benchmark \ | |
| -w ${{ env.BENCH_WARMUP }} \ | |
| -n ${{ env.BENCH_ITERS }} \ | |
| | tee benchmark_main.txt | |
| - name: Build comparison summary | |
| id: summary | |
| if: always() | |
| run: | | |
| { | |
| echo "summary<<BENCH_EOF" | |
| echo "### Benchmark results" | |
| echo "" | |
| if [ "${{ github.event_name == 'issue_comment' || github.event_name == 'pull_request' }}" = "true" ] && [ "${{ steps.base_support.outputs.supported }}" = "true" ]; then | |
| echo "Comparing against \`${{ steps.pr.outputs.base_ref }}\`" | |
| elif [ "${{ github.event_name == 'issue_comment' || github.event_name == 'pull_request' }}" = "true" ]; then | |
| echo "_\`${{ steps.pr.outputs.base_ref }}\` does not yet support the SOFIE_BENCHMARK harness — showing PR results only._" | |
| fi | |
| echo "" | |
| echo '```' | |
| echo "── PR ──────────────────────────────────────────────────────────────" | |
| cat sofie-pr/build/benchmark/benchmark_pr.txt 2>/dev/null || echo "(no results)" | |
| if [ -f sofie-main/build/benchmark/benchmark_main.txt ]; then | |
| echo "" | |
| echo "── ${{ steps.pr.outputs.base_ref }} ──────────────────────────────────────────────" | |
| cat sofie-main/build/benchmark/benchmark_main.txt | |
| fi | |
| echo '```' | |
| echo "BENCH_EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Post results as PR comment | |
| if: always() && (github.event_name == 'issue_comment' || github.event_name == 'pull_request') | |
| uses: marocchino/sticky-pull-request-comment@v2 | |
| with: | |
| number: ${{ steps.pr.outputs.number }} | |
| header: benchmark | |
| message: ${{ steps.summary.outputs.summary }} | |
| - name: Write job summary | |
| if: always() | |
| run: echo "${{ steps.summary.outputs.summary }}" >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload benchmark results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results-${{ github.run_id }} | |
| path: | | |
| sofie-pr/build/benchmark/benchmark_pr.txt | |
| sofie-main/build/benchmark/benchmark_main.txt | |
| if-no-files-found: ignore |