diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml new file mode 100644 index 00000000000000..316bc20b41011c --- /dev/null +++ b/.github/workflows/metadrive-simulation.yml @@ -0,0 +1,263 @@ +name: MetaDrive Simulation Test + +on: + pull_request: + branches: [master] + paths: + - ".github/workflows/metadrive-simulation.yml" + - "tools/sim/**" + - "selfdrive/**" + - "system/**" + - "cereal/**" + - "openpilot/**" + - "pyproject.toml" + - "uv.lock" + workflow_dispatch: + +concurrency: + group: metadrive-simulation-${{ github.ref }} + cancel-in-progress: true + +jobs: + metadrive-simulation: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + + env: + DISPLAY: ":99" + LIBGL_ALWAYS_SOFTWARE: "1" + GALLIUM_DRIVER: "llvmpipe" + LP_NUM_THREADS: "4" + MESA_GL_VERSION_OVERRIDE: "3.3" + MESA_GLSL_VERSION_OVERRIDE: "330" + SIMULATION: "1" + PASSIVE: "0" + SKIP_FW_QUERY: "1" + ONNXCPU: "1" + CI: "true" + PYTHONUNBUFFERED: "1" + + steps: + - name: Checkout openpilot + uses: actions/checkout@v4 + with: + submodules: recursive + lfs: false + + - name: Pull font LFS assets + run: | + git lfs install + git lfs pull --include="openpilot/selfdrive/assets/fonts/*,openpilot/selfdrive/assets/sounds/*,openpilot/selfdrive/modeld/models/*" --exclude="" + + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet /opt/ghc "$AGENT_TOOLSDIRECTORY" || true + df -h + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential \ + git \ + curl \ + ca-certificates \ + capnproto \ + libcapnp-dev \ + ffmpeg \ + libavcodec-dev \ + libavformat-dev \ + libavutil-dev \ + libswscale-dev \ + libegl1 \ + libegl1-mesa-dev \ + libgles2-mesa-dev \ + libgl1-mesa-dev \ + libgl1-mesa-dri \ + libglx-mesa0 \ + libglu1-mesa-dev \ + mesa-utils \ + mesa-common-dev \ + xvfb \ + x11-utils \ + pulseaudio \ + portaudio19-dev + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Start Xvfb + run: | + Xvfb :99 -screen 0 1920x1080x24 -ac +extension GLX +render -noreset > /tmp/xvfb.log 2>&1 & + echo "XVFB_PID=$!" >> "$GITHUB_ENV" + sleep 5 + + if ! xdpyinfo -display :99 >/dev/null 2>&1; then + echo "Xvfb failed to start" + cat /tmp/xvfb.log + exit 1 + fi + + - name: Verify software OpenGL + run: | + glxinfo | grep -E "OpenGL renderer|OpenGL version" || true + + - name: Start virtual audio + run: | + pulseaudio --start --exit-idle-time=-1 || true + sleep 2 + pactl load-module module-null-sink sink_name=virtual-speaker || true + pactl set-default-sink virtual-speaker || true + pactl info || true + pactl list sinks short || true + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + python -m pip install -e ".[testing]" + python -m pip install \ + "metadrive-simulator @ git+https://github.com/commaai/metadrive.git@minimal" \ + "imgui @ git+https://github.com/commaai/dependencies.git@release-imgui#subdirectory=imgui" \ + pytest-timeout + + - name: Pull MetaDrive assets + run: | + python - <<'PY' + from metadrive.pull_asset import pull_asset + pull_asset(False) + PY + + - name: Build openpilot + run: | + scons -u -j$(nproc) + + - name: Run MetaDrive simulation test + id: metadrive_test + timeout-minutes: 15 + run: | + "$pythonLocation/bin/python" -m pytest \ + "$GITHUB_WORKSPACE/openpilot/tools/sim/tests/test_metadrive_bridge.py::TestMetaDriveBridge::test_driving" \ + -o addopts="" \ + -v \ + --tb=short \ + --timeout=900 \ + --log-cli-level=INFO \ + -s + - name: Print failure diagnostics + if: failure() + run: | + echo "=== Disk ===" + df -h || true + + echo "=== Memory ===" + free -h || true + + echo "=== OOM / kernel messages ===" + dmesg -T | grep -Ei "killed process|out of memory|oom|modeld|python" | tail -100 || true + + echo "=== Running processes ===" + ps aux | sort -nrk 4 | head -40 || true + + echo "=== Find logs ===" + find /tmp "$HOME/.comma"* "$GITHUB_WORKSPACE" -type f \( \ + -name "*.log" -o \ + -name "crash*" -o \ + -name "error*" -o \ + -name "*.txt" \ + \) 2>/dev/null | sort | head -200 + + echo "=== Tail logs ===" + for f in $(find /tmp "$HOME/.comma"* "$GITHUB_WORKSPACE" -type f \( \ + -name "*.log" -o \ + -name "crash*" -o \ + -name "error*" \ + \) 2>/dev/null | sort | head -80); do + echo "" + echo "----- $f -----" + tail -200 "$f" || true + done + + - name: Print simulation logs on failure + if: failure() + run: | + echo "=== Searching for crash/error logs ===" + find /tmp "$HOME/.comma" "$GITHUB_WORKSPACE" -type f \( \ + -name "*.log" -o \ + -name "crash*" -o \ + -name "error*" -o \ + -name "*.txt" \ + \) 2>/dev/null | sort | head -200 + + echo "=== Showing likely crash/error logs ===" + for f in $(find /tmp "$HOME/.comma" "$GITHUB_WORKSPACE" -type f \( \ + -name "*.log" -o \ + -name "crash*" -o \ + -name "error*" \ + \) 2>/dev/null | sort | head -50); do + echo "" + echo "----- $f -----" + tail -200 "$f" || true + done + + - name: Collect simulation artifacts + if: always() + run: | + mkdir -p simulation_artifacts + + for location in /tmp /tmp/comma "$HOME/.comma" "$GITHUB_WORKSPACE"; do + if [ -d "$location" ]; then + find "$location" \( \ + -name "*.qlog" -o \ + -name "*.rlog" -o \ + -name "*.hevc" -o \ + -name "fcamera.hevc" -o \ + -name "dcamera.hevc" -o \ + -name "ecamera.hevc" -o \ + -name "error.log" -o \ + -name "crash.log" -o \ + -name "*.log" \ + \) -type f -exec cp --parents {} simulation_artifacts/ \; 2>/dev/null || true + fi + done + + cat > simulation_artifacts/metadata.txt << EOF + MetaDrive Simulation Test + ========================= + Run ID: ${{ github.run_id }} + Run attempt: ${{ github.run_attempt }} + Commit: ${{ github.sha }} + Test outcome: ${{ steps.metadrive_test.outcome }} + Date: $(date -Iseconds) + + System + ====== + OS: $(uname -a) + Python: $(python --version 2>&1) + OpenGL: + $(glxinfo | grep -E "OpenGL renderer|OpenGL version" || true) + + Files + ===== + EOF + + find simulation_artifacts -type f -maxdepth 10 -print >> simulation_artifacts/metadata.txt + cat simulation_artifacts/metadata.txt + + - name: Upload simulation artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: metadrive-simulation-${{ github.run_id }}-${{ github.run_attempt }} + path: simulation_artifacts/ + retention-days: 30 + if-no-files-found: warn + + - name: Cleanup + if: always() + run: | + pulseaudio --kill || true + if [ -n "${XVFB_PID:-}" ]; then + kill "$XVFB_PID" || true + fi \ No newline at end of file