Full CI Matrix #316
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Full CI Matrix | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - 'v*' | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 2 * * *' # Daily at 2 AM UTC | |
| env: | |
| PYTHON_VERSIONS: '["3.10", "3.11", "3.12"]' | |
| OS_MATRIX: '["ubuntu-latest", "windows-latest", "macos-latest"]' | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: ${{ github.repository }} | |
| permissions: | |
| contents: read | |
| packages: write | |
| security-events: write | |
| jobs: | |
| # Only run if not triggered by another workflow or contains [full-ci] in commit | |
| should-run: | |
| name: Check if should run | |
| runs-on: ubuntu-latest | |
| outputs: | |
| should_run: ${{ steps.check.outputs.should_run }} | |
| steps: | |
| - name: Check conditions | |
| id: check | |
| run: | | |
| if [[ "${{ github.event_name }}" == "schedule" ]] || \ | |
| [[ "${{ github.event_name }}" == "workflow_dispatch" ]] || \ | |
| [[ "${{ github.ref }}" == "refs/heads/main" ]] || \ | |
| [[ "${{ github.ref }}" =~ ^refs/tags/v ]] || \ | |
| [[ "${{ contains(github.event.head_commit.message, '[full-ci]') }}" == "true" ]]; then | |
| echo "should_run=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "should_run=false" >> $GITHUB_OUTPUT | |
| fi | |
| full-test-matrix: | |
| name: Full Test - ${{ matrix.os }} / Python ${{ matrix.python-version }} | |
| needs: should-run | |
| if: needs.should-run.outputs.should_run == 'true' | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 45 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macos-latest] | |
| python-version: ['3.10', '3.11', '3.12'] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v5.0.0 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.6.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: 'pip' | |
| - name: Cache dependencies | |
| uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| .pytest_cache | |
| ~/.pyrit | |
| key: ${{ runner.os }}-py${{ matrix.python-version }}-full-${{ hashFiles('**/requirements*.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-py${{ matrix.python-version }}-full- | |
| - name: Install system dependencies (Ubuntu) | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libpq-dev python3-dev | |
| - name: Install system dependencies (macOS) | |
| if: runner.os == 'macOS' | |
| run: | | |
| brew install postgresql | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel | |
| # Install all project dependencies | |
| pip install -r requirements.txt || true | |
| pip install -r violentutf/requirements.txt || true | |
| pip install -r violentutf_api/requirements.txt || true | |
| # Install test dependencies | |
| pip install pytest pytest-cov pytest-timeout pytest-xdist pytest-mock | |
| pip install black isort flake8 mypy bandit safety types-requests types-PyYAML | |
| - name: Run comprehensive tests | |
| run: | | |
| pytest tests/unit -v --cov=violentutf --cov=violentutf_api --cov-report=xml --cov-report=html --timeout=300 -n auto | |
| - name: Upload coverage reports | |
| if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11' | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: coverage-reports | |
| path: | | |
| coverage.xml | |
| htmlcov/ | |
| security-scanning: | |
| name: Comprehensive Security Scanning | |
| needs: should-run | |
| if: needs.should-run.outputs.should_run == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v5.0.0 | |
| - name: Set up Python | |
| uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.6.0 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| - name: Run Semgrep | |
| uses: returntocorp/semgrep-action@v1 | |
| with: | |
| config: >- | |
| p/security-audit | |
| p/python | |
| p/django | |
| p/flask | |
| p/jwt | |
| - name: Run comprehensive bandit scan | |
| run: | | |
| pip install bandit | |
| bandit -r . -f sarif -o bandit.sarif 2>/dev/null || true | |
| # Also generate human-readable report | |
| bandit -r . -f txt -o bandit-report.txt 2>/dev/null || true | |
| - name: Upload bandit SARIF | |
| uses: github/codeql-action/upload-sarif@v3 | |
| with: | |
| sarif_file: bandit.sarif | |
| - name: Run pip-audit | |
| run: | | |
| pip install pip-audit | |
| pip-audit --desc --format json --output pip-audit-report.json || true | |
| # Generate summary | |
| python3 -c "import json; \ | |
| data=json.load(open('pip-audit-report.json')); \ | |
| vulns=data.get('vulnerabilities',[]); \ | |
| print(f'Found {len(vulns)} vulnerable dependencies'); \ | |
| [print(f'- {v.get(\"name\",\"unknown\")}: {v.get(\"description\",\"no desc\")[:50]}...') for v in vulns[:5]]" | |
| - name: Run OWASP dependency check | |
| run: | | |
| # Download and run OWASP dependency check | |
| wget https://github.com/jeremylong/DependencyCheck/releases/download/v8.4.3/dependency-check-8.4.3-release.zip | |
| unzip dependency-check-8.4.3-release.zip | |
| ./dependency-check/bin/dependency-check.sh \ | |
| --project "ViolentUTF" \ | |
| --scan . \ | |
| --format JSON \ | |
| --out dependency-check-report.json \ | |
| --suppression dependency-check-suppression.xml || true | |
| performance-benchmarks: | |
| name: Performance Benchmarks | |
| needs: should-run | |
| if: needs.should-run.outputs.should_run == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v5.0.0 | |
| - name: Set up Python | |
| uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.6.0 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt || true | |
| pip install pytest-benchmark memory_profiler line_profiler | |
| - name: Run performance benchmarks | |
| run: | | |
| # Run benchmark tests if they exist | |
| pytest tests/benchmarks/ -v --benchmark-only --benchmark-json=benchmark-results.json || true | |
| # Memory profiling | |
| python -m memory_profiler tests/memory_tests.py > memory-profile.txt || true | |
| - name: Upload benchmark results | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: performance-results | |
| path: | | |
| benchmark-results.json | |
| memory-profile.txt | |
| docker-build-and-scan: | |
| name: Docker Build and Security Scan | |
| needs: should-run | |
| if: needs.should-run.outputs.should_run == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v5.0.0 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.11.1 | |
| - name: Log in to Container Registry | |
| if: github.event_name != 'pull_request' | |
| uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.11.1 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build Docker images | |
| run: | | |
| # Build all service images | |
| docker compose build --parallel | |
| - name: Run Trivy security scan | |
| run: | | |
| # Install trivy | |
| wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add - | |
| echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/trivy.list | |
| sudo apt-get update && sudo apt-get install trivy -y | |
| # Scan all built images | |
| docker images --format "{{.Repository}}:{{.Tag}}" | grep -v "<none>" | while read image; do | |
| echo "Scanning $image" | |
| trivy image --format sarif --output "trivy-${image//\//-}.sarif" "$image" || true | |
| done | |
| - name: Upload Trivy scan results | |
| uses: github/codeql-action/upload-sarif@v3 | |
| with: | |
| sarif_file: 'trivy-*.sarif' | |
| - name: Push images (if main branch) | |
| if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' | |
| run: | | |
| # Tag and push images | |
| docker compose push | |
| integration-suite: | |
| name: Full Integration Test Suite | |
| needs: [full-test-matrix, docker-build-and-scan] | |
| if: false # Disabled for online CI - can be enabled for local testing | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| services: | |
| postgres: | |
| image: postgres:15 | |
| env: | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: keycloak | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 5432:5432 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v5.0.0 | |
| - name: Set up Python | |
| uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.6.0 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| - name: Start all services | |
| run: | | |
| # Create necessary directories | |
| mkdir -p violentutf/app_data/violentutf/pyrit_memory | |
| # Start services with docker compose | |
| docker compose up -d | |
| # Wait for all services to be healthy | |
| timeout 300s bash -c ' | |
| until docker compose ps | grep -E "(healthy|running)" | wc -l | grep -q "6"; do | |
| echo "Waiting for services..." | |
| docker compose ps | |
| sleep 10 | |
| done | |
| ' | |
| - name: Run integration tests | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install pytest pytest-asyncio httpx | |
| # Install all dependencies | |
| for req in $(find . -name "requirements*.txt" -type f); do | |
| pip install -r "$req" || true | |
| done | |
| # Run comprehensive integration tests | |
| pytest tests/integration/ -v --tb=short || true | |
| pytest tests/e2e/ -v --tb=short || true | |
| - name: Test PyRIT functionality | |
| run: | | |
| # Test PyRIT orchestrators | |
| python -c "from pyrit.orchestrator import PromptSendingOrchestrator; print('PyRIT orchestrator import successful')" | |
| - name: Test Garak functionality | |
| run: | | |
| # Test Garak scanner | |
| garak --list-probes || true | |
| - name: Collect service logs | |
| if: always() | |
| run: | | |
| docker compose logs > docker-services.log | |
| - name: Upload service logs | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: integration-logs | |
| path: docker-services.log | |
| code-analysis: | |
| name: Advanced Code Analysis | |
| needs: should-run | |
| if: needs.should-run.outputs.should_run == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v5.0.0 | |
| - name: Set up Python | |
| uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.6.0 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| - name: Install analysis tools | |
| run: | | |
| pip install radon xenon pydocstyle vulture prospector[with_everything] | |
| - name: Cyclomatic complexity analysis | |
| run: | | |
| echo "## Cyclomatic Complexity Analysis" > analysis-report.md | |
| echo "" >> analysis-report.md | |
| radon cc . -a -j > radon-cc.json | |
| python -c 'import json; data = json.load(open("radon-cc.json")); files = len(data); print(f"Analyzed {files} files")' | |
| radon cc . -a -nc >> analysis-report.md | |
| - name: Maintainability index | |
| run: | | |
| echo "" >> analysis-report.md | |
| echo "## Maintainability Index" >> analysis-report.md | |
| echo "" >> analysis-report.md | |
| radon mi . -j > radon-mi.json | |
| radon mi . -nc >> analysis-report.md | |
| - name: Code quality metrics | |
| run: | | |
| echo "" >> analysis-report.md | |
| echo "## Code Quality Metrics" >> analysis-report.md | |
| echo "" >> analysis-report.md | |
| xenon --max-absolute B --max-modules B --max-average A . >> analysis-report.md || true | |
| - name: Documentation coverage | |
| run: | | |
| echo "" >> analysis-report.md | |
| echo "## Documentation Coverage" >> analysis-report.md | |
| echo "" >> analysis-report.md | |
| pydocstyle --count . >> analysis-report.md || true | |
| - name: Dead code detection | |
| run: | | |
| echo "" >> analysis-report.md | |
| echo "## Dead Code Detection" >> analysis-report.md | |
| echo "" >> analysis-report.md | |
| vulture . >> analysis-report.md || true | |
| - name: Upload analysis report | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: code-analysis | |
| path: | | |
| analysis-report.md | |
| radon-*.json | |
| release-preparation: | |
| name: Release Preparation | |
| needs: [full-test-matrix, security-scanning] # integration-suite disabled for CI | |
| if: | | |
| needs.should-run.outputs.should_run == 'true' && | |
| (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v5.0.0 | |
| - name: Generate release notes | |
| run: | | |
| echo "## Release Summary" > release-notes.md | |
| echo "" >> release-notes.md | |
| echo "### Commits since last release" >> release-notes.md | |
| git log --oneline --no-merges $(git describe --tags --abbrev=0)..HEAD >> release-notes.md || \ | |
| git log --oneline --no-merges -20 >> release-notes.md | |
| - name: Package release artifacts | |
| run: | | |
| # Create release package | |
| mkdir -p release-package | |
| cp -r violentutf violentutf_api scripts release-package/ | |
| cp README.md LICENSE requirements.txt docker-compose.yml release-package/ | |
| tar -czf violentutf-release.tar.gz release-package/ | |
| - name: Upload release artifacts | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: release-artifacts | |
| path: | | |
| violentutf-release.tar.gz | |
| release-notes.md | |
| final-summary: | |
| name: Full CI Summary | |
| needs: [full-test-matrix, security-scanning, performance-benchmarks, docker-build-and-scan, code-analysis] # integration-suite disabled for CI | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - name: Generate summary | |
| run: | | |
| echo "# Full CI Matrix Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "## Test Results" >> $GITHUB_STEP_SUMMARY | |
| echo "- Full Test Matrix: ${{ needs.full-test-matrix.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- Security Scanning: ${{ needs.security-scanning.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- Performance Benchmarks: ${{ needs.performance-benchmarks.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- Docker Build: ${{ needs.docker-build-and-scan.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- Integration Suite: skipped (disabled for CI)" >> $GITHUB_STEP_SUMMARY | |
| echo "- Code Analysis: ${{ needs.code-analysis.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Total Runtime**: ~20-30 minutes" >> $GITHUB_STEP_SUMMARY | |
| echo "**Test Matrix**: 3 OS × 4 Python versions = 12 combinations" >> $GITHUB_STEP_SUMMARY | |
| echo "**Triggered by**: ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY |