Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions .github/workflows/install-deps-macos.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/bin/bash

brew install llvm@14 gcc@13 cppcheck openldap bear

echo "$(brew --prefix llvm@14)/bin" >> "$GITHUB_PATH"
echo "$(brew --prefix gcc@13)/bin" >> "$GITHUB_PATH"

# Create g++ symlink matching Linux CI naming
GCC_BIN="$(brew --prefix gcc@13)/bin"
ln -sf "$GCC_BIN/g++-13" "$GCC_BIN/g++"
ln -sf "$GCC_BIN/gcc-13" "$GCC_BIN/gcc"

# Create intercept-build wrapper using bear.
# The LLVM intercept-build is broken on macOS ARM64 (libear.dylib arch
# mismatch with SIP). Bear provides equivalent functionality.
WRAPPER_DIR="$(pwd)/build/intercept-build-wrapper"
mkdir -p "$WRAPPER_DIR"
cat > "$WRAPPER_DIR/intercept-build" << 'EOF'
#!/bin/bash
CDB=""
CMD=()
while [[ $# -gt 0 ]]; do
case "$1" in
--cdb) CDB="$2"; shift 2 ;;
--help) echo "intercept-build wrapper using bear"; exit 0 ;;
*) CMD+=("$1"); shift ;;
esac
done
[[ -z "$CDB" ]] && CDB="compile_commands.json"
exec bear --output "$CDB" -- "${CMD[@]}"
EOF
chmod +x "$WRAPPER_DIR/intercept-build"
echo "$WRAPPER_DIR" >> "$GITHUB_PATH"
56 changes: 53 additions & 3 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,23 +46,31 @@ jobs:

tools:
name: Tools (report-converter, etc.)
runs-on: ubuntu-24.04
runs-on: ${{ matrix.os }}
continue-on-error: ${{ matrix.os == 'macos-latest' }}

strategy:
matrix:
os: [ubuntu-24.04, macos-latest]

steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Setup Bazel
if: runner.os == 'Linux'
uses: abhinavsingh/setup-bazel@v3
with:
version: 4.0.0
- name: Install common dependencies
if: runner.os == 'Linux'
run: |
sudo apt-get update -q
sudo apt-get install gcc-multilib

- name: Run build-logger tests
if: runner.os == 'Linux'
working-directory: analyzer/tools/build-logger
run: |
pip install -r requirements_py/dev/requirements.txt
Expand Down Expand Up @@ -95,24 +103,35 @@ jobs:
make test

- name: Run bazel-compile-commands tests
if: runner.os == 'Linux'
working-directory: tools/bazel
run: |
pip install -r requirements_py/dev/requirements.txt
make test

analyzer:
name: Analyzer
runs-on: ubuntu-24.04
runs-on: ${{ matrix.os }}
continue-on-error: ${{ matrix.os == 'macos-latest' }}

strategy:
matrix:
os: [ubuntu-24.04, macos-latest]

steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v4
with:
python-version: '3.10'

- name: Install dependencies
- name: Install dependencies (Linux)
if: runner.os == 'Linux'
run: sh .github/workflows/install-deps.sh

- name: Install dependencies (macOS)
if: runner.os == 'macOS'
run: sh .github/workflows/install-deps-macos.sh

- name: Build the package
run: |
make pip_dev_deps
Expand Down Expand Up @@ -181,6 +200,37 @@ jobs:
working-directory: web
run: make test_unit_cov

web-macos:
name: Web (macOS)
runs-on: macos-latest
continue-on-error: true

steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v4
with:
python-version: '3.10'

- name: Install dependencies
run: sh .github/workflows/install-deps-macos.sh

- name: Run tests
run: |
make pip_dev_deps
pip3 install -r web/requirements_py/auth/requirements.txt
BUILD_UI_DIST=NO make package

# Run full functional test suite.
cd web
make test_matrix_sqlite
env:
CC_TEST_API_WORKERS: "1"
CC_TEST_TASK_WORKERS: "1"

- name: Run unit tests coverage
working-directory: web
run: make test_unit_cov

gui:
name: GUI
runs-on: ubuntu-24.04
Expand Down
12 changes: 9 additions & 3 deletions analyzer/codechecker_analyzer/analysis_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@


import glob
import logging
import os
import shlex
import shutil
Expand Down Expand Up @@ -118,10 +119,14 @@ def worker_result_handler(results, metadata_tool, output_path):
PROGRESS_ACTIONS = None


def init_worker(checked_num, action_num):
def init_worker(checked_num, action_num, log_level=None):
global PROGRESS_CHECKED_NUM, PROGRESS_ACTIONS
PROGRESS_CHECKED_NUM = checked_num
PROGRESS_ACTIONS = action_num
# With spawn, workers need explicit logger setup (no fork inheritance).
if log_level:
from codechecker_common.logger import setup_logger
setup_logger(log_level)


def save_output(base_file_name, out, err):
Expand Down Expand Up @@ -701,9 +706,10 @@ def signal_handler(signum, _):
# Start checking parallel.
checked_var = multiprocess.Value('i', 1)
actions_num = multiprocess.Value('i', len(actions))
pool = multiprocess.Pool(jobs,
log_level = logging.getLevelName(LOG.getEffectiveLevel())
pool = multiprocess.Pool(jobs, # pylint: disable=not-callable
initializer=init_worker,
initargs=(checked_var, actions_num))
initargs=(checked_var, actions_num, log_level))
signal.signal(signal.SIGINT, signal_handler)

# If the analysis has failed, we help debugging.
Expand Down
18 changes: 14 additions & 4 deletions analyzer/codechecker_analyzer/buildlog/log_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@

from codechecker_analyzer.analyzers.clangsa.analyzer import ClangSA

from codechecker_common.compatibility import multiprocessing
import multiprocess
from multiprocess.managers import SyncManager

from codechecker_common.logger import get_logger
from codechecker_common.util import load_json

Expand Down Expand Up @@ -1241,6 +1243,11 @@ class CompileActionUniqueingType(Enum):
# recognizing symlink and remove duplication


def _init_log_parser_worker(compiler_info_dict):
"""Set shared manager dict in spawn workers."""
ImplicitCompilerInfo.compiler_info = compiler_info_dict


def _process_entry_worker(args):
"""
Worker function for processing compilation database entries in parallel.
Expand Down Expand Up @@ -1339,7 +1346,7 @@ def parse_unique_log(compilation_database,
__contains_no_intrinsic_headers.cache_clear()

if jobs is None:
jobs = multiprocessing.cpu_count()
jobs = multiprocess.cpu_count()

# Prepare entries for parallel processing
entries = extend_compilation_database_entries(compilation_database)
Expand All @@ -1352,12 +1359,15 @@ def parse_unique_log(compilation_database,
# Here we overwrite ImplicitCompilerInfo.compiker_info with a dict type
# that can be used in multiprocess environment, since the next section
# is executed in a process pool.
manager = multiprocessing.SyncManager()
manager = SyncManager()
manager.start()
ImplicitCompilerInfo.compiler_info = manager.dict()

# Process entries in parallel using imap_unordered with chunk size 1024
with multiprocessing.Pool(jobs) as pool:
with multiprocess.Pool( # pylint: disable=not-callable
jobs,
initializer=_init_log_parser_worker,
initargs=(ImplicitCompilerInfo.compiler_info,)) as pool:
# Convert generator to list for map function
worker_args_list = list(worker_args)
results = pool.map(_process_entry_worker, worker_args_list)
Expand Down
2 changes: 1 addition & 1 deletion analyzer/codechecker_analyzer/cli/analyze.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from functools import partial

from tu_collector import tu_collector
from multiprocess import cpu_count # type: ignore

from codechecker_analyzer import analyzer, analyzer_context, \
compilation_database
Expand All @@ -31,7 +32,6 @@
from codechecker_analyzer.buildlog import log_parser

from codechecker_common import arg, logger, cmd_config, review_status_handler
from codechecker_common.compatibility.multiprocessing import cpu_count
from codechecker_common.skiplist_handler import SkipListHandler, \
SkipListHandlers
from codechecker_common.util import load_json
Expand Down
3 changes: 2 additions & 1 deletion analyzer/codechecker_analyzer/cli/check.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
import sys
import tempfile

from multiprocess import cpu_count # type: ignore

from codechecker_analyzer.analyzers import analyzer_types
from codechecker_analyzer.arg import \
OrderedCheckersAction, OrderedConfigAction, \
Expand All @@ -33,7 +35,6 @@
EPILOG_ENV_VAR as parse_epilog_env_var

from codechecker_common import arg, cmd_config, logger
from codechecker_common.compatibility.multiprocessing import cpu_count
from codechecker_common.source_code_comment_handler import \
REVIEW_STATUS_VALUES

Expand Down
11 changes: 8 additions & 3 deletions analyzer/codechecker_analyzer/pre_analysis_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
Run pre analysis, collect statistics or CTU data.
"""

import logging
import os
import shlex
import shutil
Expand Down Expand Up @@ -76,10 +77,13 @@ def collect_statistics(action, source, clangsa_config, statistics_data):
PROGRESS_ACTIONS = None


def init_worker(checked_num, action_num):
def init_worker(checked_num, action_num, log_level=None):
global PROGRESS_CHECKED_NUM, PROGRESS_ACTIONS
PROGRESS_CHECKED_NUM = checked_num
PROGRESS_ACTIONS = action_num
if log_level:
from codechecker_common.logger import setup_logger
setup_logger(log_level)


def pre_analyze(params):
Expand Down Expand Up @@ -167,9 +171,10 @@ def signal_handler(signum, _):
processed_var = multiprocess.Value('i', 0)
actions_num = multiprocess.Value('i', len(actions))

pool = multiprocess.Pool(jobs,
log_level = logging.getLevelName(LOG.getEffectiveLevel())
pool = multiprocess.Pool(jobs, # pylint: disable=not-callable
initializer=init_worker,
initargs=(processed_var, actions_num))
initargs=(processed_var, actions_num, log_level))

if statistics_data:
# Statistics collection is enabled setup temporary
Expand Down
8 changes: 6 additions & 2 deletions analyzer/tests/functional/analyze/test_analyze.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

import glob
import json
import sys
import os
import pathlib
import re
Expand Down Expand Up @@ -1283,8 +1284,11 @@ def test_disable_all_checkers(self):
errors="ignore")
out, _ = process.communicate()

# Checkers of all 3 analyzers are disabled.
self.assertEqual(out.count("No checkers enabled for"), 5)
# Checkers of all available analyzers are disabled.
# Linux has 5 (clangsa, clang-tidy, cppcheck, gcc, infer),
# macOS has 4 (no infer binary available).
expected_count = 4 if sys.platform == "darwin" else 5
self.assertEqual(out.count("No checkers enabled for"), expected_count)

def test_analyzer_and_checker_config(self):
"""Test analyzer configuration through command line flags."""
Expand Down
Loading
Loading