diff --git a/.github/workflows/codechecker_master_analysis.yml b/.github/workflows/codechecker_master_analysis.yml index 531068a23d..c5ac12b33d 100644 --- a/.github/workflows/codechecker_master_analysis.yml +++ b/.github/workflows/codechecker_master_analysis.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.10' - name: Install dependencies run: | pip install $(grep -iE "pylint|pycodestyle" analyzer/requirements_py/dev/requirements.txt) diff --git a/.github/workflows/codechecker_pr_analysis.yml b/.github/workflows/codechecker_pr_analysis.yml index 53285119b1..a4ba00961e 100644 --- a/.github/workflows/codechecker_pr_analysis.yml +++ b/.github/workflows/codechecker_pr_analysis.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.10' - name: Install dependencies run: | pip install $(grep -iE "pylint|pycodestyle" analyzer/requirements_py/dev/requirements.txt) diff --git a/.github/workflows/config_coverage.yml b/.github/workflows/config_coverage.yml index 7c03fba876..e546ca4ac9 100644 --- a/.github/workflows/config_coverage.yml +++ b/.github/workflows/config_coverage.yml @@ -31,7 +31,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.10' - name: "Install dependencies" run: | # Some packages, e.g. build-essential and curl are available diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index b992762590..defbac9a65 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.10' - uses: actions/setup-node@v1 with: node-version: '16.x' @@ -63,7 +63,7 @@ jobs: steps: - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.10' - uses: actions/download-artifact@master with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 55ec5554ff..785a24d87b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,13 +20,30 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.10' - name: Install dependencies run: | pip install $(grep -iE "pylint|pycodestyle" analyzer/requirements_py/dev/requirements.txt) - name: Run pycodestyle & pylint run: make -k pycodestyle pylint + type-checker: + name: Type checker (mypy) + + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Create venv-dev + run: | + make venv_dev + - name: Run mypy + run: | + make mypy + tools: name: Tools (report-converter, etc.) runs-on: ubuntu-24.04 @@ -35,7 +52,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.10' - name: Setup Bazel uses: abhinavsingh/setup-bazel@v3 with: @@ -91,7 +108,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.10' - name: Install dependencies run: sh .github/workflows/install-deps.sh @@ -109,25 +126,6 @@ jobs: working-directory: analyzer run: make test_unit_cov - common: - name: Common libraries - runs-on: ubuntu-24.04 - - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install requirements - working-directory: codechecker_common - run: | - pip install -r requirements_py/dev/requirements.txt - - - name: Run mypy tests - working-directory: codechecker_common/tests - run: make mypy - web: name: Web runs-on: ubuntu-24.04 @@ -153,7 +151,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.10' - name: Install dependencies run: sh .github/workflows/install-deps.sh @@ -198,7 +196,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.10' - uses: actions/setup-node@v1 with: node-version: '16.x' diff --git a/Makefile b/Makefile index 697f5da6a6..337687651d 100644 --- a/Makefile +++ b/Makefile @@ -179,6 +179,9 @@ pycodestyle: pycodestyle_in_env: $(ACTIVATE_DEV_VENV) && $(PYCODE_CMD) +mypy: + $(ACTIVATE_DEV_VENV) && mypy --config-file pyproject.toml + test: test_analyzer test_web test_in_env: test_analyzer_in_env test_web_in_env diff --git a/analyzer/codechecker_analyzer/cli/analyze.py b/analyzer/codechecker_analyzer/cli/analyze.py index 7daae8ffad..69467af6c6 100644 --- a/analyzer/codechecker_analyzer/cli/analyze.py +++ b/analyzer/codechecker_analyzer/cli/analyze.py @@ -1094,6 +1094,47 @@ def __update_review_status_config(args): os.symlink(args.review_status_config, rs_config_to_send) +def __update_analysis_config_files(args): + """ + Copy analysis related configuration files + (e.g. skipfile, review_status_config) to report_dir/conf/. + """ + conf_dir = os.path.join(args.output_path, "conf") + + # Remove any config files used during previous analysis + if os.path.isdir(conf_dir): + shutil.rmtree(conf_dir) + + # Create a new conf directory + os.makedirs(conf_dir) + + def add_file_to_conf_dir(file_path: str): + if not os.path.isfile(file_path): + return + + file_path = os.path.abspath(file_path) + filename = os.path.basename(file_path) + shutil.copyfile(file_path, os.path.join(conf_dir, filename)) + + # Add analysis config files e.g., skipfile, review_status_config, etc. + if getattr(args, "skipfile", None): + add_file_to_conf_dir(args.skipfile) + + if getattr(args, "review_status_config", None): + add_file_to_conf_dir(args.review_status_config) + + if getattr(args, "config_file", None): + add_file_to_conf_dir(args.config_file) + + # Add cc-verbatim-args-file + # + # Example: --saargs , --tidyargs , + # --analyzer-config clangsa:cc-verbatim-args-file=, etc. + for a_conf in args.analyzer_config: + if a_conf.option == "cc-verbatim-args-file": + add_file_to_conf_dir(a_conf.value) + + def __cleanup_metadata(metadata_prev, metadata): """ Cleanup metadata. @@ -1455,6 +1496,7 @@ def main(args): __update_skip_file(args) __update_review_status_config(args) + __update_analysis_config_files(args) LOG.debug("Cleanup metadata file started.") __cleanup_metadata(metadata_prev, metadata) diff --git a/analyzer/codechecker_analyzer/cli/parse.py b/analyzer/codechecker_analyzer/cli/parse.py index bf702f24de..523426172c 100644 --- a/analyzer/codechecker_analyzer/cli/parse.py +++ b/analyzer/codechecker_analyzer/cli/parse.py @@ -299,7 +299,6 @@ def get_report_dir_status(compile_commands: List[dict[str, str]], for c in compile_commands: for analyzer in supported_analyzers: file, directory, cmd = c["file"], c["directory"], c["command"] - file = os.path.abspath(file) filename = os.path.basename(file) action_hash = analyzer_action_hash(file, directory, cmd) @@ -407,10 +406,25 @@ def print_status(report_dir: str, compile_cmd_path) sys.exit(1) + # Convert all relative compile_cmd.json file paths to absolute + for c in compile_commands: + if not os.path.isabs(c["file"]): + c["file"] = os.path.abspath( + os.path.join(c["directory"], c["file"])) + if files: - files_filter = [os.path.abspath(fp) for fp in files] + file_filter = [os.path.abspath(fp) for fp in files] + + invalid_file_filter = [fp for fp in file_filter + if not os.path.isfile(fp)] + + if invalid_file_filter: + LOG.error("File filter (--file) contains nonexistent files:") + LOG.error(invalid_file_filter) + sys.exit(1) + compile_commands = list( - filter(lambda c: c["file"] in files_filter, compile_commands)) + filter(lambda c: c["file"] in file_filter, compile_commands)) if not compile_commands and not export: LOG.warning("File not found in the compilation database!") @@ -704,7 +718,7 @@ def get_output_file_path(default_file_name: str) -> Optional[str]: if os.path.isdir(input_dir) and os.path.isfile(compile_cmd_json): print_status(input_dir, False, - getattr(args, 'files', None)) + None) if statistics.num_of_reports: sys.exit(2) diff --git a/analyzer/tests/functional/cmdline/test_cmdline.py b/analyzer/tests/functional/cmdline/test_cmdline.py index 34b2091acb..d1ca045ec2 100644 --- a/analyzer/tests/functional/cmdline/test_cmdline.py +++ b/analyzer/tests/functional/cmdline/test_cmdline.py @@ -211,7 +211,9 @@ def test_checkers_guideline(self): 'Malloc', 'MallocSizeof', 'clang-diagnostic-format-overflow', - 'overflow-non-kprintf'])) + 'overflow-non-kprintf', + 'gcc-allocation-size', + 'gcc-out-of-bounds'])) checkers_cmd = [env.codechecker_cmd(), 'checkers', '--guideline'] _, out, _ = run_cmd(checkers_cmd) diff --git a/analyzer/tests/functional/parse_status/test_parse_status.py b/analyzer/tests/functional/parse_status/test_parse_status.py index 6c4fb59e0b..48efec952b 100644 --- a/analyzer/tests/functional/parse_status/test_parse_status.py +++ b/analyzer/tests/functional/parse_status/test_parse_status.py @@ -72,6 +72,9 @@ def __run_cmd(self, cmd): print(out) print(err) + if process.returncode != 0: + return err + output = out.splitlines(True) processed_output = [] for line in output: @@ -107,6 +110,10 @@ def __log_and_analyze(self): self.__run_cmd(analyze_cmd) + def __get_file_list(self, parsed_json, analyzer, list_type): + return list(map(os.path.basename, + parsed_json["analyzers"][analyzer][list_type])) + def test_parse_status_summary(self): self.__log_and_analyze() @@ -156,14 +163,69 @@ def test_parse_status_detailed_json(self): parsed_json = json.loads(out) - def get_file_list(analyzer, list_type): - return list(map( - os.path.basename, - parsed_json["analyzers"][analyzer][list_type])) - - self.assertListEqual(get_file_list("clangsa", "up-to-date"), + self.assertListEqual(self.__get_file_list(parsed_json, + "clangsa", "up-to-date"), ["a.cpp", "b.cpp"]) - self.assertListEqual(get_file_list("clang-tidy", "up-to-date"), + self.assertListEqual(self.__get_file_list(parsed_json, + "clang-tidy", "up-to-date"), []) - self.assertListEqual(get_file_list("clang-tidy", "missing"), + self.assertListEqual(self.__get_file_list(parsed_json, + "clang-tidy", "missing"), ["a.cpp", "b.cpp"]) + + def test_parse_status_filter(self): + self.__log_and_analyze() + + file_filter = str(os.path.abspath( + os.path.join(self.test_dir, "a.cpp"))) + + parse_status_cmd = [self._codechecker_cmd, "parse", + "--status", "-e", "json", "--detailed", + self.report_dir, "--file", file_filter] + out = self.__run_cmd(parse_status_cmd) + + parsed_json = json.loads(out) + + self.assertListEqual(self.__get_file_list(parsed_json, + "clangsa", "up-to-date"), + ["a.cpp"]) + self.assertListEqual(self.__get_file_list(parsed_json, + "clang-tidy", "up-to-date"), + []) + self.assertListEqual(self.__get_file_list(parsed_json, + "clang-tidy", "missing"), + ["a.cpp"]) + + def test_parse_status_relative_filter(self): + self.__log_and_analyze() + + file_filter = "a.cpp" + + parse_status_cmd = [self._codechecker_cmd, "parse", + "--status", "-e", "json", "--detailed", + self.report_dir, "--file", file_filter] + out = self.__run_cmd(parse_status_cmd) + + parsed_json = json.loads(out) + + self.assertListEqual(self.__get_file_list(parsed_json, + "clangsa", "up-to-date"), + ["a.cpp"]) + self.assertListEqual(self.__get_file_list(parsed_json, + "clang-tidy", "up-to-date"), + []) + self.assertListEqual(self.__get_file_list(parsed_json, + "clang-tidy", "missing"), + ["a.cpp"]) + + def test_parse_status_invalid_filter(self): + self.__log_and_analyze() + + file_filter = "nonexistent_file.cpp" + + parse_status_cmd = [self._codechecker_cmd, "parse", + "--status", "-e", "json", "--detailed", + self.report_dir, "--file", file_filter] + out = self.__run_cmd(parse_status_cmd) + + self.assertIn("File filter (--file) contains nonexistent files", out) diff --git a/codechecker_common/compatibility/multiprocessing.py b/codechecker_common/compatibility/multiprocessing.py index 754294014d..74ccfa1d4a 100644 --- a/codechecker_common/compatibility/multiprocessing.py +++ b/codechecker_common/compatibility/multiprocessing.py @@ -13,18 +13,20 @@ # pylint: disable=no-name-in-module # pylint: disable=unused-import if sys.platform in ["darwin", "win32"]: - from multiprocess import \ - Pipe, Pool, Process, \ - Queue, \ - Value, \ + from multiprocess import ( # type: ignore + Pipe, Pool, Process, + Queue, + Value, cpu_count - from multiprocess.managers import SyncManager + ) + from multiprocess.managers import SyncManager # type: ignore else: from concurrent.futures import ProcessPoolExecutor as Pool - from multiprocessing import \ - Pipe, \ - Process, \ - Queue, \ - Value, \ + from multiprocessing import ( + Pipe, + Process, + Queue, + Value, cpu_count + ) from multiprocessing.managers import SyncManager diff --git a/codechecker_common/requirements_py/dev/requirements.txt b/codechecker_common/requirements_py/dev/requirements.txt index b33bcd812c..951166bd10 100644 --- a/codechecker_common/requirements_py/dev/requirements.txt +++ b/codechecker_common/requirements_py/dev/requirements.txt @@ -1,6 +1,6 @@ portalocker~=3.0 coverage<=5.5.0 -mypy<=1.7.1 +mypy~=1.19.0 PyYAML~=6.0 types-PyYAML~=6.0 setuptools~=80.0 diff --git a/codechecker_common/review_status_handler.py b/codechecker_common/review_status_handler.py index 20c40207a3..a189b8dd82 100644 --- a/codechecker_common/review_status_handler.py +++ b/codechecker_common/review_status_handler.py @@ -184,7 +184,7 @@ def __report_matches_rule(self, report: Report, rule: dict): report.checker_name != rule['filters']['checker_name']: return False - if 'report_hash' in rule['filters'] and not \ + if 'report_hash' in rule['filters'] and report.report_hash and not \ report.report_hash.startswith(rule['filters']['report_hash']): return False @@ -211,7 +211,7 @@ def get_review_status(self, report: Report) -> SourceReviewStatus: return review_status # 3. Return "unreviewed" otherwise. - return SourceReviewStatus(bug_hash=report.report_hash) + return SourceReviewStatus(bug_hash=report.report_hash or "") def set_review_status_config(self, config_file): """ @@ -289,7 +289,7 @@ def get_review_status_from_config( message=rule['actions']['reason'] .encode(encoding='utf-8', errors='ignore') if 'reason' in rule['actions'] else b'', - bug_hash=report.report_hash, + bug_hash=report.report_hash or "", in_source=True) return None @@ -343,7 +343,7 @@ def get_review_status_from_source( return SourceReviewStatus( status=status, message=message.encode('utf-8'), - bug_hash=report.report_hash, + bug_hash=report.report_hash or "", in_source=True) if len(src_comment_data) > 1: diff --git a/codechecker_common/tests/Makefile b/codechecker_common/tests/Makefile deleted file mode 100644 index 984afc9d68..0000000000 --- a/codechecker_common/tests/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -# Environment variables to run tests. - -CURRENT_DIR = ${CURDIR} -# Root of the repository. -REPO_ROOT ?= $(CURRENT_DIR)/../.. - -MYPY_CMD = mypy --ignore-missing-imports $(REPO_ROOT)/codechecker_common - -mypy: - $(MYPY_CMD) - -mypy_in_env: venv_dev - $(ACTIVATE_DEV_VENV) && $(MYPY_CMD) diff --git a/config/labels/analyzers/cppcheck.json b/config/labels/analyzers/cppcheck.json index eaf37cf436..936fd6233f 100644 --- a/config/labels/analyzers/cppcheck.json +++ b/config/labels/analyzers/cppcheck.json @@ -21,7 +21,8 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:fio39-c" ], "cppcheck-StlMissingComparison": [ "profile:default", @@ -58,7 +59,8 @@ "memory-safety:cwe-121", "memory-safety:cwe-122", "memory-safety:cwe-126", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:arr30-c" ], "cppcheck-arrayIndexOutOfBoundsCond": [ "profile:default", @@ -100,7 +102,8 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:dcl30-c" ], "cppcheck-autovarInvalidDeallocation": [ "guideline:memory-safety", @@ -108,7 +111,8 @@ "profile:extreme", "profile:sensitive", "memory-safety:cwe-590", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:mem34-c" ], "cppcheck-badBitmaskCheck": [ "profile:default", @@ -132,7 +136,8 @@ "profile:sensitive", "memory-safety:cwe-121", "memory-safety:cwe-122", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:arr30-c" ], "cppcheck-catchExceptionByValue": [ "severity:STYLE" @@ -188,6 +193,10 @@ "profile:sensitive", "severity:HIGH" ], + "cppcheck-subtractPointers": [ + "severity:HIGH", + "sei-cert-c:arr36-c" + ], "cppcheck-compareValueOutOfTypeRangeError": [ "severity:STYLE" ], @@ -246,7 +255,9 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:ctr50-cpp", + "sei-cert-c:str53-cpp" ], "cppcheck-copyCtorAndEqOperator": [ "profile:default", @@ -284,7 +295,8 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:dcl30-c" ], "cppcheck-danglingReference": [ "profile:default", @@ -299,7 +311,9 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:exp54-cpp", + "sei-cert-c:mem50-cpp" ], "cppcheck-deallocDealloc": [ "profile:default", @@ -319,13 +333,15 @@ "profile:extreme", "profile:sensitive", "memory-safety:cwe-416", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:mem30-c", + "sei-cert-c:mem50-cpp" ], "cppcheck-derefInvalidIterator": [ "profile:default", "profile:extreme", "profile:sensitive", - "severity:MEDIUM" + "severity:HIGH" ], "cppcheck-divideSizeof": [ "profile:default", @@ -339,7 +355,9 @@ "profile:extreme", "profile:sensitive", "memory-safety:cwe-415", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:mem36-c", + "sei-cert-c:mem56-cpp" ], "cppcheck-duplInheritedMember": [ "profile:default", @@ -398,8 +416,12 @@ "profile:sensitive", "severity:MEDIUM" ], + "cppcheck-throwInEntryPoint": [ + "severity:HIGH", + "sei-cert-c:err55-cpp" + ], "cppcheck-fflushOnInputStream": [ - "severity:LOW" + "severity:MEDIUM" ], "cppcheck-floatConversionOverflow": [ "profile:default", @@ -495,7 +517,8 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:int32-c" ], "cppcheck-internalAstError": [ "severity:CRITICAL" @@ -507,7 +530,8 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:str52-cpp" ], "cppcheck-invalidContainerLoop": [ "severity:HIGH" @@ -522,7 +546,8 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:flp32-c" ], "cppcheck-invalidFunctionArgBool": [ "profile:default", @@ -552,10 +577,11 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:mem50-cpp" ], "cppcheck-invalidPointerCast": [ - "severity:LOW" + "severity:MEDIUM" ], "cppcheck-invalidPrintfArgType_float": [ "profile:default", @@ -701,8 +727,9 @@ "cppcheck-legacyUninitvar": [ "guideline:sei-cert-c", "profile:security", + "severity:HIGH", "sei-cert-c:exp33-c", - "severity:HIGH" + "sei-cert-c:exp53-cpp" ], "cppcheck-literalWithCharPtrCompare": [ "profile:default", @@ -732,19 +759,22 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:mem31-c" ], "cppcheck-memleakOnRealloc": [ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:err33-c" ], "cppcheck-memsetClass": [ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:exp62-cpp" ], "cppcheck-memsetClassFloat": [ "severity:LOW" @@ -776,7 +806,8 @@ "profile:extreme", "profile:sensitive", "memory-safety:cwe-762", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:mem51-cpp" ], "cppcheck-mismatchSize": [ "profile:default", @@ -821,7 +852,9 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:msc37-c", + "sei-cert-c:msc52-cpp" ], "cppcheck-moduloAlwaysTrueFalse": [ "profile:default", @@ -854,7 +887,8 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:MEDIUM" + "severity:MEDIUM", + "sei-cert-c:ctr50-cpp" ], "cppcheck-negativeIndex": [ "guideline:memory-safety", @@ -902,7 +936,8 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:exp34-c" ], "cppcheck-nullPointerArithmetic": [ "profile:default", @@ -920,17 +955,26 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:MEDIUM" + "severity:MEDIUM", + "sei-cert-c:err33-c" + ], + "cppcheck-nullPointerArithmeticOutOfMemory": [ + "severity:MEDIUM", + "sei-cert-c:err33-c" ], "cppcheck-nullPointerOutOfMemory": [ "profile:extreme", "profile:sensitive", - "severity:MEDIUM" + "severity:MEDIUM", + "sei-cert-c:err33-c", + "sei-cert-c:exp34-c", + "sei-cert-c:mem52-cpp" ], "cppcheck-nullPointerOutOfResources": [ "profile:extreme", "profile:sensitive", - "severity:MEDIUM" + "severity:MEDIUM", + "sei-cert-c:err33-c" ], "cppcheck-nullPointerRedundantCheck": [ "profile:default", @@ -1014,10 +1058,10 @@ "severity:STYLE" ], "cppcheck-pointerOutOfBounds": [ - "severity:LOW" + "severity:MEDIUM" ], "cppcheck-pointerOutOfBoundsCond": [ - "severity:LOW" + "severity:MEDIUM" ], "cppcheck-pointerPositive": [ "severity:STYLE" @@ -1032,7 +1076,7 @@ "severity:LOW" ], "cppcheck-preprocessorErrorDirective": [ - "severity:HIGH" + "severity:CRITICAL" ], "cppcheck-publicAllocationError": [ "profile:default", @@ -1044,7 +1088,7 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:MEDIUM" + "severity:HIGH" ], "cppcheck-purgedConfiguration": [ "severity:LOW" @@ -1111,7 +1155,8 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:fio42-c" ], "cppcheck-rethrowNoCurrentException": [ "profile:default", @@ -1140,7 +1185,10 @@ "profile:extreme", "profile:sensitive", "memory-safety:cwe-562", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:dcl30-c", + "sei-cert-c:exp54-cpp", + "sei-cert-c:exp61-cpp" ], "cppcheck-returnLocalVariable": [ "profile:default", @@ -1200,16 +1248,18 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:int34-c" ], "cppcheck-shiftNegativeLHS": [ - "severity:LOW" + "severity:MEDIUM" ], "cppcheck-shiftTooManyBits": [ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:int34-c" ], "cppcheck-shiftTooManyBitsSigned": [ "profile:default", @@ -1344,7 +1394,8 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:str30-c" ], "cppcheck-suspiciousCase": [ "profile:default", @@ -1362,7 +1413,8 @@ "severity:MEDIUM" ], "cppcheck-syntaxError": [ - "severity:CRITICAL" + "severity:CRITICAL", + "sei-cert-c:pre32-c" ], "cppcheck-thisSubtraction": [ "profile:default", @@ -1371,7 +1423,7 @@ "severity:MEDIUM" ], "cppcheck-thisUseAfterFree": [ - "severity:MEDIUM" + "severity:HIGH" ], "cppcheck-throwInNoexceptFunction": [ "profile:default", @@ -1428,7 +1480,9 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:exp53-cpp", + "sei-cert-c:mem53-cpp" ], "cppcheck-uninitstring": [ "profile:default", @@ -1442,13 +1496,16 @@ "profile:extreme", "profile:sensitive", "memory-safety:cwe-457", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:exp33-c" ], "cppcheck-unknownEvaluationOrder": [ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:exp30-c", + "sei-cert-c:exp50-cpp" ], "cppcheck-unknownMacro": [ "severity:CRITICAL" @@ -1580,7 +1637,8 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:exp58-cpp" ], "cppcheck-va_start_subsequentCalls": [ "profile:default", @@ -1610,7 +1668,8 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:oop52-cpp" ], "cppcheck-writeReadOnlyFile": [ "profile:default", @@ -1646,7 +1705,8 @@ "profile:default", "profile:extreme", "profile:sensitive", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:int33-c" ], "cppcheck-zerodivcond": [ "profile:default", diff --git a/config/labels/analyzers/gcc.json b/config/labels/analyzers/gcc.json index 706663ab6e..a3a32f1294 100644 --- a/config/labels/analyzers/gcc.json +++ b/config/labels/analyzers/gcc.json @@ -4,24 +4,30 @@ "gcc-allocation-size": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-allocation-size", "profile:extreme", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:exp36-c", + "sei-cert-c:mem35-c", + "sei-cert-c:mem54-cpp" ], "gcc-deref-before-check": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-deref-before-check", "profile:extreme", - "severity:MEDIUM" + "severity:MEDIUM", + "sei-cert-c:exp34-c" ], "gcc-double-fclose": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-double-fclose", "profile:extreme", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:fio42-c" ], "gcc-double-free": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-double-free", "guideline:memory-safety", "profile:extreme", "memory-safety:cwe-415", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:mem30-c" ], "gcc-exposure-through-output-file": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-exposure-through-output-file", @@ -36,7 +42,7 @@ "gcc-fd-access-mode-mismatch": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-fd-access-mode-mismatch", "profile:extreme", - "severity:HIGH" + "severity:MEDIUM" ], "gcc-fd-double-close": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-fd-double-close", @@ -46,39 +52,47 @@ "gcc-fd-leak": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-fd-leak", "profile:extreme", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:pos35-c", + "sei-cert-c:pos38-c" ], "gcc-fd-phase-mismatch": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-fd-phase-mismatch", "profile:extreme", - "severity:HIGH" + "severity:MEDIUM" ], "gcc-fd-type-mismatch": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-fd-type-mismatch", "profile:extreme", - "severity:HIGH" + "severity:MEDIUM" ], "gcc-fd-use-after-close": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-fd-use-after-close", "profile:extreme", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:pos35-c" ], "gcc-fd-use-without-check": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-fd-use-without-check", "profile:extreme", - "severity:MEDIUM" + "severity:MEDIUM", + "sei-cert-c:pos35-c", + "sei-cert-c:pos38-c" ], "gcc-file-leak": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-file-leak", "profile:extreme", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:fio42-c" ], "gcc-free-of-non-heap": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-free-of-non-heap", "guideline:memory-safety", "profile:extreme", "memory-safety:cwe-590", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:mem34-c", + "sei-cert-c:mem51-cpp" ], "gcc-imprecise-fp-arithmetic": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-imprecise-fp-arithmetic", @@ -93,7 +107,8 @@ "gcc-jump-through-null": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-jump-through-null", "profile:extreme", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:oop55-cpp" ], "gcc-malloc-leak": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-malloc-leak", @@ -101,26 +116,35 @@ "profile:extreme", "memory-safety:cwe-401", "memory-safety:cwe-761", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:err57-cpp", + "sei-cert-c:fio42-c", + "sei-cert-c:mem31-c" ], "gcc-mismatching-deallocation": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-mismatching-deallocation", "guideline:memory-safety", "profile:extreme", "memory-safety:cwe-762", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:exp51-cpp", + "sei-cert-c:mem51-cpp", + "sei-cert-c:mem57-cpp" ], "gcc-null-argument": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-null-argument", "profile:extreme", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:err33-c", + "sei-cert-c:exp34-c" ], "gcc-null-dereference": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-null-dereference", "guideline:memory-safety", "profile:extreme", "memory-safety:cwe-476", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:exp34-c" ], "gcc-out-of-bounds": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-out-of-bounds", @@ -132,32 +156,48 @@ "memory-safety:cwe-126", "memory-safety:cwe-127", "memory-safety:cwe-843", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:arr30-c", + "sei-cert-c:arr38-c", + "sei-cert-c:arr39-c", + "sei-cert-c:exp36-c", + "sei-cert-c:int10-c", + "sei-cert-c:mem33-c", + "sei-cert-c:mem35-c", + "sei-cert-c:mem54-cpp", + "sei-cert-c:oop55-cpp" ], "gcc-possible-null-argument": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-possible-null-argument", "profile:extreme", - "severity:LOW" + "severity:MEDIUM", + "sei-cert-c:pos54-c", + "sei-cert-c:err33-c" ], "gcc-possible-null-dereference": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-possible-null-dereference", "profile:extreme", - "severity:LOW" + "severity:MEDIUM", + "sei-cert-c:exp34-c", + "sei-cert-c:err33-c" ], "gcc-putenv-of-auto-var": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-putenv-of-auto-var", "profile:extreme", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:pos34-c" ], "gcc-shift-count-negative": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-shift-count-negative", "profile:extreme", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:int34-c" ], "gcc-shift-count-overflow": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-shift-count-overflow", "profile:extreme", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:int34-c" ], "gcc-stale-setjmp-buffer": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-stale-setjmp-buffer", @@ -197,39 +237,54 @@ "gcc-unsafe-call-within-signal-handler": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-unsafe-call-within-signal-handler", "profile:extreme", - "severity:MEDIUM" + "severity:HIGH", + "sei-cert-c:sig30-c" ], "gcc-use-after-free": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-use-after-free", "guideline:memory-safety", "profile:extreme", "memory-safety:cwe-416", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:mem30-c" ], "gcc-use-of-pointer-in-stale-stack-frame": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-use-of-pointer-in-stale-stack-frame", "profile:extreme", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:dcl30-c", + "sei-cert-c:exp61-cpp" ], "gcc-use-of-uninitialized-value": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-use-of-uninitialized-value", "profile:extreme", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:arr38-c", + "sei-cert-c:dcl41-c", + "sei-cert-c:err56-cpp", + "sei-cert-c:exp33-c", + "sei-cert-c:exp53-cpp", + "sei-cert-c:oop53-cpp", + "sei-cert-c:oop54-cpp" ], "gcc-va-arg-type-mismatch": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-va-arg-type-mismatch", "profile:extreme", - "severity:MEDIUM" + "severity:HIGH", + "sei-cert-c:msc39-c" ], "gcc-va-list-exhausted": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-va-list-exhausted", "profile:extreme", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:dcl50-cpp", + "sei-cert-c:exp47-c" ], "gcc-va-list-leak": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-va-list-leak", "profile:extreme", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:exp58-cpp" ], "gcc-va-list-use-after-va-end": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-va-list-use-after-va-end", @@ -239,11 +294,52 @@ "gcc-write-to-const": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-write-to-const", "profile:extreme", - "severity:HIGH" + "severity:HIGH", + "sei-cert-c:str30-c" ], "gcc-write-to-string-literal": [ "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-write-to-string-literal", "profile:extreme", + "severity:HIGH", + "sei-cert-c:str30-c" + ], + "gcc-infinite-loop": [ + "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-infinite-loop", + "profile:extreme", + "severity:HIGH", + "sei-cert-c:exp45-c" + ], + "gcc-overlapping-buffers": [ + "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-overlapping-buffers", + "profile:extreme", + "severity:HIGH", + "sei-cert-c:exp43-c" + ], + "gcc-symbol-too-complex": [ + "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-symbol-too-complex", + "profile:extreme", + "severity:LOW" + ], + "gcc-throw-of-unexpected-type": [ + "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-throw-of-unexpected-type", + "profile:extreme", + "severity:HIGH", + "sei-cert-c:err55-cpp" + ], + "gcc-too-complex": [ + "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-too-complex", + "profile:extreme", + "severity:LOW" + ], + "gcc-undefined-behavior-ptrdiff": [ + "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-undefined-behavior-ptrdiff", + "profile:extreme", + "severity:HIGH", + "sei-cert-c:arr36-c" + ], + "gcc-undefined-behavior-strtok": [ + "doc_url:https://gcc.gnu.org/onlinedocs/gcc/Static-Analyzer-Options.html#index-Wanalyzer-undefined-behavior-strtok", + "profile:extreme", "severity:HIGH" ] } diff --git a/docs/web/background_tasks.md b/docs/web/background_tasks.md index b53fedf176..2d0d167b20 100644 --- a/docs/web/background_tasks.md +++ b/docs/web/background_tasks.md @@ -32,10 +32,13 @@ Tasks are generally spawned by API handlers, executed in the control flow of a T 1. An **API** request arrives (later, this might be extended with a _`cron`_ -like scheduler) which exercises an endpoint that results in the need for a task. 2. _(Optionally)_ some conformance checks are executed on the input, in order to not even create the task if the input is ill-formed. -3. A task **`token`** is _`ALLOCATED`_: the record is written into the database, and now we have a unique identifier for the task. -4. The task is **pushed** to the _task queue_ of the CodeChecker server, resulting in the _`ENQUEUED`_ status. -5. The task's identifier **`token`** is returned to the user. +3. A task **`token`** is _`ALLOCATED`_: the **`BackgroundTask`** record is written into the database, and now we have a unique identifier for the task. +4. The task is **pushed** to a shared, synchronised _task queue_ of the CodeChecker server, resulting in the _`ENQUEUED`_ status. + * `AbstractTask` subclasses **MUST** be `pickle`-able and reasonably small. + * The library offers means to store additional large data on the file system, in a temporary directory specific to the task. +5. The **`task token`** is returned to the user via the RPC API call, and the API worker is free too respond to other requests. 6. The API hander exits and the Thrift RPC connection is terminated. +7. In a loop with some frequency, the user exercises the `getTaskInfo()` API (executed in the context of any _API worker_ process, synchronised over the database) to query whether the task was completed, if the user wishes to receive this information. The API request dispatching of the CodeChecker server has a **`TaskManager`** instance which should be passed to the API handler implementation, if not already available. Then, you can use this _`TaskManager`_ object to perform the necessary actions to enqueue the execution of a task: @@ -118,7 +121,7 @@ The business logic of tasks are implemented by subclassing the _`AbstractTask`_ 4. The implementation does its thing, periodically calling _`task_manager.heartbeat()`_ to update the progress timestamp of the task, and, if appropriate, checking with _`task_manager.should_cancel()`_ whether the admins requested the task to cancel or the server is shutting down. 5. If _`should_cancel()`_ returned `True`, the task does some appropriate clean-up, and exits by raising the special _`TaskCancelHonoured`_ exception, indicating that it responded to the request. (At this point, the status becomes either _`CANCELLED`_ or _`DROPPED`_, depending on the circumstances of the service.) 6. Otherwise, or if the task is for some reason not cancellable without causing damage, the task executes its logic. -7. If the task's _`_implementation()`_ method exits cleanly, it reaches the _`COMPLETED`_ status; otherwise, if any exception escapes from the _`_implementation()`_ method, the task becomes _`FAILED`_. +7. If the task's _`_implementation()`_ method exits cleanly, it reaches the _`COMPLETED`_ status; otherwise, if any exception escapes from the _`_implementation()`_ method, the task becomes _`FAILED`_, and exception information is logged into the `BackgroundTask.comments` column of the database. **Caution!** Tasks, executing in a separate background process part of the many processes spawned by a CodeChecker server, no longer have the ability to synchronously communicate with the user! This also includes the lack of ability to "return" a value: tasks **only exercise side-effects**, but do not calculate a "result". @@ -170,6 +173,32 @@ class MyTask(AbstractTask): foo(element) ``` +### Abnormal path 1: admin cancellation + +At any point following _`ALLOCATED`_ status, but most likely in the _`ENQUEUED`_ and _`RUNNING`_ statuses, a **`SUPERUSER`** may issue a _`cancelTask()`_ order. +This will set `BackgroundTask.cancel_flag`, and the task is expected (although not required!) to poll its own _`should_cancel()`_ status internally in checkpoints, and terminate gracefully to this request. This is done by **`_implementation()`** exiting by raising a **`TaskCancelHonoured`** exception. +(If the task does not raise one, it will be allowed to conclude normally, or fail in some other manner. +Tasks cancelled gracefully will have the _`CANCELLED`_ status. + +For example, a background task that performs an action over a set of input files generally should be implemented like this: + +```py3 +def _implementation(tm: TaskManager): + for file in INPUTS: + if tm.should_cancel(self): + ROLLBACK() + raise TaskCancelHonoured(self) + + DO_LOGIC(file) +``` + +### Abnormal path 2: server shutdown + +Alternatively, at any point in this life cycle, the server might receive the command to terminate itself (kill signals `SIGINT`, `SIGTERM`; alternatively caused by `CodeChecker server --stop`). Following the termination of _API workers_, the _background workers_ will also shut down one by one. +At this point, the default behaviour is to cause a special _cancel event_ which tasks currently _`RUNNING`_ may still gracefully honour, as-if it was a `SUPERUSER`'s single-task cancel request. All other tasks that have not started executing yet and are in the _`ALLOCATED`_ or _`ENQUEUED`_ status will never start. + +All tasks not in a _normal termination state_ will be set to the _`DROPPED`_ status, with the `comments` field containing a log about the specifics of in which state the task was dropped, and why. (Together, _`CANCELLED`_ and _`DROPPED`_ are the _"abnormal termination states"_, indicating that the task terminated due to some external influence.) + Client-side handling -------------------- diff --git a/docs/web/server_config.md b/docs/web/server_config.md index 23f6e98ffd..1f1bf8e217 100644 --- a/docs/web/server_config.md +++ b/docs/web/server_config.md @@ -9,15 +9,28 @@ using the package's installed `config/server_config.json` as a template. Table of Contents ================= +* [Task handling](#task-handling) + * [Number of API worker processes](#number-of-api-worker-processes) + * [Number of task worker processes](#number-of-task-worker-processes) + * [Run limitation](#run-limitations) * [Storage](#storage) * [Directory of analysis statistics](#directory-of-analysis-statistics) * [Limits](#Limits) * [Maximum size of failure zips](#maximum-size-of-failure-zips) * [Size of the compilation database](#size-of-the-compilation-database) + * [Keepalive](#keepalive) + * [Idle time](#idle-time) + * [Interval time](#interval-time) + * [Probes](#probes) * [Authentication](#authentication) +* [Secrets](#secrets) + * [server_secrets.json](#server_secretsjson) + * [Environmental variables](#environmental-variables) + +## Task handling -## Number of API worker processes +### Number of API worker processes The `worker_processes` section of the config file controls how many processes will be started on the server to process API requests. @@ -33,6 +46,37 @@ processes will be started on the server to process background jobs. The server needs to be restarted if the value is changed in the config file. +### `--machine-id` +Unfortunately, servers don't always terminate gracefully (cue the aforementioned +`SIGKILL`, but also the container, VM, or the host machine could simply die +during execution, in ways the server is not able to handle). Because tasks are +not shared across server processes, and there are crucial bits of information in +the now dead process's memory which would have been needed to execute the task, +a server later restarting in place of a previously dead one should be able to +identify which tasks its "predecessor" left behind without clean-up. + +This is achieved by storing the running computer's identifier, configurable via +`CodeChecker server --machine-id`, as an additional piece of information for +each task. By default, the machine ID is constructed from +`gethostname():portnumber`, e.g., `cc-server:80`. + +In containerised environments, relying on `gethostname()` may not be entirely +stable! For example, Docker exposes the first 12 digits of the container's +unique hash as the _"hostname"_ of the insides of the container. If the +container is started with `--restart always` or `--restart unless-stopped`, then +this is fine, however, more advanced systems, such as _Docker swarm_ will +**create a new container** in case the old one died (!), resulting in a new +value of `gethostname()`. + +In such environments, service administrators must pay additional caution and +configure their instances by setting `--machine-id` for subsequent executions of +the "same" server accordingly. If a server with machine ID **`M`** starts up +(usually after a container or "system" restart), it will set every task not in +any "termination states" and associated with machine ID **`M`** to the +_`DROPPED`_ status (with an appropriately formatted comment accompanying), +signifying that the _previous instance_ "dropped" these tasks, but had no chance +of recording this fact. + ## Run limitation The `max_run_count` section of the config file controls how many runs can be stored on the server for a product. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..b547b1b141 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,13 @@ +[tool.mypy] +verbosity = 1 +show_error_codes = true + +files = [ + "codechecker_common/" +] + +mypy_path = [ + "analyzer/", + "web/", + "tools/report-converter/" +] diff --git a/web/api/js/codechecker-api-node/dist/codechecker-api-6.69.0.tgz b/web/api/js/codechecker-api-node/dist/codechecker-api-6.69.0.tgz deleted file mode 100644 index c55bae37e2..0000000000 Binary files a/web/api/js/codechecker-api-node/dist/codechecker-api-6.69.0.tgz and /dev/null differ diff --git a/web/api/js/codechecker-api-node/dist/codechecker-api-6.70.0.tgz b/web/api/js/codechecker-api-node/dist/codechecker-api-6.70.0.tgz new file mode 100644 index 0000000000..09d3b765e7 Binary files /dev/null and b/web/api/js/codechecker-api-node/dist/codechecker-api-6.70.0.tgz differ diff --git a/web/api/js/codechecker-api-node/package.json b/web/api/js/codechecker-api-node/package.json index 8921159775..b47d543df9 100644 --- a/web/api/js/codechecker-api-node/package.json +++ b/web/api/js/codechecker-api-node/package.json @@ -1,6 +1,6 @@ { "name": "codechecker-api", - "version": "6.69.0", + "version": "6.70.0", "description": "Generated node.js compatible API stubs for CodeChecker server.", "main": "lib", "homepage": "https://github.com/Ericsson/codechecker", diff --git a/web/api/py/codechecker_api/dist/codechecker_api.tar.gz b/web/api/py/codechecker_api/dist/codechecker_api.tar.gz index e3707cf43d..695b1d232b 100644 Binary files a/web/api/py/codechecker_api/dist/codechecker_api.tar.gz and b/web/api/py/codechecker_api/dist/codechecker_api.tar.gz differ diff --git a/web/api/py/codechecker_api/setup.py b/web/api/py/codechecker_api/setup.py index 07209f6809..ecf0d97ee1 100644 --- a/web/api/py/codechecker_api/setup.py +++ b/web/api/py/codechecker_api/setup.py @@ -8,7 +8,7 @@ with open('README.md', encoding='utf-8', errors="ignore") as f: long_description = f.read() -api_version = '6.69.0' +api_version = '6.70.0' setup( name='codechecker_api', diff --git a/web/api/py/codechecker_api_shared/dist/codechecker_api_shared.tar.gz b/web/api/py/codechecker_api_shared/dist/codechecker_api_shared.tar.gz index ff580aaf65..202acc1df2 100644 Binary files a/web/api/py/codechecker_api_shared/dist/codechecker_api_shared.tar.gz and b/web/api/py/codechecker_api_shared/dist/codechecker_api_shared.tar.gz differ diff --git a/web/api/py/codechecker_api_shared/setup.py b/web/api/py/codechecker_api_shared/setup.py index 605a39c2b8..f789554aa4 100644 --- a/web/api/py/codechecker_api_shared/setup.py +++ b/web/api/py/codechecker_api_shared/setup.py @@ -8,7 +8,7 @@ with open('README.md', encoding='utf-8', errors="ignore") as f: long_description = f.read() -api_version = '6.69.0' +api_version = '6.70.0' setup( name='codechecker_api_shared', diff --git a/web/api/report_server.thrift b/web/api/report_server.thrift index 3745c7e84b..60a133427f 100644 --- a/web/api/report_server.thrift +++ b/web/api/report_server.thrift @@ -946,6 +946,21 @@ service codeCheckerDBAccess { 5: i64 offset) throws (1: codechecker_api_shared.RequestFailed requestError), + // Returns detailed report statistics grouped by file. + // The inner map contains total report count ("reports") and + // counts per severity, review status and detection status. + // If the run id list is empty the metrics will be counted + // for all of the runs and in compare mode all of the runs + // will be used as a baseline excluding the runs in compare data. + // PERMISSION: PRODUCT_VIEW + map> getFileCountsSummary( + 1: list runIds, + 2: ReportFilter reportFilter, + 3: CompareData cmpData, + 4: i64 limit, + 5: i64 offset) + throws (1: codechecker_api_shared.RequestFailed requestError), + // If the run id list is empty the metrics will be counted // for all of the runs and in compare mode all of the runs // will be used as a baseline excluding the runs in compare data. diff --git a/web/client/codechecker_client/helpers/results.py b/web/client/codechecker_client/helpers/results.py index 4285b5db4b..bfc42ef6ae 100644 --- a/web/client/codechecker_client/helpers/results.py +++ b/web/client/codechecker_client/helpers/results.py @@ -140,6 +140,11 @@ def getDetectionStatusCounts(self, runIds, reportFilter, cmpData): def getFileCounts(self, runIds, reportFilter, cmpData, limit, offset): pass + @thrift_client_call + def getFileCountsSummary(self, runIds, reportFilter, cmpData, limit, + offset): + pass + @thrift_client_call def getCheckerCounts(self, base_run_ids, reportFilter, cmpData, limit, offset): diff --git a/web/codechecker_web/shared/version.py b/web/codechecker_web/shared/version.py index eb8d5245f7..75d538edef 100644 --- a/web/codechecker_web/shared/version.py +++ b/web/codechecker_web/shared/version.py @@ -20,7 +20,7 @@ # The newest supported minor version (value) for each supported major version # (key) in this particular build. SUPPORTED_VERSIONS = { - 6: 69 + 6: 70 } # Used by the client to automatically identify the latest major and minor diff --git a/web/requirements.txt b/web/requirements.txt index cd21c5f769..7316363c30 100644 --- a/web/requirements.txt +++ b/web/requirements.txt @@ -11,6 +11,7 @@ sqlalchemy~=2.0 alembic~=1.5 portalocker~=3.0 psutil~=7.0 +types-psutil~=7.0 multiprocess~=0.70 thrift~=0.22 gitpython~=3.0 diff --git a/web/server/codechecker_server/api/authentication.py b/web/server/codechecker_server/api/authentication.py index f12f8e6cd6..4dc9f39672 100644 --- a/web/server/codechecker_server/api/authentication.py +++ b/web/server/codechecker_server/api/authentication.py @@ -68,20 +68,13 @@ def __require_privilaged_access(self): "The server must be start by using privilaged access to " "execute this action.") - def __has_permission(self, permission) -> bool: - """ True if the current user has given permission rights. """ - if self.__manager.is_enabled and not self.__auth_session: - return False - - return self.hasPermission(permission, None) - def __require_permission_view(self): """ Checks if the curret user has PERMISSION_VIEW rights. Throws an exception if it is not. """ permission = codechecker_api_shared.ttypes.Permission.PERMISSION_VIEW - if not self.__has_permission(permission): + if not self.hasPermission(permission, None): raise codechecker_api_shared.ttypes.RequestFailed( codechecker_api_shared.ttypes.ErrorCode.UNAUTHORIZED, "You are not authorized to execute this action.") @@ -598,6 +591,7 @@ def getPermissionsForUser(self, scope, extra_params, perm_filter): # handler. params = ThriftAuthHandler.__unpack_extra_params(extra_params, session) + is_auth_enabled = self.__manager.is_enabled perms = [] for perm in permissions.get_permissions(scope): @@ -605,14 +599,15 @@ def getPermissionsForUser(self, scope, extra_params, perm_filter): handler = make_handler(perm, params) if should_return and perm_filter.given: - should_return = handler.has_permission(self.__auth_session) + should_return = handler.has_permission(self.__auth_session, + is_auth_enabled) if should_return and perm_filter.canManage: # If the user has any of the permissions that are # authorised to manage the currently iterated permission, # the filter passes. should_return = require_manager( - perm, params, self.__auth_session) + perm, params, self.__auth_session, is_auth_enabled) if should_return: perms.append(perm) @@ -631,7 +626,8 @@ def getAuthorisedNames(self, permission, extra_params): perm, params = ThriftAuthHandler.__create_permission_args( permission, extra_params, session) - if not require_manager(perm, params, self.__auth_session): + if not require_manager(perm, params, self.__auth_session, + self.__manager.is_enabled): raise codechecker_api_shared.ttypes.RequestFailed( codechecker_api_shared.ttypes.ErrorCode.UNAUTHORIZED, f"You can not manage the permission '{perm.name}'") @@ -654,7 +650,8 @@ def addPermission(self, permission, auth_name, is_group, extra_params): perm, params = ThriftAuthHandler.__create_permission_args( permission, extra_params, session) - if not require_manager(perm, params, self.__auth_session): + if not require_manager(perm, params, self.__auth_session, + self.__manager.is_enabled): raise codechecker_api_shared.ttypes.RequestFailed( codechecker_api_shared.ttypes.ErrorCode.UNAUTHORIZED, f"You can not manage the permission '{perm.name}'") @@ -677,7 +674,8 @@ def removePermission(self, permission, auth_name, is_group, extra_params): perm, params = ThriftAuthHandler.__create_permission_args( permission, extra_params, session) - if not require_manager(perm, params, self.__auth_session): + if not require_manager(perm, params, self.__auth_session, + self.__manager.is_enabled): raise codechecker_api_shared.ttypes.RequestFailed( codechecker_api_shared.ttypes.ErrorCode.UNAUTHORIZED, f"You can not manage the permission '{perm.name}'") @@ -703,7 +701,8 @@ def hasPermission(self, permission, extra_params): permission, extra_params, session) return require_permission(perm, params, - self.__auth_session) + self.__auth_session, + self.__manager.is_enabled) # ============= Authorization, permission management ============= diff --git a/web/server/codechecker_server/api/product_server.py b/web/server/codechecker_server/api/product_server.py index 51f69ca800..222ade2d01 100644 --- a/web/server/codechecker_server/api/product_server.py +++ b/web/server/codechecker_server/api/product_server.py @@ -76,15 +76,9 @@ def __require_permission(self, required, args=None): if 'config_db_session' not in args: args['config_db_session'] = session - # Anonymous access is only allowed if authentication is - # turned off - if self.__server.manager.is_enabled and not self.__auth_session: - raise codechecker_api_shared.ttypes.RequestFailed( - codechecker_api_shared.ttypes.ErrorCode.UNAUTHORIZED, - "You are not authorized to execute this action.") - if not any(permissions.require_permission( - perm, args, self.__auth_session) + perm, args, self.__auth_session, + self.__server.manager.is_enabled) for perm in required): raise codechecker_api_shared.ttypes.RequestFailed( codechecker_api_shared.ttypes.ErrorCode.UNAUTHORIZED, @@ -95,11 +89,13 @@ def __require_permission(self, required, args=None): def __administrating(self, args): """ True if the current user can administrate the given product. """ if permissions.require_permission(permissions.SUPERUSER, args, - self.__auth_session): + self.__auth_session, + self.__server.manager.is_enabled): return True if permissions.require_permission(permissions.PRODUCT_ADMIN, args, - self.__auth_session): + self.__auth_session, + self.__server.manager.is_enabled): return True return False @@ -126,9 +122,11 @@ def __get_product(self, session, product): 'productID': product.id} has_product_permission = permissions.require_permission( - permissions.PRODUCT_VIEW, args, self.__auth_session) + permissions.PRODUCT_VIEW, args, self.__auth_session, + self.__server.manager.is_enabled) has_global_permission = permissions.require_permission( - permissions.PERMISSION_VIEW, args, self.__auth_session) + permissions.PERMISSION_VIEW, args, self.__auth_session, + self.__server.manager.is_enabled) has_access_permission = has_product_permission or has_global_permission admin_perm_name = permissions.PRODUCT_ADMIN.name @@ -180,7 +178,8 @@ def isAdministratorOfAnyProduct(self): 'productID': prod.id} if permissions.require_permission( permissions.PRODUCT_ADMIN, - args, self.__auth_session): + args, self.__auth_session, + self.__server.manager.is_enabled): return True return False diff --git a/web/server/codechecker_server/api/report_server.py b/web/server/codechecker_server/api/report_server.py index 221dc382ad..e45785c4f8 100644 --- a/web/server/codechecker_server/api/report_server.py +++ b/web/server/codechecker_server/api/report_server.py @@ -1509,15 +1509,9 @@ def __require_permission(self, required): args = dict(self.__permission_args) args['config_db_session'] = session - # Anonymous access is only allowed if authentication is - # turned off - if self._manager.is_enabled and not self._auth_session: - raise codechecker_api_shared.ttypes.RequestFailed( - codechecker_api_shared.ttypes.ErrorCode.UNAUTHORIZED, - "You are not authorized to execute this action.") - if not any(permissions.require_permission( - perm, args, self._auth_session) + perm, args, self._auth_session, + self._manager.is_enabled) for perm in required): raise codechecker_api_shared.ttypes.RequestFailed( codechecker_api_shared.ttypes.ErrorCode.UNAUTHORIZED, @@ -1891,7 +1885,7 @@ def getAnalysisInfo(self, analysis_info_filter, limit, offset): for cmd in analysis_info_query: command = zlib.decompress(cmd.analyzer_command) \ - .decode("utf-8") + .decode("utf-8") if cmd.analyzer_command else "" checkers_q = session \ .query(Checker.analyzer_name, @@ -3023,6 +3017,8 @@ def getGuidelineRules( guidelines: List[ttypes.Guideline] ): """ Return the list of rules to each guideline that given. """ + self.__require_view() + guideline_rules = defaultdict(list) for guideline in guidelines: rules = self._context.guideline.rules_of_guideline( @@ -3722,6 +3718,100 @@ def getFileCounts(self, run_ids, report_filter, cmp_data, limit, offset): results[fp] = count return results + @exc_to_thrift_reqfail + @timeit + def getFileCountsSummary(self, run_ids, report_filter, cmp_data, + limit, offset): + # Returns detailed report statistics grouped by file. + # The inner map contains total report count ("reports") and + # counts per severity, review status and detection status. + # If the run id list is empty the metrics will be counted + # for all of the runs and in compare mode all of the runs + # will be used as a baseline excluding the runs in compare data. + # PERMISSION: PRODUCT_VIEW + self.__require_view() + + limit = verify_limit_range(limit) + + results = {} + with DBSession(self._Session) as session: + filter_expression, join_tables = process_report_filter( + session, run_ids, report_filter, cmp_data) + + # Get distinct file paths with pagination. + distinct_file_path = session.query(File.filepath.distinct()) \ + .join(Report, Report.file_id == File.id) + + if report_filter.annotations is not None: + distinct_file_path = distinct_file_path.outerjoin( + ReportAnnotations, + ReportAnnotations.report_id == Report.id) + distinct_file_path = distinct_file_path.group_by( + Report.id) + + distinct_file_path = apply_report_filter( + distinct_file_path, filter_expression, join_tables, [File]) + + if limit: + distinct_file_path = distinct_file_path.limit(limit) \ + .offset(offset) + + count_col = Report.bug_id.distinct() if \ + report_filter.isUnique else Report.bug_id + + # Query: file path, severity, review status, + # detection status, and count. + stmt = session.query( + File.filepath, + Checker.severity, + Report.review_status, + Report.detection_status, + func.count(count_col).label('cnt')) \ + .join(Report, Report.file_id == File.id) \ + .join(Checker, Report.checker_id == Checker.id) \ + .filter(File.filepath.in_(distinct_file_path)) + + stmt = apply_report_filter( + stmt, filter_expression, join_tables, [File, Checker]) + + stmt = stmt.group_by( + File.filepath, + Checker.severity, + Report.review_status, + Report.detection_status) + + severity_names = ttypes.Severity._VALUES_TO_NAMES + + for fp, sev, review_st, detect_st, cnt in stmt: + if fp not in results: + results[fp] = {} + + file_summary = results[fp] + + # Total report count. + file_summary["reports"] = \ + file_summary.get("reports", 0) + cnt + + # Severity count. + sev_name = severity_names.get(sev, str(sev)) + sev_key = f"severity:{sev_name}" + file_summary[sev_key] = \ + file_summary.get(sev_key, 0) + cnt + + # Review status count. + if review_st: + rs_key = f"review_status:{review_st}" + file_summary[rs_key] = \ + file_summary.get(rs_key, 0) + cnt + + # Detection status count. + if detect_st: + ds_key = f"detection_status:{detect_st}" + file_summary[ds_key] = \ + file_summary.get(ds_key, 0) + cnt + + return results + @exc_to_thrift_reqfail @timeit def getRunHistoryTagCounts(self, run_ids, report_filter, cmp_data, limit, diff --git a/web/server/codechecker_server/api/tasks.py b/web/server/codechecker_server/api/tasks.py index 54f5750ba0..c75b7c2847 100644 --- a/web/server/codechecker_server/api/tasks.py +++ b/web/server/codechecker_server/api/tasks.py @@ -22,6 +22,7 @@ from codechecker_common.logger import get_logger from codechecker_server.profiler import timeit +from codechecker_server.session_manager import SessionManager from ..database.config_db_model import BackgroundTask as DBTask, Product from ..database.database import DBSession, conv @@ -113,9 +114,11 @@ class ThriftTaskHandler: def __init__(self, configuration_database_sessionmaker, + session_manager: SessionManager, task_manager: TaskManager, auth_session): self._config_db = configuration_database_sessionmaker + self._session_manager = session_manager self._task_manager = task_manager self._auth_session = auth_session @@ -152,12 +155,14 @@ def getTaskInfo(self, token: str) -> TaskInfo: permissions.PRODUCT_ACCESS, {"config_db_session": session, "productID": associated_product.id}, - self._auth_session) + self._auth_session, + self._session_manager.is_enabled) else: has_right_to_query_status = permissions.require_permission( permissions.SUPERUSER, {"config_db_session": session}, - self._auth_session) + self._auth_session, + self._session_manager.is_enabled) if not has_right_to_query_status: raise RequestFailed( @@ -188,7 +193,8 @@ def getTasks(self, filters: TaskFilter) -> List[AdministratorTaskInfo]: if not permissions.require_permission( permissions.SUPERUSER, {"config_db_session": session}, - self._auth_session): + self._auth_session, + self._session_manager.is_enabled): raise RequestFailed( ErrorCode.UNAUTHORIZED, "Querying service tasks (not associated with a " @@ -199,7 +205,8 @@ def getTasks(self, filters: TaskFilter) -> List[AdministratorTaskInfo]: if not permissions.require_permission( permissions.PRODUCT_ACCESS, {"config_db_session": session, "productID": prod_id}, - self._auth_session)] + self._auth_session, + self._session_manager.is_enabled)] if no_access_products: no_access_products = [session.get(Product, product_id) .endpoint @@ -299,7 +306,8 @@ def getTasks(self, filters: TaskFilter) -> List[AdministratorTaskInfo]: has_superuser = permissions.require_permission( permissions.SUPERUSER, {"config_db_session": session}, - self._auth_session) + self._auth_session, + self._session_manager.is_enabled) if not has_superuser: continue else: @@ -314,7 +322,8 @@ def getTasks(self, filters: TaskFilter) -> List[AdministratorTaskInfo]: permissions.PRODUCT_ACCESS, {"config_db_session": session, "productID": db_task.product_id}, - self._auth_session) + self._auth_session, + self._session_manager.is_enabled) if not product_access_rights[db_task.product_id]: continue @@ -351,12 +360,14 @@ def cancelTask(self, token: str) -> bool: permissions.PRODUCT_ADMIN, {"config_db_session": session, "productID": associated_product.id}, - self._auth_session) + self._auth_session, + self._session_manager.is_enabled) else: has_right_to_cancel = permissions.require_permission( permissions.SUPERUSER, {"config_db_session": session}, - self._auth_session) + self._auth_session, + self._session_manager.is_enabled) if not has_right_to_cancel: raise RequestFailed( diff --git a/web/server/codechecker_server/cli/server.py b/web/server/codechecker_server/cli/server.py index 6076c71c24..ebb9328eb0 100644 --- a/web/server/codechecker_server/cli/server.py +++ b/web/server/codechecker_server/cli/server.py @@ -466,6 +466,7 @@ def check_product_db_status(cfg_sql_server, migration_root, environ): :returns: dictionary of product endpoints with database statuses """ + LOG.info("Connecting to product databases ...") engine = cfg_sql_server.create_engine() config_session = sessionmaker(bind=engine) sess = config_session() @@ -1023,9 +1024,9 @@ def server_init_start(args): None, force_upgrade) - prod_statuses = check_product_db_status(cfg_sql_server, - context.run_migration_root, - environ) + prod_statuses = check_product_db_status(cfg_sql_server, + context.run_migration_root, + environ) print_prod_status(prod_statuses) non_ok_db = False diff --git a/web/server/codechecker_server/permissions.py b/web/server/codechecker_server/permissions.py index 07df5cc0c3..f09ee74ef8 100644 --- a/web/server/codechecker_server/permissions.py +++ b/web/server/codechecker_server/permissions.py @@ -198,17 +198,17 @@ def remove_permission(self, auth_name, is_group=False, self._perm_name, 'group' if is_group else 'user', auth_name) - def has_permission(self, auth_session): + def has_permission(self, auth_session, is_auth_enabled): """ Returns whether or not the given authenticated user session (or None, if authentication is disabled on the server!) is given the current permission. """ if not auth_session: - # If the user does not have an auth_session it means it is a guest - # and the server is running in authentication disabled mode. - # All permissions are automatically granted in this case. - return True + # If the user does not have an auth_session it means it is a guest. + # All permissions are automatically granted when authentication is + # not enabled on the server. + return not is_auth_enabled elif auth_session.is_root and self._perm_name == 'SUPERUSER': # The special master superuser (root) automatically has the @@ -644,17 +644,19 @@ def initialise_defaults(scope, extra_params): handler._rem_perm_impl('*', False) -def require_permission(permission, extra_params, user): +def require_permission(permission, extra_params, user, is_auth_enabled): """ Returns whether or not the given user has the given permission. :param extra_params: The scope-specific argument dict, which already contains a valid database session. + :param is_auth_enabled: Guest users are handled differently when + authentication is disabled on the server. """ handler = handler_from_scope_params(permission, extra_params) - if handler.has_permission(user): + if handler.has_permission(user, is_auth_enabled): return True # If the user for some reason does not have the permission directly @@ -663,7 +665,7 @@ def require_permission(permission, extra_params, user): while ancestors: handler = handler_from_scope_params(ancestors[0], extra_params) - if handler.has_permission(user): + if handler.has_permission(user, is_auth_enabled): return True else: ancestors = ancestors[1:] + ancestors[0].inherited_from @@ -671,19 +673,21 @@ def require_permission(permission, extra_params, user): return False -def require_manager(permission, extra_params, user): +def require_manager(permission, extra_params, user, is_auth_enabled): """ Returns whether or not the given user has rights to manage the given permission. :param extra_params: The scope-specific argument dict, which already contains a valid database session. + :param is_auth_enabled: Guest users are handled differently when + authentication is disabled on the server. """ for manager in permission.managed_by: manager_handler = handler_from_scope_params(manager, extra_params) - if manager_handler.has_permission(user): + if manager_handler.has_permission(user, is_auth_enabled): return True return False diff --git a/web/server/codechecker_server/server.py b/web/server/codechecker_server/server.py index 2ff131f316..b06ef3a005 100644 --- a/web/server/codechecker_server/server.py +++ b/web/server/codechecker_server/server.py @@ -166,23 +166,29 @@ def __check_session_header(self): def __handle_readiness(self): """ Handle readiness probe. """ - with DBSession(self.server.config_session) as cfg_sess: - try: - cfg_sess.query(ORMConfiguration).count() - - self.send_response(200) - self.end_headers() - self.wfile.write(b'CODECHECKER_SERVER_IS_READY') - except Exception: - self.send_response(500) - self.end_headers() - self.wfile.write(b'CODECHECKER_SERVER_IS_NOT_READY') + try: + with DBSession(self.server.config_session) as cfg_sess: + try: + cfg_sess.query(ORMConfiguration).count() + + self.send_response(200) + self.end_headers() + self.wfile.write(b'CODECHECKER_SERVER_IS_READY') + except Exception: + self.send_response(500) + self.end_headers() + self.wfile.write(b'CODECHECKER_SERVER_IS_NOT_READY') + except BrokenPipeError: + pass def __handle_liveness(self): """ Handle liveness probe. """ - self.send_response(200) - self.end_headers() - self.wfile.write(b'CODECHECKER_SERVER_IS_LIVE') + try: + self.send_response(200) + self.end_headers() + self.wfile.write(b'CODECHECKER_SERVER_IS_LIVE') + except BrokenPipeError: + pass def end_headers(self): """ @@ -463,6 +469,7 @@ def do_POST(self): elif request_endpoint == "Tasks": task_handler = TaskHandler_v6( self.server.config_session, + self.server.manager, self.server.task_manager, self.auth_session) processor = TaskAPI_v6.Processor(task_handler) @@ -518,10 +525,11 @@ def do_POST(self): return except BrokenPipeError as ex: + # This is considered normal. The client can close the TCP + # connection for various reasons and the server cannot + # write to a closed socket. LOG.warning("%s failed with BrokenPipeError: %s", api_info, str(ex)) - import traceback - traceback.print_exc() except Exception as ex: if isinstance(ex, ProductNotFoundError): LOG.debug("%s failed with Exception: %s", api_info, str(ex)) diff --git a/web/server/vue-cli/config/webpack.common.js b/web/server/vue-cli/config/webpack.common.js index c2e09e6fdf..5c7e51afee 100644 --- a/web/server/vue-cli/config/webpack.common.js +++ b/web/server/vue-cli/config/webpack.common.js @@ -62,7 +62,8 @@ module.exports = { '@cc/report-server-types': join('codechecker-api', 'lib', 'report_server_types.js'), '@cc/shared-types': join('codechecker-api', 'lib', 'codechecker_api_shared_types.js'), 'thrift': join('thrift', 'lib', 'nodejs', 'lib', 'thrift', 'browser.js'), - 'Vuetify': join('vuetify', 'lib', 'components') + 'Vuetify': join('vuetify', 'lib', 'components'), + 'vue$': join('vue', 'dist', 'vue.esm.js') } }, module: { diff --git a/web/server/vue-cli/e2e/pages/filePathTree.js b/web/server/vue-cli/e2e/pages/filePathTree.js new file mode 100644 index 0000000000..86d05a9706 --- /dev/null +++ b/web/server/vue-cli/e2e/pages/filePathTree.js @@ -0,0 +1,67 @@ +// ------------------------------------------------------------------------- +// Part of the CodeChecker project, under the Apache License v2.0 with +// LLVM Exceptions. See LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// ------------------------------------------------------------------------- + +// Page object for the file-path filter's tree-based menu content +// (FilePathFilter.vue). Shares the same overall layout as the report page +// (login is performed via browser.page.login() in the spec), but only +// exposes the tree-specific elements/sections. + +const commands = { + waitForProgressBarNotPresent() { + this.pause(500, () => { + this.waitForElementNotPresent("@progressBar"); + }); + return this; + }, + + openFilePathFilterMenu() { + const filterSection = this.section.filePathFilter; + filterSection.click("@expansionBtn"); + filterSection.click("@settings"); + this.expect.section("@filePathTreeMenu").to.be.visible.before(5000); + return this; + } +}; + +module.exports = { + url: function() { + return this.api.launchUrl + "/e2e/reports?review-status=Unreviewed&" + + "review-status=Confirmed%20bug&detection-status=New&" + + "detection-status=Reopened&detection-status=Unresolved"; + }, + commands: [ commands ], + elements: { + page: ".v-data-table", + progressBar: ".v-data-table__progress" + }, + sections: { + filePathFilter: { + selector: "#filepath", + elements: { + expansionBtn: ".expansion-btn", + settings: ".settings-btn", + clearBtn: ".clear-btn", + selectedItems: ".selected-item" + } + }, + filePathTreeMenu: { + // The settings-menu popup that hosts the file-path tree. + selector: ".settings-menu.menuable__content__active", + elements: { + searchInput: "header input[type='text']", + anywhereSwitch: ".v-input--switch", + tree: ".file-path-tree", + treeNode: ".file-path-tree .v-treeview-node", + treeRootNode: ".file-path-tree > .v-treeview-node", + treeNodeRoot: ".file-path-tree .v-treeview-node__root", + treeItemLabel: ".file-path-tree .tree-item-label", + treeCheckbox: ".file-path-tree .v-treeview-node__checkbox", + applyBtn: ".apply-btn", + clearAllBtn: ".clear-all-btn" + } + } + } +}; diff --git a/web/server/vue-cli/e2e/pages/reportsTree.js b/web/server/vue-cli/e2e/pages/reportsTree.js new file mode 100644 index 0000000000..f3210f04f1 --- /dev/null +++ b/web/server/vue-cli/e2e/pages/reportsTree.js @@ -0,0 +1,76 @@ +// ------------------------------------------------------------------------- +// Part of the CodeChecker project, under the Apache License v2.0 with +// LLVM Exceptions. See LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// ------------------------------------------------------------------------- + +// Page object for the new view-mode toggle (Report List / File Tree) +// and the file-tree container in web/server/vue-cli/src/views/Reports.vue. + +const commands = { + waitForProgressBarNotPresent() { + this.pause(500, () => { + this.waitForElementNotPresent("@progressBar"); + }); + return this; + }, + + switchToTreeView() { + this + .click("@treeViewBtn") + .pause(500) + .waitForElementNotPresent("@progressBar"); + this.expect.element("@treeViewContainer").to.be.visible.before(5000); + return this; + }, + + switchToTableView() { + this + .click("@tableViewBtn") + .pause(500) + .waitForElementNotPresent("@progressBar"); + this.expect.element("@dataTable").to.be.visible.before(5000); + return this; + } +}; + +module.exports = { + url: function() { + return this.api.launchUrl + "/e2e/reports?review-status=Unreviewed&" + + "review-status=Confirmed%20bug&detection-status=New&" + + "detection-status=Reopened&detection-status=Unresolved"; + }, + commands: [ commands ], + elements: { + page: ".v-data-table", + progressBar: ".v-data-table__progress", + + // View-mode toggle (Vuetify v-btn-toggle with two v-btn entries). + viewModeToggle: ".v-btn-toggle", + tableViewBtn: ".v-btn-toggle .v-btn:nth-child(1)", + treeViewBtn: ".v-btn-toggle .v-btn:nth-child(2)", + + dataTable: ".v-data-table", + + treeViewContainer: ".tree-view-container", + treeHeader: ".tree-view-container .tree-header", + treeHeaderName: ".tree-view-container .tree-header-name", + treeHeaderCell: ".tree-view-container .tree-header-cell", + treeRow: ".tree-view-container .tree-row", + treeItemLabel: + ".tree-view-container .tree-item-label.clickable", + treeStatCell: ".tree-view-container .tree-stat-cell" + }, + sections: { + // Re-declare the file-path filter section so this spec can assert that + // clicking a tree item populates it. + filePathFilter: { + selector: "#filepath", + elements: { + expansionBtn: ".expansion-btn", + clearBtn: ".clear-btn", + selectedItems: ".selected-item" + } + } + } +}; diff --git a/web/server/vue-cli/e2e/specs/filePathTree.js b/web/server/vue-cli/e2e/specs/filePathTree.js new file mode 100644 index 0000000000..90d906d1b2 --- /dev/null +++ b/web/server/vue-cli/e2e/specs/filePathTree.js @@ -0,0 +1,184 @@ +// ------------------------------------------------------------------------- +// Part of the CodeChecker project, under the Apache License v2.0 with +// LLVM Exceptions. See LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// ------------------------------------------------------------------------- + +// E2E tests for the new tree view inside the File path filter +// (web/server/vue-cli/src/components/Report/ReportFilter/Filters/ +// FilePathFilter.vue). + +module.exports = { + before(browser) { + browser.resizeWindow(1600, 1000); + + const login = browser.page.login(); + const filePathTreePage = browser.page.filePathTree(); + + login + .navigate(filePathTreePage.url()) + .loginAsRoot(); + + browser.assert.urlContains("/e2e/reports"); + + filePathTreePage + .waitForElementVisible("@page", 10000) + .waitForProgressBarNotPresent(); + }, + + after(browser) { + browser.perform(() => { + browser.end(); + }); + }, + + "open the file path filter tree menu" (browser) { + const page = browser.page.filePathTree(); + page.openFilePathFilterMenu(); + + page.expect.section("@filePathTreeMenu").to.be.visible.before(5000); + + const menu = page.section.filePathTreeMenu; + // The tree should render at least one node when reports exist. + menu.expect.element("@tree").to.be.present.before(5000); + menu.api.elements("@treeRootNode", ({ result }) => { + browser.assert.ok( + Array.isArray(result.value) && result.value.length > 0, + "file-path tree should contain at least one root node" + ); + }); + }, + + "filter the tree using the search input" (browser) { + const page = browser.page.filePathTree(); + const menu = page.section.filePathTreeMenu; + + // Filter for everything (matches all files). + menu.clearAndSetValue("@searchInput", "*", menu); + menu + .pause(500) + .api.elements("@treeRootNode", ({ result }) => { + browser.assert.ok( + Array.isArray(result.value) && result.value.length > 0, + "wildcard filter should keep at least one root node visible" + ); + }); + + // Filter for a string that should match nothing. + menu.clearAndSetValue("@searchInput", + "definitely_not_a_real_path_zzz", menu); + menu + .pause(500) + .api.elements("@treeRootNode", ({ result }) => { + browser.assert.ok( + Array.isArray(result.value) && result.value.length === 0, + "non-matching filter should hide every node" + ); + }); + + // Reset filter for subsequent tests. + menu.clearAndSetValue("@searchInput", "", menu); + menu.pause(300); + }, + + "toggle the anywhere-on-report-path switch" (browser) { + const page = browser.page.filePathTree(); + const menu = page.section.filePathTreeMenu; + + menu.expect.element("@anywhereSwitch").to.be.visible.before(5000); + menu + .click("@anywhereSwitch") + .pause(300) + .click("@anywhereSwitch") + .pause(300); + // No assertion on tree contents (data-dependent); we only verify the + // switch is interactive and does not throw. + }, + + "expand the first folder node" (browser) { + const page = browser.page.filePathTree(); + const menu = page.section.filePathTreeMenu; + + // Click the first node label area to toggle expansion (open-on-click). + menu + .click({ selector: "@treeNodeRoot", index: 0 }) + .pause(500); + + // After expansion, the total visible node count must not decrease. + menu.api.elements("@treeNode", ({ result }) => { + browser.assert.ok( + Array.isArray(result.value) && result.value.length >= 1, + "expanding a folder should reveal at least the original nodes" + ); + }); + }, + + "select a tree item and apply" (browser) { + const page = browser.page.filePathTree(); + const menu = page.section.filePathTreeMenu; + + menu.expect.element("@treeCheckbox").to.be.present.before(5000); + + // Tick the first checkbox (independent selection mode means a single + // node is enough to enable the Apply button). + menu + .click({ selector: "@treeCheckbox", index: 0 }) + .pause(300); + + menu.click("@applyBtn"); + + // Menu must close after applying. + page.expect.section("@filePathTreeMenu") + .to.not.be.present.before(5000); + + page.waitForProgressBarNotPresent(); + + page.section.filePathFilter.api.elements("@selectedItems", + ({ result }) => { + browser.assert.ok( + Array.isArray(result.value) && result.value.length >= 1, + "applying a tree selection should add at least one filter chip" + ); + }); + }, + + "clear the file path filter via the section clear button" (browser) { + const page = browser.page.filePathTree(); + const section = page.section.filePathFilter; + + section.click("@clearBtn"); + page.waitForProgressBarNotPresent(); + + section.api.elements("@selectedItems", ({ result }) => { + browser.assert.ok( + Array.isArray(result.value) && result.value.length === 0, + "clear button should remove every file-path filter chip" + ); + }); + }, + + "clear-all button inside the tree menu closes and clears" (browser) { + const page = browser.page.filePathTree(); + page.openFilePathFilterMenu(); + + const menu = page.section.filePathTreeMenu; + + menu + .click({ selector: "@treeCheckbox", index: 0 }) + .pause(300) + .click("@clearAllBtn"); + + page.expect.section("@filePathTreeMenu") + .to.not.be.present.before(5000); + + page.waitForProgressBarNotPresent(); + + page.section.filePathFilter.api.elements("@selectedItems", + ({ result }) => { + browser.assert.ok( + Array.isArray(result.value) && result.value.length === 0, + "clear-all should leave no file-path filter chips" + ); + }); + } +}; diff --git a/web/server/vue-cli/e2e/specs/reportsTreeView.js b/web/server/vue-cli/e2e/specs/reportsTreeView.js new file mode 100644 index 0000000000..04cc4d00ae --- /dev/null +++ b/web/server/vue-cli/e2e/specs/reportsTreeView.js @@ -0,0 +1,122 @@ +// ------------------------------------------------------------------------- +// Part of the CodeChecker project, under the Apache License v2.0 with +// LLVM Exceptions. See LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// ------------------------------------------------------------------------- + +// E2E tests for the new file-tree view mode and the severity / review-status +// header columns added to web/server/vue-cli/src/views/Reports.vue. + +module.exports = { + before(browser) { + browser.resizeWindow(1600, 1000); + + const login = browser.page.login(); + const treePage = browser.page.reportsTree(); + + login + .navigate(treePage.url()) + .loginAsRoot(); + + browser.assert.urlContains("/e2e/reports"); + + treePage + .waitForElementVisible("@page", 10000) + .waitForProgressBarNotPresent(); + + // Make sure the file-path filter section is expanded so that we can + // inspect the chips later. + const reportPage = browser.page.report(); + reportPage.section.filePathFilter.click("@expansionBtn"); + }, + + after(browser) { + browser.perform(() => { + browser.end(); + }); + }, + + "default view is the data table" (browser) { + const page = browser.page.reportsTree(); + page.expect.element("@viewModeToggle").to.be.visible.before(5000); + page.expect.element("@dataTable").to.be.visible.before(5000); + page.expect.element("@treeViewContainer").to.not.be.present.before(5000); + }, + + "switching to tree view shows the tree container" (browser) { + const page = browser.page.reportsTree(); + page.switchToTreeView(); + + page.expect.element("@treeViewContainer").to.be.visible.before(5000); + page.expect.element("@dataTable").to.not.be.present.before(5000); + }, + + "tree header shows severity and review-status columns" (browser) { + const page = browser.page.reportsTree(); + + page.expect.element("@treeHeader").to.be.visible.before(5000); + page.expect.element("@treeHeaderName").text.to.contain("Name"); + + // The header has 1 "All" + 5 severities + 4 review statuses = 10 + // tree-header-cell elements. + browser.elements("css selector", ".tree-header-cell", function (result) { + browser.assert.ok( + Array.isArray(result.value) && result.value.length === 10, + "tree header should contain 10 stat columns " + + "(All + 5 severities + 4 review statuses), got " + + (result.value ? result.value.length : "none") + ); + }); + }, + + "tree rows render stat cells" (browser) { + const page = browser.page.reportsTree(); + + page.expect.element("@treeRow").to.be.present.before(5000); + browser.elements("css selector", + ".tree-view-container .tree-row", function (result) { + browser.assert.ok( + Array.isArray(result.value) && result.value.length > 0, + "the tree should render at least one row" + ); + }); + }, + + "clicking a tree item populates the file path filter" (browser) { + const page = browser.page.reportsTree(); + + page.expect.element("@treeItemLabel").to.be.present.before(5000); + + page + .click({ selector: "@treeItemLabel", index: 0 }) + .pause(500) + .waitForElementNotPresent("@progressBar"); + + page.section.filePathFilter.api.elements("@selectedItems", + ({ result }) => { + browser.assert.ok( + Array.isArray(result.value) && result.value.length >= 1, + "clicking a tree node must add at least one file-path filter chip" + ); + }); + }, + + "switching back to table view clears the file path filter" (browser) { + const page = browser.page.reportsTree(); + + page.switchToTableView(); + + // Both view-toggle buttons in Reports.vue invoke + // setReportFilter({ filepath: null }) on click, so the chip set + // populated in the previous test must now be empty. + page.section.filePathFilter.api.elements("@selectedItems", + ({ result }) => { + browser.assert.ok( + Array.isArray(result.value) && result.value.length === 0, + "switching back to the table view should clear file-path filter" + ); + }); + + page.expect.element("@dataTable").to.be.visible.before(5000); + } +}; diff --git a/web/server/vue-cli/package-lock.json b/web/server/vue-cli/package-lock.json index 7f9266f546..160a54d7e1 100644 --- a/web/server/vue-cli/package-lock.json +++ b/web/server/vue-cli/package-lock.json @@ -11,7 +11,7 @@ "@mdi/font": "^6.5.95", "chart.js": "^2.9.4", "chartjs-plugin-datalabels": "^0.7.0", - "codechecker-api": "file:../../api/js/codechecker-api-node/dist/codechecker-api-6.69.0.tgz", + "codechecker-api": "file:../../api/js/codechecker-api-node/dist/codechecker-api-6.70.0.tgz", "codemirror": "^5.65.0", "date-fns": "^2.28.0", "js-cookie": "^3.0.1", @@ -5059,9 +5059,9 @@ } }, "node_modules/codechecker-api": { - "version": "6.69.0", - "resolved": "file:../../api/js/codechecker-api-node/dist/codechecker-api-6.69.0.tgz", - "integrity": "sha512-+vw8RNIzxsjfAzx+YlqEB9UNuxiFav+/d5gUaALvNel6wk2jrWF2thh+xcrTTVqu/4jvZXSE/uZwOJBmw5S4og==", + "version": "6.70.0", + "resolved": "file:../../api/js/codechecker-api-node/dist/codechecker-api-6.70.0.tgz", + "integrity": "sha512-YMEkiEJFWbsufbqDGIlfhkcfK4MdJvXDXcEVyEKzfPMSPcBVM4c5yJ5RbDuxT2eIsMzGS6sE2pVE8YzEBJH27w==", "license": "SEE LICENSE IN LICENSE", "dependencies": { "thrift": "0.13.0-hotfix.1" diff --git a/web/server/vue-cli/package.json b/web/server/vue-cli/package.json index 58a7fa3253..b2c1a426bb 100644 --- a/web/server/vue-cli/package.json +++ b/web/server/vue-cli/package.json @@ -29,7 +29,7 @@ "@mdi/font": "^6.5.95", "chart.js": "^2.9.4", "chartjs-plugin-datalabels": "^0.7.0", - "codechecker-api": "file:../../api/js/codechecker-api-node/dist/codechecker-api-6.69.0.tgz", + "codechecker-api": "file:../../api/js/codechecker-api-node/dist/codechecker-api-6.70.0.tgz", "codemirror": "^5.65.0", "date-fns": "^2.28.0", "js-cookie": "^3.0.1", diff --git a/web/server/vue-cli/src/components/Report/ReportFilter/Filters/FilePathFilter.vue b/web/server/vue-cli/src/components/Report/ReportFilter/Filters/FilePathFilter.vue index ccee7da0b0..74059dd5f3 100644 --- a/web/server/vue-cli/src/components/Report/ReportFilter/Filters/FilePathFilter.vue +++ b/web/server/vue-cli/src/components/Report/ReportFilter/Filters/FilePathFilter.vue @@ -5,15 +5,90 @@ :bus="bus" :fetch-items="fetchItems" :selected-items="selectedItems" - :search="search" :loading="loading" :limit="defaultLimit" :panel="panel" @clear="clear(true)" @input="setSelectedItems" > -