Skip to content

Share checker configurations across analysis runs via CheckerSet #4909

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
barnabasdomozi wants to merge 1 commit into
base: master
Choose a base branch
from
+373 −62
Open

Share checker configurations across analysis runs via CheckerSet #4909

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 63 additions & 35 deletions web/server/codechecker_server/api/mass_store_run.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,9 @@
from ..database.config_db_model import Product
from ..database.database import DBSession
from ..database.run_db_model import \
AnalysisInfo, AnalysisInfoChecker, AnalyzerStatistic, \
AnalysisInfo, AnalyzerStatistic, \
BugPathEvent, BugReportPoint, \
Checker, \
Checker, CheckerSet, CheckerSetItem, \
ExtendedReportData, \
File, FileContent, \
Report as DBReport, ReportAnnotations, ReviewStatus as ReviewStatusRule, \
Expand All @@ -63,6 +63,8 @@
from ..task_executors.task_manager import TaskManager
from .thrift_enum_helper import report_extended_data_type_str

from sqlalchemy.orm import Session as SA_Session


LOG = get_logger('server')

Expand Down Expand Up @@ -1002,7 +1004,7 @@ def __store_analysis_statistics(

def __store_analysis_info(
self,
session: DBSession,
session: SA_Session,
run_history: RunHistory
):
""" Store analysis info for the given run history. """
Expand All @@ -1012,38 +1014,64 @@ def __store_analysis_info(
analyzer_command.encode("utf-8"),
zlib.Z_BEST_COMPRESSION)

analysis_info_rows = session \
.query(AnalysisInfo) \
.filter(AnalysisInfo.analyzer_command == cmd) \
.all()

if analysis_info_rows:
# It is possible when multiple runs are stored
# simultaneously to the server with the same analysis
# command that multiple entries are stored into the
# database. In this case we will select the first one.
analysis_info = analysis_info_rows[0]
else:
analysis_info = AnalysisInfo(analyzer_command=cmd)

# Obtain the ID eagerly to be able to use the M-to-N table.
session.add(analysis_info)
session.flush()
session.refresh(analysis_info, ["id"])

for analyzer in mip.analyzers:
q = session \
.query(Checker) \
.filter(Checker.analyzer_name == analyzer)
db_checkers = {r.checker_name: r for r in q.all()}

connection_rows = [AnalysisInfoChecker(
analysis_info, db_checkers[chk], is_enabled)
for chk, is_enabled
in mip.checkers.get(analyzer, {}).items()]
for r in connection_rows:
session.add(r)

enabled_checkers: List[int] = []
disabled_checkers: List[int] = []
for analyzer in mip.analyzers:
q = session \
.query(Checker) \
.filter(Checker.analyzer_name == analyzer)
db_checkers = {r.checker_name: r for r in q.all()}

for chk, is_enabled in \
mip.checkers.get(analyzer, {}).items():
if is_enabled:
enabled_checkers.append(db_checkers[chk].id)
else:
disabled_checkers.append(db_checkers[chk].id)

# Check if the CheckerSet was already used before.
checker_set_hash = CheckerSet.compute_hash(enabled_checkers,
disabled_checkers)
checker_set = session.query(CheckerSet) \
.filter(CheckerSet.hash_digest == checker_set_hash) \
.first()

# If not, create a new CheckerSet and insert to db
if not checker_set:
try:
with session.begin_nested():
checker_set = CheckerSet(checker_set_hash)
# Obtain CheckerSet id eagerly.
session.add(checker_set)
session.flush()
session.refresh(checker_set, ["id"])
LOG.info(
"[%s] Created new CheckerSet with hash '%s'",
self._name, checker_set_hash)

# Insert checkers as elements of this newly
# created CheckerSet
for e in enabled_checkers:
session.add(CheckerSetItem(checker_set.id,
e, True))
for d in disabled_checkers:
session.add(CheckerSetItem(checker_set.id,
d, False))
except Exception:
# Meanwhile, another store operation already
# inserted this CheckerSet, query the existing
# one from db
checker_set = session.query(CheckerSet) \
.filter(
CheckerSet.hash_digest == checker_set_hash) \
.first()

if not checker_set:
raise RuntimeError(
"Failed to query CheckerSet from database!")

analysis_info = AnalysisInfo(analyzer_command=cmd,
checker_set_id=checker_set.id)
run_history.analysis_info.append(analysis_info)
self.__analysis_info[src_dir_path] = analysis_info

Expand Down
22 changes: 11 additions & 11 deletions web/server/codechecker_server/api/report_server.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,10 @@
from ..database.config_db_model import Product
from ..database.database import conv, DBSession, escape_like
from ..database.run_db_model import \
AnalysisInfo, AnalysisInfoChecker as DB_AnalysisInfoChecker, \
AnalysisInfo, \
AnalyzerStatistic, \
BugPathEvent, BugReportPoint, \
CleanupPlan, CleanupPlanReportHash, Checker, Comment, \
CleanupPlan, CleanupPlanReportHash, Checker, CheckerSetItem, Comment, \
ExtendedReportData, \
File, FileContent, \
Report, ReportAnnotations, ReportAnalysisInfo, ReviewStatus, \
Expand Down Expand Up @@ -1949,11 +1949,11 @@ def getAnalysisInfo(self, analysis_info_filter, limit, offset):
checkers_q = session \
.query(Checker.analyzer_name,
Checker.checker_name,
DB_AnalysisInfoChecker.enabled) \
.join(Checker, DB_AnalysisInfoChecker.checker_id ==
CheckerSetItem.enabled) \
.join(Checker, CheckerSetItem.checker_id ==
Checker.id) \
.filter(DB_AnalysisInfoChecker.
analysis_info_id == cmd.id)
.filter(CheckerSetItem.checker_set_id ==
cmd.checker_set_id)

checkers: Dict[str, Dict[str, API_AnalysisInfoChecker]] = \
defaultdict(dict)
Expand Down Expand Up @@ -3383,12 +3383,12 @@ def getCheckerStatusVerificationDetails(self, run_ids, report_filter):
)
.join(RunHistory)
.join(AnalysisInfo, RunHistory.analysis_info)
.join(DB_AnalysisInfoChecker, (
(AnalysisInfo.id ==
DB_AnalysisInfoChecker.analysis_info_id)
& (DB_AnalysisInfoChecker.enabled.is_(True))))
.join(CheckerSetItem, (
(AnalysisInfo.checker_set_id ==
CheckerSetItem.checker_set_id)
& (CheckerSetItem.enabled.is_(True))))
.join(Checker,
DB_AnalysisInfoChecker.checker_id == Checker.id)
CheckerSetItem.checker_id == Checker.id)
.outerjoin(Report, ((Checker.id == Report.checker_id)
& (Run.id == Report.run_id)))
.filter(RunHistory.id == max_run_histories.subquery()
Expand Down
68 changes: 52 additions & 16 deletions web/server/codechecker_server/database/run_db_model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@
from datetime import datetime, timedelta
from math import ceil
import os
from typing import Optional
import json
import hashlib
from typing import Optional, List

from sqlalchemy import Boolean, Column, DateTime, Enum, ForeignKey, Integer, \
LargeBinary, MetaData, String, UniqueConstraint, Table, Text, JSON
Expand Down Expand Up @@ -53,29 +55,58 @@ def __init__(self, analyzer_name: str, checker_name: str, severity: int):
self.severity = severity


class AnalysisInfoChecker(Base):
__tablename__ = "analysis_info_checkers"
class CheckerSet(Base):
__tablename__ = "checker_set"

analysis_info_id = Column(Integer,
ForeignKey("analysis_info.id",
deferrable=True,
initially="DEFERRED",
ondelete="CASCADE"),
primary_key=True)
id = Column(Integer, autoincrement=True, primary_key=True)
hash_digest = Column(String, unique=True, nullable=False)

def __init__(self, hash_digest: str):
self.hash_digest = hash_digest

# We compute a hash from the enabled_checkers and
# disabled_checkers lists to generate a unique identifier
# for each CheckerSet.
# The goal is to speed up report storage to the server:
# when a user stores results to the server, we first compute
# this hash and then check if it was already inserted to the database.
# If the hash exists in this table, that means the particular CheckerSet
# was already used before.
@staticmethod
def compute_hash(enabled_checkers: List[int],
disabled_checkers: List[int]) -> str:
# Sort lists to create identical hashes.
enabled_checkers.sort()
disabled_checkers.sort()

checker_set_dict = {"e": enabled_checkers, "d": disabled_checkers}
return hashlib.sha256(
json.dumps(checker_set_dict).encode()).hexdigest()


class CheckerSetItem(Base):
__tablename__ = "checker_set_items"

checker_set_id = Column(Integer,
ForeignKey("checker_set.id",
deferrable=True,
initially="DEFERRED",
ondelete="CASCADE"),
primary_key=True)
checker_id = Column(Integer,
ForeignKey("checkers.id",
deferrable=True,
initially="DEFERRED",
ondelete="RESTRICT"),
primary_key=True)
enabled = Column(Boolean)
enabled = Column(Boolean, nullable=False)

def __init__(self,
analysis_info: "AnalysisInfo",
checker: Checker,
checker_set_id: int,
checker_id: int,
is_enabled: bool):
self.analysis_info_id = analysis_info.id
self.checker_id = checker.id
self.checker_set_id = checker_set_id
self.checker_id = checker_id
self.enabled = is_enabled


Expand All @@ -84,10 +115,15 @@ class AnalysisInfo(Base):

id = Column(Integer, autoincrement=True, primary_key=True)
analyzer_command = Column(LargeBinary)
available_checkers = relationship(AnalysisInfoChecker, uselist=True)
checker_set_id = Column(Integer,
ForeignKey("checker_set.id",
deferrable=True,
initially="DEFERRED",
ondelete="CASCADE"))

def __init__(self, analyzer_command: bytes):
def __init__(self, analyzer_command: bytes, checker_set_id: int):
self.analyzer_command = analyzer_command
self.checker_set_id = checker_set_id


class Run(Base):
Expand Down
Loading
Loading