diff --git a/api_app/chatbot_manager/agent/system_prompt.txt b/api_app/chatbot_manager/agent/system_prompt.txt index 2996d940c7..b626cbe017 100644 --- a/api_app/chatbot_manager/agent/system_prompt.txt +++ b/api_app/chatbot_manager/agent/system_prompt.txt @@ -5,7 +5,7 @@ Answer with short, data-driven summaries. Always cite the tools you used at the [Tools — when to use each] - search_jobs: find or list the current user's own jobs — by observable value, tag, or md5, or with no filter to list their recent jobs. Use for any question about the user's jobs: "show me my jobs", "list my jobs", "what jobs do I have?", "do I have any jobs?", "find my recent jobs", "find jobs for domain X". - get_job_details: full detail of ONE job by id. Use after search_jobs, or when the user asks "details of job #N". -- summarize_job: high-level summary of ONE job (status, analyzers, findings). Use for "summarize job #N". +- summarize_job: summary AND verdict of ONE job — status, analyzers, and whether the observable is malicious, suspicious, clean or trusted, with the analyzer evidence. Use for "summarize job #N", "evaluate the results of job #N", "is job #N malicious?". - list_investigations: browse investigations with optional status/name filters. Use for "show my investigations", "what investigations are open?". - get_investigation_tree: full tree of ONE investigation by id with all its jobs. Use after list_investigations. - summarize_investigation: high-level summary of ONE investigation (status, job breakdown). Use for "summarize investigation #N". @@ -26,3 +26,4 @@ Answer with short, data-driven summaries. Always cite the tools you used at the [Response style] - One paragraph unless the user asks for a list or breakdown. - Mention which tools you called at the end: "(used: search_jobs, summarize_job)". +- With a verdict: always quote its headline verbatim first, even for a yes/no question, then name the supporting analyzers and the silent count. A disagreeing analyzer is contradicting, not silent. Never invent a score. diff --git a/api_app/chatbot_manager/agent/tools/summarize_job.py b/api_app/chatbot_manager/agent/tools/summarize_job.py index 444a9bd762..9946b4cb13 100644 --- a/api_app/chatbot_manager/agent/tools/summarize_job.py +++ b/api_app/chatbot_manager/agent/tools/summarize_job.py @@ -3,27 +3,35 @@ from langchain_core.tools import tool +from api_app.chatbot_manager.evaluation import evaluate_job +from api_app.chatbot_manager.serializers.job import SummarizeJobResultSerializer from api_app.choices import ReportStatus +from api_app.models import Job def make_summarize_job_tool(user): # Built per-request and closed over `user`: the lookup is scoped with visible_for_user # (owner + same-org AMBER/RED + globally-visible CLEAR/GREEN), matching the REST - # JobViewSet / UI (multi-tenancy enforced here). The payload is human-readable prose - # (meant to be relayed to the user) wrapped in the same envelope as the other tools. + # JobViewSet / UI (multi-tenancy enforced here). The payload pairs human-readable prose + # (meant to be relayed to the user) with the structured verdict, in the same envelope as + # the other tools. @tool("summarize_job") def summarize_job(job_id: int) -> str: - """Return a concise human-readable summary of an IntelOwl job. + """Summarize an IntelOwl job AND report IntelOwl's verdict on the observable. + + The verdict says whether the observable is malicious, suspicious, clean, trusted, or has + no evaluation, and lists the analyzers supporting or contradicting it. It is IntelOwl's + own reconciled evaluation — the same one shown on the job page — not an opinion of yours. Args: job_id: The numeric ID of the job to summarize. Returns: - JSON string with shape {"errors": [...], "summary": "..." | null}. + JSON string with shape + {"errors": [...], "summary": "..." | null, "verdict": {...} | null}, where `verdict` + carries `headline` (relay it as-is), `bucket`, `reliability`, `supporting`, + `contradicting` and the analyzers that had no opinion. """ - from api_app.chatbot_manager.serializers.job import SummarizeJobResultSerializer - from api_app.models import Job - try: job = ( Job.objects.select_related("analyzable") @@ -33,7 +41,11 @@ def summarize_job(job_id: int) -> str: ) except Job.DoesNotExist: return SummarizeJobResultSerializer( - {"errors": [f"Job with ID {job_id} not found or not accessible."], "summary": None} + { + "errors": [f"Job with ID {job_id} not found or not accessible."], + "summary": None, + "verdict": None, + } ).to_json() analyzers = list(job.analyzers_to_execute.values_list("name", flat=True)) @@ -43,8 +55,15 @@ def summarize_job(job_id: int) -> str: r.config.name for r in job.analyzerreports.all() if r.status != ReportStatus.SUCCESS.value ] + verdict = evaluate_job(job) lines = [ f"Job #{job.pk}", + # The headline is duplicated here on purpose. It is also carried structurally in + # `verdict`, but a live smoke against qwen2.5:3b showed the model reproduces the prose + # fields of this summary verbatim while paraphrasing the structured object — dropping + # the reliability and confusing contradicting analyzers with silent ones. Stating the + # copy-ready sentence in the prose is what makes the narration match the badge. + f" Verdict : {verdict.headline}", f" Observable : {job.analyzable.name} ({job.analyzable.classification})", f" MD5 : {job.analyzable.md5}", f" Status : {job.status}", @@ -58,6 +77,10 @@ def summarize_job(job_id: int) -> str: if failed_reports: lines.append(f" Failed : {', '.join(failed_reports)}") - return SummarizeJobResultSerializer({"errors": [], "summary": "\n".join(lines)}).to_json() + # The verdict is ALSO kept structured, so the model can relay exact analyzer names and + # numbers instead of paraphrasing them; only the headline is echoed into the prose above. + return SummarizeJobResultSerializer( + {"errors": [], "summary": "\n".join(lines), "verdict": verdict} + ).to_json() return summarize_job diff --git a/api_app/chatbot_manager/evaluation.py b/api_app/chatbot_manager/evaluation.py new file mode 100644 index 0000000000..28cc7c2924 --- /dev/null +++ b/api_app/chatbot_manager/evaluation.py @@ -0,0 +1,199 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +"""Objective reading of a job's verdict for the chatbot. + +Nothing here computes a verdict: the headline is IntelOwl's own reconciled evaluation +(`EvaluationEngineModule`) mapped through the shared `classify()`, so the chatbot says exactly +the word the job-page badge shows. This module only reads it, attributes it to the analyzers +that produced it, and reports honestly what did not answer. The LLM is never involved. + +Tenancy: no `Job` query happens here — the caller passes a job already resolved through +`visible_for_user`, so the tool keeps the single tenancy boundary it already had. +""" + +from dataclasses import dataclass, field + +from api_app.choices import Classification +from api_app.data_model_manager.classify import classify +from api_app.data_model_manager.enums import DataModelVerdictBuckets +from api_app.engines_manager.engines.evaluation import EvaluationEngineModule +from api_app.models import Job + +# The evidence lists are fed to a 3B model with an 8k context window: a job can carry 100+ +# analyzer reports, so the lists are capped and the full size is reported as a count instead. +MAX_EVIDENCE_ANALYZERS = 10 +MAX_SILENT_ANALYZERS = 10 + +# Honest reasons for the absence of a verdict. The reader never fabricates one. +REASON_GENERIC = "IntelOwl does not evaluate generic observables." +REASON_NO_EVALUATION = "No analyzer produced an evaluation for this observable." + + +@dataclass +class AnalyzerVerdict: + """One analyzer's own evaluation, as stored in the DataModel its report produced.""" + + name: str + evaluation: str + reliability: int + + +@dataclass +class JobEvaluation: + """A job's reconciled verdict plus the evidence behind it. + + `bucket` is one of the five presentation buckets shared with the visualizer badge; + `supporting` / `contradicting` / `silent` partition the analyzers that ran, so "we don't + know" is always attributable to named analyzers instead of being an opaque shrug. + """ + + bucket: str + evaluation: str | None + reliability: int + headline: str + analyst_override: bool = False + reason: str | None = None + supporting: list[AnalyzerVerdict] = field(default_factory=list) + contradicting: list[AnalyzerVerdict] = field(default_factory=list) + silent: list[str] = field(default_factory=list) + silent_count: int = 0 + analyzers_considered: int = 0 + + +def _format_headline( + bucket: str, + reliability: int, + supporting_count: int, + contradicting_count: int, + silent_count: int, + analyzers_considered: int, + analyst_override: bool, + reason: str | None, +) -> str: + """Build the one sentence the model is expected to relay verbatim. + + A copy-ready string keeps a small model from paraphrasing the numbers into something the + badge does not say (the same failure class as the placeholder analyzer names). + + All three counts are stated explicitly, and the silent one even when it is zero. An earlier + version reported only "N of M analyzers support it": a live smoke showed the model then + inferred the remaining M-N analyzers were all contradicting and asserted there were no silent + ones, which is the opposite of the honest-absence reporting this module exists to provide. + """ + if reason: + return f"{bucket} — {reason}" + if analyst_override: + # A manual analyst event outranks the analyzers in the engine's reconciliation, so the + # analyzers are reported as agreeing/disagreeing, never as the source of the verdict — + # crediting them would misattribute where it came from. The counts are still stated, + # because omitting them is what made the model invent "no silent analyzers". + return ( + f"{bucket} (reliability {reliability}/10) — set by an analyst decision on this " + f"observable; of {analyzers_considered} analyzers that ran, {supporting_count} agree, " + f"{contradicting_count} disagree, {silent_count} silent" + ) + # Count-neutral wording ("1 supporting", not "1 support it"), reusing the same three words as + # the payload keys and the prompt's narration rule so the model has one vocabulary, not two. + return ( + f"{bucket} (reliability {reliability}/10) — {analyzers_considered} analyzers ran: " + f"{supporting_count} supporting, {contradicting_count} contradicting, {silent_count} silent" + ) + + +def _partition_analyzers( + job: Job, evaluation: str | None +) -> tuple[list[AnalyzerVerdict], list[AnalyzerVerdict], list[str]]: + """Split the analyzers that ran into supporting / contradicting / silent. + + Attribution goes through `data_model_object_id` rather than the report's `data_model` + GenericForeignKey: the FK resolves lazily, one query per report, while the whole set is + already available in the single queryset `get_analyzers_data_models()` runs. Evidence is + ordered by reliability so the caps in `evaluate_job` keep the strongest evidence. + """ + data_models_by_pk = {data_model.pk: data_model for data_model in job.get_analyzers_data_models()} + supporting, contradicting, silent = [], [], [] + for report in job.analyzerreports.all(): + data_model = data_models_by_pk.get(report.data_model_object_id) + if data_model is None or not data_model.evaluation: + # Ran but expressed no opinion: a blocklist miss, a timeout, or a missing API key. + silent.append(report.config.name) + continue + verdict = AnalyzerVerdict( + name=report.config.name, + evaluation=data_model.evaluation, + reliability=data_model.reliability, + ) + if data_model.evaluation == evaluation: + supporting.append(verdict) + else: + contradicting.append(verdict) + supporting.sort(key=lambda item: item.reliability, reverse=True) + contradicting.sort(key=lambda item: item.reliability, reverse=True) + return supporting, contradicting, silent + + +def evaluate_job(job: Job) -> JobEvaluation: + """Read the reconciled verdict of `job` and the per-analyzer evidence behind it. + + The headline is recomputed live with the platform's own `EvaluationEngineModule` instead of + reading `job.data_model`: the engine modules run asynchronously *after* the pipeline saves a + transient, un-reconciled merge (`engines_manager/models.py`), so the stored scalar can be + wrong for a window. `EvaluationEngineModule.run` is a pure read of the same two sources, so + recomputing removes that race by construction and guarantees the chatbot cannot diverge from + the badge. + + Pure function: no writes, and no `Job` lookup — `job` must already be scoped to the + requesting user by the caller. + """ + if job.analyzable.classification == Classification.GENERIC.value: + # The engine skips generic observables entirely and no DataModel class exists for them, + # so `get_analyzers_data_models()` would raise NotImplementedError here. + no_evaluation = DataModelVerdictBuckets.NO_EVALUATION.value + return JobEvaluation( + bucket=no_evaluation, + evaluation=None, + reliability=0, + headline=f"{no_evaluation} — {REASON_GENERIC}", + reason=REASON_GENERIC, + ) + + headline = EvaluationEngineModule(job).run() or {} + evaluation = headline.get("evaluation") + # The engine averages reliability into a float and then stores it through an integer column: + # `merge()` assigns it and saves, and Django's IntegerField.get_prep_value does `int(value)`, + # which TRUNCATES. Truncating here too is what makes the chatbot say the same word as the + # badge — rounding would turn a stored 5 (Avg 5.5 -> suspicious) into 6 (malicious). + reliability = int(headline.get("reliability") or 0) + bucket = classify(evaluation, reliability) + # Reported as a flag only. The engine resolves user events with the job owner's visibility + # (`Job.get_user_events_data_model`), so surfacing any detail of the event itself — author, + # reason, tags — could expose data the *requesting* user cannot see. The flag adds nothing + # beyond the verdict value, which is already public to anyone who can see the job. + analyst_override = job.get_user_events_data_model().exists() + + supporting, contradicting, silent = _partition_analyzers(job, evaluation) + reason = REASON_NO_EVALUATION if evaluation is None else None + analyzers_considered = len(supporting) + len(contradicting) + len(silent) + return JobEvaluation( + bucket=bucket, + evaluation=evaluation, + reliability=reliability, + headline=_format_headline( + bucket, + reliability, + len(supporting), + len(contradicting), + len(silent), + analyzers_considered, + analyst_override, + reason, + ), + analyst_override=analyst_override, + reason=reason, + supporting=supporting[:MAX_EVIDENCE_ANALYZERS], + contradicting=contradicting[:MAX_EVIDENCE_ANALYZERS], + silent=silent[:MAX_SILENT_ANALYZERS], + silent_count=len(silent), + analyzers_considered=analyzers_considered, + ) diff --git a/api_app/chatbot_manager/serializers/job.py b/api_app/chatbot_manager/serializers/job.py index 65a2c2342f..5a975511f4 100644 --- a/api_app/chatbot_manager/serializers/job.py +++ b/api_app/chatbot_manager/serializers/job.py @@ -74,5 +74,35 @@ class JobDetailResultSerializer(ToolResultSerializer): job = JobDetailToolSerializer(allow_null=True) +class AnalyzerVerdictSerializer(serializers.Serializer): + """One analyzer's own evaluation, read from the DataModel its report produced.""" + + name = serializers.CharField(read_only=True) + evaluation = serializers.CharField(read_only=True) + reliability = serializers.IntegerField(read_only=True) + + +class JobVerdictSerializer(serializers.Serializer): + """IntelOwl's reconciled verdict on a job plus the evidence behind it. + + Mirrors the `JobEvaluation` dataclass (`chatbot_manager/evaluation.py`). `bucket` is the same + word the DataModel visualizer badge shows; `reason` is set only when there is no verdict, so + the model can explain the absence instead of inventing one. + """ + + bucket = serializers.CharField(read_only=True) + evaluation = serializers.CharField(read_only=True, allow_null=True) + reliability = serializers.IntegerField(read_only=True) + headline = serializers.CharField(read_only=True) + analyst_override = serializers.BooleanField(read_only=True) + reason = serializers.CharField(read_only=True, allow_null=True) + supporting = AnalyzerVerdictSerializer(many=True, read_only=True) + contradicting = AnalyzerVerdictSerializer(many=True, read_only=True) + silent = serializers.ListField(child=serializers.CharField(), read_only=True) + silent_count = serializers.IntegerField(read_only=True) + analyzers_considered = serializers.IntegerField(read_only=True) + + class SummarizeJobResultSerializer(ToolResultSerializer): summary = serializers.CharField(allow_null=True) + verdict = JobVerdictSerializer(allow_null=True) diff --git a/frontend/src/components/chat/QuickActions.jsx b/frontend/src/components/chat/QuickActions.jsx index 9106de97e3..ea337e6aae 100644 --- a/frontend/src/components/chat/QuickActions.jsx +++ b/frontend/src/components/chat/QuickActions.jsx @@ -10,11 +10,13 @@ import { Button } from "reactstrap"; export const JOB_DETAIL_RE = /^\/jobs\/(\d+)(?:\/|$)/; export const INVESTIGATION_DETAIL_RE = /^\/investigation\/(\d+)(?:\/|$)/; +// One chip covers both intents: summarize_job now returns IntelOwl's verdict alongside the +// metadata, so a separate "Evaluate results" chip would send a different phrasing to the same +// tool and get the same answer. The message keeps the exact wording the prompt routes on. const JOB_ACTIONS = [ - { label: "Summarize this job", message: "Summarize job #{id}" }, + { label: "Summarize & evaluate", message: "Summarize job #{id}" }, { label: "Which plugins ran?", message: "Which plugins ran on job #{id}?" }, { label: "Show job details", message: "Show me the details of job #{id}" }, - { label: "Evaluate results", message: "Evaluate the results of job #{id}" }, ]; const INVESTIGATION_ACTIONS = [ diff --git a/frontend/tests/components/chat/QuickActions.test.jsx b/frontend/tests/components/chat/QuickActions.test.jsx index 399e388c64..3b0dded0fc 100644 --- a/frontend/tests/components/chat/QuickActions.test.jsx +++ b/frontend/tests/components/chat/QuickActions.test.jsx @@ -29,10 +29,11 @@ describe("QuickActions", () => { mockLocation("/jobs/42"); render(); - expect(screen.getByText("Summarize this job")).toBeInTheDocument(); + expect(screen.getByText("Summarize & evaluate")).toBeInTheDocument(); expect(screen.getByText("Which plugins ran?")).toBeInTheDocument(); expect(screen.getByText("Show job details")).toBeInTheDocument(); - expect(screen.getByText("Evaluate results")).toBeInTheDocument(); + // the verdict now ships inside summarize_job, so the separate evaluate chip is gone + expect(screen.queryByText("Evaluate results")).not.toBeInTheDocument(); // generic chips must not appear expect(screen.queryByText("Show my recent jobs")).not.toBeInTheDocument(); }); @@ -40,7 +41,7 @@ describe("QuickActions", () => { it("shows job-specific chips on a job sub-page", () => { mockLocation("/jobs/42/visualizer/DNS"); render(); - expect(screen.getByText("Summarize this job")).toBeInTheDocument(); + expect(screen.getByText("Summarize & evaluate")).toBeInTheDocument(); }); it("shows investigation-specific chips on an investigation page", () => { @@ -73,19 +74,10 @@ describe("QuickActions", () => { const onSend = jest.fn(); render(); - await userEvent.click(screen.getByText("Summarize this job")); + await userEvent.click(screen.getByText("Summarize & evaluate")); expect(onSend).toHaveBeenCalledWith("Summarize job #42"); }); - it("calls onSend with resolved id on Evaluate results click", async () => { - mockLocation("/jobs/42"); - const onSend = jest.fn(); - render(); - - await userEvent.click(screen.getByText("Evaluate results")); - expect(onSend).toHaveBeenCalledWith("Evaluate the results of job #42"); - }); - it("calls onSend with the raw message on generic pages", async () => { mockLocation("/dashboard"); const onSend = jest.fn(); @@ -100,7 +92,7 @@ describe("QuickActions", () => { const onSend = jest.fn(); render(); - await userEvent.click(screen.getByText("Summarize this job")); + await userEvent.click(screen.getByText("Summarize & evaluate")); expect(onSend).not.toHaveBeenCalled(); }); diff --git a/tests/api_app/chatbot_manager/test_agent.py b/tests/api_app/chatbot_manager/test_agent.py index ed36e0f75e..81d41b269d 100644 --- a/tests/api_app/chatbot_manager/test_agent.py +++ b/tests/api_app/chatbot_manager/test_agent.py @@ -297,6 +297,40 @@ def test_jobs_questions_are_anchored_to_search_jobs(self): self.assertIn("search_jobs", _SYSTEM_PROMPT) +class SystemPromptVerdictRoutingTestCase(TestCase): + """Both former quick-action intents ("summarize" and "evaluate") must land on summarize_job. + + The verdict was folded into summarize_job instead of getting its own tool, so there is no + routing decision left for the model to get wrong; this test only guarantees the prompt keeps + advertising both phrasings on that one tool. It never touches Ollama. + """ + + def test_summarize_and_evaluate_intents_share_one_tool(self): + lower = _SYSTEM_PROMPT.lower() + # Collected rather than `next()`-ed: a bare next() raises StopIteration if the cue is ever + # renamed, which reads as a crash instead of a failed assertion. The count is asserted too, + # since a second summarize_job cue would make the routing ambiguous. + summarize_cues = [line for line in lower.splitlines() if line.startswith("- summarize_job:")] + self.assertEqual(len(summarize_cues), 1, f"expected one summarize_job cue, got {summarize_cues}") + summarize_cue = summarize_cues[0] + self.assertIn("summarize job #n", summarize_cue) + self.assertIn("evaluate the results of job #n", summarize_cue) + self.assertIn("is job #n malicious?", summarize_cue) + # no evaluate_job tool exists: the fold is the whole point + self.assertNotIn("evaluate_job", lower) + + def test_prompt_tells_the_model_what_to_relay_from_the_verdict(self): + """The narration contract the live smoke (Task 5) measures: headline first, then counts. + + A structured `verdict` object only helps if the model actually reads the fields; the + prompt is the only lever for that, so the three field names are pinned here. + """ + lower = _SYSTEM_PROMPT.lower() + self.assertIn("headline", lower) + self.assertIn("supporting", lower) + self.assertIn("silent", lower) + + class ParseKeepAliveTestCase(TestCase): """OLLAMA_KEEP_ALIVE is coerced to what Ollama expects: int seconds or a duration string.""" diff --git a/tests/api_app/chatbot_manager/test_evaluation.py b/tests/api_app/chatbot_manager/test_evaluation.py new file mode 100644 index 0000000000..aea85ff106 --- /dev/null +++ b/tests/api_app/chatbot_manager/test_evaluation.py @@ -0,0 +1,209 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +"""Unit tests for the chatbot's job-verdict reader. + +The reader never invokes the LLM and never queries Job, so these tests seed DataModels +directly and assert the reader agrees with the platform's own reconciliation. +""" + +from uuid import uuid4 + +from django.test import TestCase + +from api_app.analyzables_manager.models import Analyzable +from api_app.analyzers_manager.models import AnalyzerConfig, AnalyzerReport +from api_app.chatbot_manager.evaluation import ( + MAX_EVIDENCE_ANALYZERS, + REASON_GENERIC, + REASON_NO_EVALUATION, + evaluate_job, +) +from api_app.choices import TLP, Classification +from api_app.data_model_manager.enums import DataModelEvaluations, DataModelVerdictBuckets +from api_app.data_model_manager.models import DomainDataModel +from api_app.engines_manager.engines.evaluation import EvaluationEngineModule +from api_app.models import Job +from api_app.user_events_manager.models import UserAnalyzableEvent +from certego_saas.apps.user.models import User + + +class JobEvaluationTestCase(TestCase): + def setUp(self): + self.user, _ = User.objects.get_or_create(username="chatbot_verdict_user") + self.analyzable, _ = Analyzable.objects.get_or_create( + name="verdict.example.com", classification=Classification.DOMAIN + ) + self.job = self._make_job(self.analyzable) + # The seeded DB ships hundreds of configs; a slice gives distinct (job, config) pairs. + self.configs = list(AnalyzerConfig.objects.all()[:15]) + + def tearDown(self): + Job.objects.filter(user=self.user).delete() + UserAnalyzableEvent.objects.filter(user=self.user).delete() + + def _make_job(self, analyzable): + return Job.objects.create( + user=self.user, + analyzable=analyzable, + status=Job.STATUSES.REPORTED_WITHOUT_FAILS, + tlp=TLP.CLEAR.value, + ) + + @staticmethod + def _data_model(evaluation, reliability): + return DomainDataModel.objects.create(evaluation=evaluation, reliability=reliability) + + def _add_report(self, config, data_model=None, job=None): + report = AnalyzerReport.objects.create( + report={}, + job=job or self.job, + config=config, + status=AnalyzerReport.STATUSES.SUCCESS.value, + task_id=str(uuid4()), + parameters={}, + ) + if data_model is not None: + # GenericForeignKey assignment sets content type + object id in one go. + report.data_model = data_model + report.save() + return report + + def test_generic_observable_reports_no_evaluation_honestly(self): + """GENERIC has no DataModel class at all — the reader must say so, not fabricate.""" + generic, _ = Analyzable.objects.get_or_create( + name="some free text", classification=Classification.GENERIC + ) + result = evaluate_job(self._make_job(generic)) + self.assertEqual(result.bucket, DataModelVerdictBuckets.NO_EVALUATION.value) + self.assertEqual(result.reason, REASON_GENERIC) + self.assertIsNone(result.evaluation) + self.assertEqual(result.supporting, []) + + def test_no_data_models_reports_no_evaluation_and_lists_silent(self): + for config in self.configs[:3]: + self._add_report(config) + result = evaluate_job(self.job) + self.assertEqual(result.bucket, DataModelVerdictBuckets.NO_EVALUATION.value) + self.assertEqual(result.reason, REASON_NO_EVALUATION) + self.assertEqual(result.silent_count, 3) + self.assertEqual(sorted(result.silent), sorted(c.name for c in self.configs[:3])) + self.assertEqual(result.analyzers_considered, 3) + + def test_malicious_verdict_partitions_the_evidence(self): + self._add_report(self.configs[0], self._data_model(DataModelEvaluations.MALICIOUS.value, 8)) + self._add_report(self.configs[1], self._data_model(DataModelEvaluations.MALICIOUS.value, 8)) + self._add_report(self.configs[2], self._data_model(DataModelEvaluations.TRUSTED.value, 4)) + self._add_report(self.configs[3]) # ran, no data model -> silent + result = evaluate_job(self.job) + self.assertEqual(result.bucket, DataModelVerdictBuckets.MALICIOUS.value) + self.assertEqual(result.evaluation, DataModelEvaluations.MALICIOUS.value) + self.assertEqual(result.reliability, 8) + # assertCountEqual: the two supporting analyzers tie on reliability and `analyzerreports` + # carries no explicit ordering, so their relative order is not guaranteed by the DB. + self.assertCountEqual( + [v.name for v in result.supporting], [self.configs[0].name, self.configs[1].name] + ) + self.assertEqual([v.name for v in result.contradicting], [self.configs[2].name]) + self.assertEqual(result.silent, [self.configs[3].name]) + self.assertEqual(result.analyzers_considered, 4) + self.assertFalse(result.analyst_override) + + def test_headline_states_all_three_counts_including_zero_silence(self): + """The headline must be self-contained: supporting, contradicting AND silent counts. + + This is not cosmetic. A live smoke against qwen2.5:3b showed that a headline reporting only + "N of M analyzers support it" makes the model infer the remaining M-N analyzers all + disagree, and then state there were no silent ones — the opposite of the honest-absence + reporting this module exists to provide. The silent count is stated even when it is 0. + """ + self._add_report(self.configs[0], self._data_model(DataModelEvaluations.MALICIOUS.value, 8)) + self._add_report(self.configs[1], self._data_model(DataModelEvaluations.TRUSTED.value, 4)) + self._add_report(self.configs[2]) # ran, no data model -> silent + headline = evaluate_job(self.job).headline + self.assertIn("3 analyzers ran", headline) + self.assertIn("1 supporting", headline) + self.assertIn("1 contradicting", headline) + self.assertIn("1 silent", headline) + + def test_headline_equals_the_platform_reconciliation(self): + """The verdict must be the engine's verdict — never a chatbot-side recomputation.""" + self._add_report(self.configs[0], self._data_model(DataModelEvaluations.MALICIOUS.value, 7)) + self._add_report(self.configs[1], self._data_model(DataModelEvaluations.MALICIOUS.value, 6)) + engine_result = EvaluationEngineModule(self.job).run() + result = evaluate_job(self.job) + self.assertEqual(result.evaluation, engine_result["evaluation"]) + self.assertEqual(result.reliability, int(engine_result["reliability"])) + + def test_reliability_matches_the_value_the_engine_persists(self): + """Avg(5, 6) = 5.5 and the engine TRUNCATES it to 5 -> suspicious, not malicious. + + `merge()` assigns the float and saves; Django's IntegerField.get_prep_value does `int(v)` + (truncation, not rounding), so the badge shows 5. This test replays exactly what + `execute_engine_module` does and pins the reader to the persisted value: rounding here + would make the chatbot say "malicious" while the badge says "suspicious", the one thing + the design forbids. + """ + self._add_report(self.configs[0], self._data_model(DataModelEvaluations.MALICIOUS.value, 5)) + self._add_report(self.configs[1], self._data_model(DataModelEvaluations.MALICIOUS.value, 6)) + stored = DomainDataModel.objects.create() + stored.merge(EvaluationEngineModule(self.job).run(), append=False) + stored.refresh_from_db() + result = evaluate_job(self.job) + self.assertEqual(stored.reliability, 5) + self.assertEqual(result.reliability, stored.reliability) + self.assertEqual(result.bucket, DataModelVerdictBuckets.SUSPICIOUS.value) + + def test_stale_job_scalar_is_ignored_the_race_case(self): + """The engine runs async, so `job.data_model` can hold a stale/absent verdict. + + Here the job is still RUNNING and its scalar says `trusted` while the analyzer DataModels + say malicious: the reader must report malicious, because it recomputes from the same + sources the engine uses instead of trusting the scalar. + """ + self.job.status = Job.STATUSES.RUNNING + self.job.data_model = self._data_model(DataModelEvaluations.TRUSTED.value, 9) + self.job.save() + self._add_report(self.configs[0], self._data_model(DataModelEvaluations.MALICIOUS.value, 8)) + result = evaluate_job(self.job) + self.assertEqual(result.bucket, DataModelVerdictBuckets.MALICIOUS.value) + self.assertEqual(result.reliability, 8) + self.assertFalse(result.analyst_override) + + def test_malicious_below_the_floor_is_suspicious(self): + """Boundary: reliability 5 is under MALICIOUS_RELIABILITY_FLOOR (6).""" + self._add_report(self.configs[0], self._data_model(DataModelEvaluations.MALICIOUS.value, 5)) + self.assertEqual(evaluate_job(self.job).bucket, DataModelVerdictBuckets.SUSPICIOUS.value) + + def test_analyst_event_wins_and_is_surfaced(self): + self._add_report(self.configs[0], self._data_model(DataModelEvaluations.MALICIOUS.value, 8)) + UserAnalyzableEvent.objects.create( + user=self.user, + analyzable=self.analyzable, + data_model=self._data_model(DataModelEvaluations.TRUSTED.value, 9), + ) + result = evaluate_job(self.job) + self.assertEqual(result.bucket, DataModelVerdictBuckets.TRUSTED.value) + self.assertTrue(result.analyst_override) + self.assertIn("analyst", result.headline) + # The override headline must still carry the counts: a headline that omits them is what + # made the model assert there were no silent analyzers (see the smoke report). + self.assertIn("0 agree", result.headline) + self.assertIn("1 disagree", result.headline) + self.assertIn("0 silent", result.headline) + # the disagreeing analyzer is still shown, so the chatbot never hides the conflict + self.assertEqual([v.name for v in result.contradicting], [self.configs[0].name]) + + def test_evidence_lists_are_capped(self): + for config in self.configs[:12]: + self._add_report(config, self._data_model(DataModelEvaluations.MALICIOUS.value, 8)) + result = evaluate_job(self.job) + self.assertEqual(len(result.supporting), MAX_EVIDENCE_ANALYZERS) + self.assertEqual(result.analyzers_considered, 12) + + def test_reader_writes_nothing(self): + """Pure read: the live recompute must not persist a data model on the job.""" + self._add_report(self.configs[0], self._data_model(DataModelEvaluations.MALICIOUS.value, 8)) + evaluate_job(self.job) + self.job.refresh_from_db() + self.assertIsNone(self.job.data_model_object_id) diff --git a/tests/api_app/chatbot_manager/test_prompt.py b/tests/api_app/chatbot_manager/test_prompt.py index 2498280410..7fc1fd906f 100644 --- a/tests/api_app/chatbot_manager/test_prompt.py +++ b/tests/api_app/chatbot_manager/test_prompt.py @@ -53,11 +53,17 @@ def test_prompt_file_readable(self): ) def test_prompt_under_token_limit(self): - """System prompt must stay under 500 tokens to leave room for tool schemas - and conversation history within Ollama's 8192 context window. + """Cap the prompt so the tool schemas and the conversation history still fit in + Ollama's 8192-token context window. + + The bound counts whitespace-separated WORDS, not tokens — for this text roughly 1.3-1.4 + tokens per word, so 600 words is about 850 tokens, near a tenth of the window. The + original 500 was set when the prompt was 430 words; it had shrunk to 8 words of headroom + and was rejecting further rules rather than protecting the window, so it is raised here + together with the rule that needed the room. """ - tokens = len(_SYSTEM_PROMPT.split()) - self.assertLess(tokens, 500, f"system prompt is {tokens} tokens — exceeds 500") + words = len(_SYSTEM_PROMPT.split()) + self.assertLess(words, 600, f"system prompt is {words} words — exceeds 600") def test_prompt_includes_all_tool_names(self): """Every registered tool appears in the [Tools] section, and the hardcoded diff --git a/tests/api_app/chatbot_manager/test_query_counts.py b/tests/api_app/chatbot_manager/test_query_counts.py index b305459213..dfc89acad2 100644 --- a/tests/api_app/chatbot_manager/test_query_counts.py +++ b/tests/api_app/chatbot_manager/test_query_counts.py @@ -28,6 +28,8 @@ from api_app.chatbot_manager.models import ChatMessage, ChatSession from api_app.chatbot_manager.tasks import process_chat_message from api_app.choices import TLP, Classification +from api_app.data_model_manager.enums import DataModelEvaluations +from api_app.data_model_manager.models import DomainDataModel from api_app.investigations_manager.models import Investigation from api_app.models import Job from api_app.playbooks_manager.models import PlaybookConfig @@ -107,6 +109,25 @@ def _add_reports(self, job, configs): parameters={}, ) + @staticmethod + def _add_reports_with_data_models(job, configs): + # Same as _add_reports but each report carries an evaluated DataModel, which is the + # dimension the verdict reader walks. + for config in configs: + data_model = DomainDataModel.objects.create( + evaluation=DataModelEvaluations.MALICIOUS.value, reliability=8 + ) + report = AnalyzerReport.objects.create( + report={}, + job=job, + config=config, + status=AnalyzerReport.STATUSES.SUCCESS.value, + task_id=str(uuid4()), + parameters={}, + ) + report.data_model = data_model + report.save() + def _make_playbook(self, name): # A user-owned starting playbook supporting DOMAIN, so recommend_playbook("domain") returns it. return PlaybookConfig.objects.create( @@ -178,6 +199,20 @@ def test_summarize_job_query_count_is_constant_in_reports(self): large = _count_queries(lambda: self.summarize_job.invoke({"job_id": job.pk})) self.assertEqual(small, large) + def test_summarize_job_query_count_is_constant_in_data_models(self): + # The verdict reader adds a fixed set of queries (live reconciliation + one queryset for + # the analyzer DataModels + one for the user events). Attribution goes through + # data_model_object_id instead of the report's GenericForeignKey precisely so this stays + # flat: resolving the FK per report would be a textbook N+1. + job = self._make_job() + configs = list(AnalyzerConfig.objects.all()[:6]) + self._add_reports_with_data_models(job, configs[:1]) + self.summarize_job.invoke({"job_id": job.pk}) # warm up + small = _count_queries(lambda: self.summarize_job.invoke({"job_id": job.pk})) + self._add_reports_with_data_models(job, configs[1:6]) + large = _count_queries(lambda: self.summarize_job.invoke({"job_id": job.pk})) + self.assertEqual(small, large) + def test_list_investigations_query_count_is_constant(self): Investigation.objects.create( owner=self.user, name="perf inv 0", status=Investigation.STATUSES.CREATED.value diff --git a/tests/api_app/chatbot_manager/tools/test_summarize_job.py b/tests/api_app/chatbot_manager/tools/test_summarize_job.py new file mode 100644 index 0000000000..b527269c1c --- /dev/null +++ b/tests/api_app/chatbot_manager/tools/test_summarize_job.py @@ -0,0 +1,118 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +"""Tool-level tests for the verdict summarize_job returns (no LLM, no network).""" + +import json +from uuid import uuid4 + +from django.test import TestCase + +from api_app.analyzables_manager.models import Analyzable +from api_app.analyzers_manager.models import AnalyzerConfig, AnalyzerReport +from api_app.chatbot_manager.agent.tools import build_tools +from api_app.chatbot_manager.evaluation import REASON_GENERIC, REASON_NO_EVALUATION +from api_app.choices import TLP, Classification +from api_app.data_model_manager.enums import DataModelEvaluations, DataModelVerdictBuckets +from api_app.data_model_manager.models import DomainDataModel +from api_app.models import Job +from certego_saas.apps.user.models import User + + +class SummarizeJobVerdictTestCase(TestCase): + """summarize_job carries IntelOwl's own verdict (PR C fold): no separate evaluate tool.""" + + def setUp(self): + self.user, _ = User.objects.get_or_create(username="chatbot_summ_verdict_user") + self.analyzable, _ = Analyzable.objects.get_or_create( + name="verdict-tool.example.com", classification=Classification.DOMAIN + ) + self.job = Job.objects.create( + user=self.user, + analyzable=self.analyzable, + status=Job.STATUSES.REPORTED_WITHOUT_FAILS, + tlp=TLP.CLEAR.value, + ) + self.config = AnalyzerConfig.objects.first() + tools_by_name = {tool.name: tool for tool in build_tools(user=self.user)} + self.summarize_job = tools_by_name["summarize_job"] + + def tearDown(self): + Job.objects.filter(user=self.user).delete() + + def _add_report_with_verdict(self, evaluation, reliability): + data_model = DomainDataModel.objects.create(evaluation=evaluation, reliability=reliability) + report = AnalyzerReport.objects.create( + report={}, + job=self.job, + config=self.config, + status=AnalyzerReport.STATUSES.SUCCESS.value, + task_id=str(uuid4()), + parameters={}, + ) + report.data_model = data_model + report.save() + return report + + def test_summarize_job_includes_the_verdict(self): + self._add_report_with_verdict(DataModelEvaluations.MALICIOUS.value, 8) + payload = json.loads(self.summarize_job.invoke({"job_id": self.job.pk})) + verdict = payload["verdict"] + self.assertEqual(verdict["bucket"], DataModelVerdictBuckets.MALICIOUS.value) + self.assertEqual(verdict["evaluation"], DataModelEvaluations.MALICIOUS.value) + self.assertEqual(verdict["reliability"], 8) + self.assertIn(DataModelVerdictBuckets.MALICIOUS.value, verdict["headline"]) + self.assertEqual([item["name"] for item in verdict["supporting"]], [self.config.name]) + self.assertEqual(verdict["contradicting"], []) + self.assertEqual(verdict["silent"], []) + self.assertEqual(verdict["analyzers_considered"], 1) + self.assertFalse(verdict["analyst_override"]) + # the metadata summary is unchanged and still present + self.assertIn(f"Job #{self.job.pk}", payload["summary"]) + + def test_headline_is_echoed_into_the_prose_summary(self): + """The headline is deliberately duplicated in `summary`, not only in `verdict`. + + A live smoke against qwen2.5:3b showed the model reproduces this prose verbatim while + paraphrasing the structured object away — dropping the reliability and reporting + contradicting analyzers as silent. Removing this line regresses the narration, so it is + pinned here rather than left to the reviewer's memory. + """ + self._add_report_with_verdict(DataModelEvaluations.MALICIOUS.value, 8) + payload = json.loads(self.summarize_job.invoke({"job_id": self.job.pk})) + self.assertIn(payload["verdict"]["headline"], payload["summary"]) + + def test_summarize_job_without_evaluation_says_so(self): + payload = json.loads(self.summarize_job.invoke({"job_id": self.job.pk})) + self.assertEqual(payload["verdict"]["bucket"], DataModelVerdictBuckets.NO_EVALUATION.value) + self.assertEqual(payload["verdict"]["reason"], REASON_NO_EVALUATION) + self.assertIsNone(payload["verdict"]["evaluation"]) + + def test_summarize_job_generic_observable_has_no_verdict(self): + generic, _ = Analyzable.objects.get_or_create( + name="free text observable", classification=Classification.GENERIC + ) + job = Job.objects.create( + user=self.user, + analyzable=generic, + status=Job.STATUSES.REPORTED_WITHOUT_FAILS, + tlp=TLP.CLEAR.value, + ) + payload = json.loads(self.summarize_job.invoke({"job_id": job.pk})) + self.assertEqual(payload["verdict"]["bucket"], DataModelVerdictBuckets.NO_EVALUATION.value) + self.assertEqual(payload["verdict"]["reason"], REASON_GENERIC) + + def test_summarize_job_not_visible_returns_null_verdict(self): + """Tenancy: an invisible job is indistinguishable from a missing one — no verdict leaks.""" + other_user, _ = User.objects.get_or_create(username="chatbot_summ_verdict_other") + other_job = Job.objects.create( + user=other_user, + analyzable=self.analyzable, + status=Job.STATUSES.REPORTED_WITHOUT_FAILS, + tlp=TLP.RED.value, + ) + payload = json.loads(self.summarize_job.invoke({"job_id": other_job.pk})) + self.assertIsNone(payload["verdict"]) + self.assertIsNone(payload["summary"]) + self.assertTrue(payload["errors"]) + other_job.delete()