Fix context-rot and citation quality issues in the chat agent

.gitignore

@@ -246,4 +246,7 @@ http-client.private.env.json
 
 charts/
 test.yaml
-src/app/baml_client
+src/app/baml_client
+.DS_Store
+src/.DS_Store
+*.swp

src/app/api/api_v1/endpoints/chat.py

@@ -6,7 +6,6 @@
 import psycopg
 from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request, status
 from fastapi.responses import StreamingResponse
-from langchain_core.messages import ToolMessage
 from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
 from openai import RateLimitError
 from psycopg.rows import AsyncRowFactory, DictRow, dict_row
@@ -20,7 +19,7 @@
 from src.app.models import chat as models
 from src.app.search.services.search import SearchService, get_search_service
 from src.app.services.data_collection import get_data_collection_service
-from src.app.services.helpers import linkify_missing_citations
+from src.app.services.helpers import latest_tool_docs, linkify_missing_citations
 from src.app.shared.domain.constants import subjects as subjectsDict
 from src.app.shared.domain.exceptions import (
     EmptyQueryError,
@@ -351,7 +350,6 @@ async def agent_response(
 ):
     try:
         session_id = extract_session_cookie(request)
-        docs = []
 
         thread_id = body.thread_id if body.thread_id else None
 
@@ -400,11 +398,7 @@ async def agent_response(
                 trace_context=trace_context,
             )
 
-        all_docs = []
-        for msg in res["messages"]:
-            if isinstance(msg, ToolMessage) and getattr(msg, "artifact", None):
-                all_docs.extend(msg.artifact)
-        docs = all_docs if all_docs else None
+        docs = latest_tool_docs(res["messages"])
         content = linkify_missing_citations(
             cast(str, res["messages"][-1].content), docs or []
         )

src/app/api/api_v1/endpoints/chat_utils.py

@@ -107,9 +107,18 @@ async def _stream_agent_with_memory(
             trace_context=trace_context,
         )
 
+        docs_found = False
         async for chunk in stream:
+            if chunk.get("status") == "processing" and chunk.get("docs"):
+                docs_found = True
             yield chunk
 
+        if not docs_found:
+            # No tool call this turn — fall back to the last known results
+            # instead of leaving the client with no docs at all.
+            docs = await chatfactory.agent_get_latest_docs(thread_id, memory)
+            yield {"status": "docs_final", "docs": docs}
+
 
 def _build_final_stream_payload(
     *,
@@ -173,6 +182,9 @@ async def _stream_agent_response(
     )
 
     async for chunk in stream:
+        if chunk.get("status") == "docs_final":
+            docs = chunk.get("docs")
+            continue
         final_content, docs = _update_agent_stream_state(chunk, final_content, docs)
         if chunk.get("status") == "streaming" and chunk.get("content"):
             has_streamed_content = True

src/app/services/data_collection.py

@@ -72,7 +72,6 @@ async def register_search_data(
         corpora: list[str] | None = None,
         feature: str | None = "search",
     ) -> uuid.UUID | None:
-
         if not self.should_collect:
             logger.info("data_collection is not enabled.")
             return None
@@ -119,7 +118,6 @@ async def register_syllabus_data(
         agent_answer: str,
         feature: Literal["syllabus_creation", "syllabus_feedback"],
     ) -> uuid.UUID | None:
-
         if not self.should_collect:
             logger.info("data_collection is not enabled.")
             return None
@@ -246,7 +244,6 @@ async def register_chat_data(
         sources: list[Document],
         feature: str | None = "chat",
     ) -> tuple[uuid.UUID | None, uuid.UUID | None]:
-
         if not self.should_collect:
             logger.info("data_collection is not enabled.")
             return None, None

src/app/services/helpers.py

@@ -6,6 +6,7 @@
 import numpy
 from fastapi import HTTPException
 from json_repair import JSONReturnType
+from langchain_core.messages import BaseMessage, ToolMessage
 from langdetect import detect_langs
 from qdrant_client.http.models import models
 from welearn_database.data.models import EmbeddingModel
@@ -140,15 +141,26 @@ def stringify_docs_content(docs: List[Any]) -> str:
     return documents.strip()
 
 
-_CITATION_RE = re.compile(r'(?<!target="_blank">)\[Doc\s*(\d+)\]')
+_CITATION_RE = re.compile(
+    r'(?<!\[)(?<!target="_blank">)\[Doc\s*(\d+)\](?!\()', re.IGNORECASE
+)
 
 
 def linkify_missing_citations(text: str, docs: List[Any]) -> str:
     """
-    Wraps any bare `[Doc N]` marker in `text` with the `<a href=... target="_blank">`
-    tag for document N, using the URL from `docs[N-1]`. Markers already wrapped in an
-    <a> tag are left untouched. Safety net for when the LLM forgets to format a
-    citation as a link itself.
+    Wraps any bare `[Doc N]` marker in `text` into a double-bracket Markdown link
+    `[[Doc N]](URL)` for document N, using the URL from `docs[N-1]`. The double
+    bracket is intentional: a plain Markdown link `[Doc N](URL)` renders with its
+    brackets stripped (shows just "Doc N"), so the visible label must itself
+    contain a literal "[Doc N]" for the brackets to survive rendering.
+
+    Markers already wrapped this way, already a single-bracket Markdown link, or
+    already wrapped in an HTML `<a>` tag, are left untouched. Safety net for when
+    the LLM forgets to format a citation as a link itself.
+
+    Only matches a single document number per marker — a combined citation like
+    "[Docs 3 et 5]" is intentionally left as-is, since there's no single URL a
+    two-document marker could safely resolve to.
 
     Args:
         text: The assembled answer text.
@@ -172,11 +184,32 @@ def _replace(match: "re.Match[str]") -> str:
         url = _url_for(n)
         if not url:
             return match.group(0)
-        return f'<a href="{url}" target="_blank">[Doc {n}]</a>'
+        return f"[[Doc {n}]]({url})"
 
     return _CITATION_RE.sub(_replace, text)
 
 
+def latest_tool_docs(messages: List[BaseMessage]) -> Optional[List[Any]]:
+    """
+    Finds the most recent tool call's retrieved documents in a message list.
+
+    Walks `messages` from the end, so a turn that made no new tool call still
+    resolves to the last tool call's results instead of nothing, and a turn
+    that did call the tool never picks up a stale, older call's results.
+
+    Args:
+        messages: The full conversation message list (may span many turns).
+
+    Returns:
+        The `artifact` of the most recent `ToolMessage` that has one, or
+        None if no tool call with results is found.
+    """
+    for msg in reversed(messages):
+        if isinstance(msg, ToolMessage) and getattr(msg, "artifact", None):
+            return msg.artifact
+    return None
+
+
 def extract_json_from_response(
     response: str,
 ) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]] | str:

src/app/services/prompts.py

@@ -15,33 +15,56 @@
 - If your draft answer is turning into a list of more than ~4 items or more than one paragraph, stop and cut it down.
 - Do not open with sycophantic phrases ("That sounds fascinating!", "Great question!", "What a fantastic starting point!"). Acknowledge context matter-of-factly and respond directly.
 - Always reply in the same language the user wrote in.
+- When replying in French, always use vouvoiement (vous) — never tutoiement (tu) — regardless of how the user addressed you.
 
 **Ask before you answer at length (Socratic behavior)**
 - On the first substantive message of a new conversation, and whenever the user pivots to a new subject or topic mid-conversation, check whether you have enough context to give a genuinely useful answer: their discipline/course subject, level of study, and the kind of help they want (e.g. discussion prompts, a session plan, illustrative examples, background reading).
 - If that context is thin, do not produce a full answer yet. Ask only the clarifying questions you actually need, combined into a single short message rather than a numbered list — never more than 3 questions.
-- A clarifying-question turn is a conversational meta-turn: do not call the retrieval tool on it.
+- A clarifying-question turn is a conversational meta-turn: do not call `get_resources_about_sustainability` on it.
 - Once the user has answered, or their message already made intent and context clear, answer directly. Do not re-ask for context you already have, and do not interrogate the user turn after turn.
 
-**No links beyond what was retrieved this turn**
-- Never produce a link, URL, or `<a>` tag for anything other than a document returned by `get_resources_about_sustainability` in this same conversation turn. This includes links you might otherwise produce from general/parametric knowledge (a well-known Wikipedia page, a UN SDG page, a journal homepage, etc.). If you want to reference something you did not retrieve, name it in plain text with no link and no fabricated URL.
-
-**Using the retrieval tool**
+**Using the `get_resources_about_sustainability` retrieval tool**
+- When preparing your response to the user, call the `get_resources_about_sustainability` tool as much as possible to get additional, relevantresources that will help you answer the user's question in a way that is more accurate and sourced.
+- HOWEVER, do not call the tool for greetings, conversational meta-turns (e.g. "thanks", "can you explain that again"), clarifying-question turns (see above), or questions answerable from general knowledge where a cited source adds no value.
 - Call `get_resources_about_sustainability` at most once per response. Write a single comprehensive query that covers all aspects of the user's question.
-- Call the tool for factual, SDG-specific, or topic-based questions where curated sources add value.
-- Do not call the tool for greetings, conversational meta-turns (e.g. "thanks", "can you explain that again"), clarifying-question turns (see above), or questions answerable from general knowledge where a cited source adds no value.
+- If the user's next question stays on the same topic as your most recent `get_resources_about_sustainability` call and those results still cover it, do not call it again — keep using and citing that same set of results. Call it again only once the topic shifts or those results no longer suffice.
+- Make sure to use as many of the retrieved documents as relevant to answer the user's question, and cite them explicitly in your response next to the information that you used from their content. When citing a retrieved document, make sure to stay within the context of the document and not to make up information.
 - If the retrieved documents are insufficient to answer, say so in your response — do not make a second tool call.
 
+**No sources beyond what was retrieved**
+- Never name, describe, or link any source — an article, video, journal, dataset, or creator — other than a document returned by your most recent `get_resources_about_sustainability` call, either from this turn or from an earlier turn if you are reusing its results because the topic hasn't shifted. This applies even with no link attached: do not mention a title, journal name, or video you did not retrieve, not even in plain text.
+- If you don't have a retrieved document to support a point, make the point in your own words with no source attribution at all. Never produce a link, URL, or fabricated citation from general/parametric knowledge (a well-known Wikipedia page, a UN SDG page, a journal homepage, etc.).
+
 **Citing sources**
-- The url of each document is on a dedicated line formatted as `url:<URL>`. Copy that URL character-for-character — never substitute a Wikipedia URL, construct a URL, or modify it in any way.
-- Format every inline citation as: <a href="URL" target="_blank">[Doc N]</a> where URL is the verbatim value from the document's url line and N is the document number. Never write a bare `[Doc N]` without its surrounding `<a>` tag — the tag is what makes the citation clickable.
+- Every document's URL is on a dedicated line formatted as `url:<URL>`. Copy that URL character-for-character. Never substitute, construct, guess, or modify a URL in any way — not even a Wikipedia URL you believe is close enough.
+- Markdown renders a link as only its label text — [Duck Duck Go](https://duckduckgo.com) displays as just "Duck Duck Go"; the brackets and the URL are consumed as syntax and never shown. We need the "[Doc N]" brackets to stay visible, so wrap the whole marker in an EXTRA pair of brackets: [[Doc N]](URL) — the outer brackets are Markdown link syntax (invisible once rendered), the inner "[Doc N]" is the literal label text that survives rendering. URL is the verbatim value from that document's url line and N is its document number.
+- Wrong: "...pour modéliser des opérations industrielles Doc 2." (no brackets at all). Also wrong: "...opérations industrielles [Doc 2](URL)." (single brackets — renders with the brackets stripped, same problem). Correct: "...pour modéliser des opérations industrielles [[Doc 2]](URL)." (renders as the clickable text "[Doc 2]").
+- One document per citation marker. Never combine document numbers in a single bracket (never write "[[Doc 3 and 5]]" or "[[Docs 3 et 5]]" — a link can only point to one URL, so a combined marker is always broken). If a claim draws on two documents, place two separate markers next to each other: [[Doc 3]](url3) [[Doc 5]](url5).
+- Before citing a document for a specific claim, confirm that exact claim is actually stated in that document's content — never attribute a fact, quote, or statistic to a document that doesn't contain it, even if a different retrieved document does.
+- Only cite a document from your most recent `get_resources_about_sustainability` call — never a document number from before that call. Each call produces its own fresh Doc 1, Doc 2, etc.; once a newer call happens, the previous numbering is no longer valid, even if you cited it in an earlier response.
 - Do not invent examples, quotes, statistics, or facts not explicitly stated in the retrieved documents. If a document does not contain enough to support a claim, omit the claim.
-- If no relevant documents are retrieved, say so explicitly before drawing on general knowledge.
-- Do not cite any source that was not returned by the retrieval tool in this conversation turn.
+- Do not cite any source besides what was returned by your most recent `get_resources_about_sustainability` call.
 
-**Suggesting a next step**
-- After giving a substantive answer (not on a clarifying-question turn), if a natural next step exists — going deeper on one aspect, moving from discussion to a concrete classroom activity, or connecting the topic to the user's own discipline or course — end with one focused question that helps them plan their teaching. Never ask more than one, and do not force it every turn.
 """
 
+AGENT_REMINDER_PROMPT = """Reminder of your standing instructions — re-checking every turn, especially in a long conversation:
+- 3–4 sentences max unless the user asked for more; same language as the user, vouvoiement (vous) if French; no sycophantic openers.
+- New topic + thin context → ask up to 3 clarifying questions instead of answering; no tool call on that turn.
+- Call `get_resources_about_sustainability` at most once per response, only for factual/sourced questions — skip it if the current topic is already covered by your most recent call.
+- Never name, describe, or link a source you did not retrieve — not even without a link, not even just a title. Zero exceptions.
+- Cite only documents from your most recent `get_resources_about_sustainability` call, one document per marker, ALWAYS as [[Doc N]](URL) — double brackets, because a single-bracket Markdown link renders with its brackets stripped ("Doc 2" with no brackets at all), which is exactly the formatting error to avoid. Never combine numbers in one marker, never a URL you weren't given, never numbering from a superseded call.
+- Every citation's claim must actually be stated in that specific document — never attribute it to the wrong document or invent it."""
+
+## TAKEN OUT OF THE ABOVE PROMPT TO DEACTIVATE SUGGESTING A NEXT STEP AFTER AN ANSWER
+# **Suggesting a next step**
+# - After giving a substantive answer (not on a clarifying-question turn), if a natural next step exists — going deeper on one aspect, moving from discussion to a concrete classroom activity, or connecting the topic to the user's own discipline or course — end with one focused question that helps them plan their teaching. Never ask more than one, and do not force it every turn.
+# **Response style**
+# - Keep responses concise: 2–4 sentences by default. Expand only when the user explicitly asks for more detail.
+# - When a follow-up question would genuinely help the user think deeper or clarify their intent, end with one focused question. Do not force a question on every turn.
+# - Always reply in the same language the user wrote in.
+##
+
+
 ###########################################################
 ### /qna/chat/answer and /qna/stream — legacy chat ########
 ###########################################################

src/app/services/sql_db/sql_service.py

@@ -24,11 +24,9 @@ def __init__(self):
         self.session_maker = self._create_session()
 
     def _create_engine(self):
-
         return create_engine(self.engine_url)
 
     def _create_session(self):
-
         Session = sessionmaker(bind=self.engine)
         return Session