Initial Query Expansion

VERSION

@@ -1 +1 @@
-0.9.5
+1.0.1

docker-compose-network-github-image.yml

@@ -1,6 +1,6 @@
 services:
   concept-graphs-api:
-    image: ghcr.io/onto-med/concept-graphs/concept-graphs-api:1.0.0
+    image: ghcr.io/onto-med/concept-graphs/concept-graphs-api:1.0.1
     restart: unless-stopped
     ports:
       - 9007:9007

docker-compose-network.yml

@@ -1,6 +1,6 @@
 services:
   concept-graphs-api:
-    image: imise/top/concept-graphs-api:1.0.0
+    image: imise/top/concept-graphs-api:1.0.1
     restart: unless-stopped
     ports:
       - 9007:9007

docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   concept-graphs-api:
-    image: imise/top/concept-graphs-api:1.0.0
+    image: imise/top/concept-graphs-api:1.0.1
     restart: unless-stopped
     ports:
       - 9007:9007

main.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import pathlib
 
 import flask
@@ -15,7 +16,15 @@
 
 
 def configure_logging(logging_setup_tuples: list[tuple] | None = None) -> None:
-    """Configure application logging defaults."""
+    """Configure application logging defaults.
+
+    ``LOG_LEVEL`` controls application/root logging and defaults to ``INFO`` so
+    operational messages such as RAG indexing progress are visible in container
+    logs. Noisy dependency loggers are still kept at warning level by default.
+    """
+    log_level_name = os.getenv("LOG_LEVEL", "INFO").upper()
+    log_level = getattr(logging, log_level_name, logging.INFO)
+
     if logging_setup_tuples is None:
         logging_setup_tuples = [
             ("werkzeug", logging.WARN),
@@ -26,9 +35,11 @@ def configure_logging(logging_setup_tuples: list[tuple] | None = None) -> None:
         logging.getLogger(logger_name).setLevel(level)
 
     root_logger = logging.getLogger()
+    root_logger.setLevel(log_level)
     root_logger.propagate = False
     if root_logger.hasHandlers():
         root_logger.handlers.clear()
+    flask.logging.default_handler.setLevel(log_level)
     root_logger.addHandler(flask.logging.default_handler)
 
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "concept-graphs"
-version = "1.0.0"
+version = "1.0.1"
 description = ""
 authors = [
     { name = "Franz Matthies", email = "franz.matthies@imise.uni-leipzig.de" },
@@ -60,14 +60,21 @@ Repository = "https://github.com/Onto-Med/concept-graphs"
 vectorstore = [
     "marqo==3.18.0",
 ]
-rag = [
+llm = [
     "langchain",
     "langchain-core",
-    "langchain-community",
-    "langchain-text-splitters",
     "langchain-openai",
     "langchain-ollama",
 ]
+rag = [
+    { include-group = "llm" },
+    "langchain-community",
+    "langchain-text-splitters",
+]
+query-expansion = [
+    { include-group = "llm" },
+    "pydantic>=2,<3",
+]
 test = [
     "ruff>=0.14.0",
 ]
@@ -77,6 +84,7 @@ package = false
 default-groups = [
     "vectorstore",
     "rag",
+    "query-expansion",
 ]
 
 [tool.pytest.ini_options]

src/api/context.py

@@ -16,9 +16,22 @@ class ActiveRAG:
     process: str
     ready: bool = False
     initializing: bool = False
+    error: str | None = None
+
+    def mark_ready(self) -> None:
+        self.ready = True
+        self.initializing = False
+        self.error = None
+
+    def mark_not_ready(self, error: str | None = None) -> None:
+        self.ready = False
+        self.initializing = False
+        self.error = error
 
     def switch_readiness(self):
         self.ready = not self.ready
+        if self.ready:
+            self.error = None
 
 
 @dataclass

src/api/routes/rag.py

@@ -20,7 +20,7 @@
 )
 from src.common.threads import StoppableThread
 from src.rag.marqo_rag_utils import extract_text_from_highlights
-from src.rag.rag import RAG
+from src.rag.rag import RAG, no_source_answer
 
 
 def create_rag_blueprint(rag, processes, storage, pipeline):
@@ -74,7 +74,7 @@ def init_rag():
                     return jsonify("Starting initializing RAG component."), int(
                         HTTPResponses.OK
                     )
-                rag.active_by_process[process].switch_readiness()
+                rag.active_by_process[process].mark_ready()
                 return jsonify("Initialized RAG component."), int(HTTPResponses.OK)
             return jsonify(
                 f"Wrong content type '{request.headers.get('Content-Type')}'; need 'application/json'"
@@ -107,24 +107,42 @@ def rag_question():
             if not question:
                 return jsonify("No question supplied."), int(HTTPResponses.BAD_REQUEST)
 
+            chunks = active_rag.vectorstore.get_chunks(
+                question,
+                filter_by={"doc_id": doc_ids} if len(doc_ids) > 0 else None,
+                limit=doc_part_limit,
+            )
+            if not chunks:
+                logging.warning(
+                    "RAG vector store returned no chunks for process '%s'.", process
+                )
+                return jsonify(
+                    answer=no_source_answer(language),
+                    info=json.dumps({}, ensure_ascii=False),
+                ), int(HTTPResponses.OK)
+
             documents = list(
                 zip(
                     *itemgetter(1, -1)(
                         extract_text_from_highlights(
-                            active_rag.vectorstore.get_chunks(
-                                question,
-                                filter_by=(
-                                    {"doc_id": doc_ids} if len(doc_ids) > 0 else None
-                                ),
-                                limit=doc_part_limit,
-                            ),
+                            chunks,
                             token_limit=150,
                             lang=language,
                         )
                     )
                 )
             )
             rag_documents = active_rag.rag.documents_from(documents, concat_by="doc_id")
+            if not rag_documents:
+                logging.warning(
+                    "RAG chunk extraction produced no source documents for process '%s'.",
+                    process,
+                )
+                return jsonify(
+                    answer=no_source_answer(language),
+                    info=json.dumps({}, ensure_ascii=False),
+                ), int(HTTPResponses.OK)
+
             success, answer = active_rag.rag.build_and_invoke(
                 question, documents=rag_documents
             )

src/api/routes/status.py

@@ -48,9 +48,33 @@ def get_rag_status():
         process = string_conformity(request.args.get("process", "default"))
         active_rag = rag.active_by_process.get(process)
         if active_rag is not None:
-            return jsonify(active=active_rag.ready, name=process, error=None), int(
-                HTTPResponses.OK
-            )
+            vectorstore_index = getattr(active_rag.vectorstore, "index_name", None)
+            document_count = None
+            try:
+                vectorstore_filled = active_rag.vectorstore.is_filled()
+                document_count_getter = getattr(
+                    active_rag.vectorstore, "document_count", None
+                )
+                if document_count_getter is not None:
+                    document_count = document_count_getter()
+            except Exception as exc:
+                vectorstore_filled = False
+                error = f"Could not inspect RAG vector store: {exc}"
+            else:
+                error = active_rag.error
+                if active_rag.ready and not vectorstore_filled:
+                    error = (
+                        "RAG component is marked ready, but its vector store is empty."
+                    )
+            return jsonify(
+                active=active_rag.ready and vectorstore_filled,
+                name=process,
+                error=error,
+                initializing=active_rag.initializing,
+                vectorstore_filled=vectorstore_filled,
+                vectorstore_index=vectorstore_index,
+                vectorstore_document_count=document_count,
+            ), int(HTTPResponses.OK)
         err_string = "The RAG component is not initialized for this process."
         return jsonify(active=False, name=process, error=err_string), int(
             HTTPResponses.NOT_FOUND