diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml new file mode 100644 index 0000000..2768853 --- /dev/null +++ b/.github/workflows/smoke.yml @@ -0,0 +1,71 @@ +name: Streamlit smoke tests + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + smoke: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: | + requirements.txt + requirements-dev.txt + + - name: Install Python dependencies + run: pip install -r requirements-dev.txt + + - name: Install Playwright Chromium + run: python -m playwright install --with-deps chromium + + - name: Run Streamlit + Playwright smoke tests + run: python -m pytest tests/smoke -v + + openai-provider-smoke: + name: OpenAI provider smoke (optional, OPENAI_API_KEY) + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + steps: + - name: Check for OPENAI_API_KEY + id: key + run: | + if [ -n "$OPENAI_API_KEY" ]; then + echo "present=true" >> "$GITHUB_OUTPUT" + else + echo "present=false" >> "$GITHUB_OUTPUT" + echo "::notice title=OpenAI provider smoke skipped::OPENAI_API_KEY is not available to this run (not configured, or a fork or Dependabot PR). Run this workflow manually to check the live OpenAI path." + fi + + - if: steps.key.outputs.present == 'true' + uses: actions/checkout@v4 + + - if: steps.key.outputs.present == 'true' + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: | + requirements.txt + requirements-dev.txt + + - name: Install Python dependencies + if: steps.key.outputs.present == 'true' + run: pip install -r requirements-dev.txt + + - name: Run live OpenAI provider smoke tests + if: steps.key.outputs.present == 'true' + run: python -m pytest tests/provider -v -rs diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..6c991b7 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,84 @@ +# AGENTS.md + +Notes for contributors and coding agents working on this repo. + +## Layout + +- `chat_with_pdf_query.py`: default app. Uses `CouchbaseQueryVectorStore` (Query + Index services, Hyperscale/Composite vector indexes). +- `chat_with_pdf.py`: uses `CouchbaseSearchVectorStore` (Search service, needs `INDEX_NAME`). +- `tests/smoke/`: Streamlit + Playwright smoke tests for both entrypoints (no credentials). +- `tests/provider/`: optional live OpenAI provider smoke test (needs `OPENAI_API_KEY`, no Couchbase). +- `.github/workflows/smoke.yml`: runs the smoke tests on every PR and push to `main`, plus the optional provider job. + +## Test tiers + +| Tier | What | When | Required? | +| --- | --- | --- | --- | +| 1 | `tests/smoke` Streamlit + Playwright smoke, no secrets | Every PR, push to `main`, manual dispatch | Yes. The required PR check | +| 2 | `tests/provider` live OpenAI provider smoke | CI job `openai-provider-smoke` when `OPENAI_API_KEY` is available; skips with a notice otherwise | No. Optional signal | +| 3 | Full Couchbase-backed RAG run (checklist below) | Manually, by a maintainer, before merging dependency updates | Yes. Evidence goes in the PR | + +Tiers 1 and 2 do not replace tier 3. + +## Install + +```bash +pip install -r requirements-dev.txt # app deps + pytest + playwright +python -m playwright install --with-deps chromium +``` + +`requirements.txt` holds only the app's runtime dependencies. Dependabot bumps both files. + +## Smoke tests (CI, no credentials) + +```bash +python -m pytest tests/smoke -v +``` + +These tests need no Couchbase cluster and no OpenAI key. They start each app headless on a free port, open it in headless Chromium, and fail if a Python traceback or `st.exception` appears in the page or the server log. For each entrypoint they check: + +1. **Unconfigured start**: the real app with no settings loads, imports every real dependency, and shows the `OPENAI_API_KEY environment variable is not set` message. +2. **Auth gate**: with `AUTH_ENABLED=True`, the password form renders and rejects a wrong password. +3. **Stubbed full flow**: `tests/smoke/stubbed_app.py` runs the app with the Couchbase cluster, vector store, and cache, plus the OpenAI embeddings and chat model, swapped for in-memory LangChain fakes. The test checks that the expected controls render, uploads a small generated PDF (real `pypdf` loader and text splitter), asks a question, and asserts that the RAG answer used the retrieved PDF context and the pure-LLM answer did not. + +The tests set placeholder values for every app setting and use an isolated `HOME`, so a local `.streamlit/secrets.toml` or real credentials in your shell are never used. + +**What the smoke tests do not cover:** real Couchbase connectivity, vector index definitions, `langchain-couchbase` query/search behaviour against a cluster, OpenAI API compatibility (model names, request shapes), and the LLM cache against a real collection. The optional provider smoke covers the OpenAI part. The rest is why live validation stays manual (below). + +## OpenAI provider smoke (optional, needs `OPENAI_API_KEY`) + +```bash +OPENAI_API_KEY=... python -m pytest tests/provider -v -rs +``` + +This calls the real OpenAI API and does not touch Couchbase. It reads the `OpenAIEmbeddings(...)` and `ChatOpenAI(...)` arguments from both app entrypoints, so it checks the exact models and options the apps use: + +- **Embeddings**: `embed_query` returns a 1536-dim vector of finite floats (the README index definitions use 1536 dims). +- **Chat**: each distinct `ChatOpenAI` configuration returns a non-empty reply to a one-line prompt. + +Without `OPENAI_API_KEY` the tests are skipped with a clear reason. With an invalid key or a retired model they fail. In CI the `openai-provider-smoke` job uses the `OPENAI_API_KEY` Actions secret. Secrets are not passed to fork or Dependabot PRs, so there the job posts a "skipped" notice and passes. To check the live OpenAI path for such a PR, run the workflow manually (**Actions → Streamlit smoke tests → Run workflow**) on the PR branch. + +## Manual live validation for dependency-update PRs + +A green smoke run is necessary but not sufficient to merge a dependency bump (for example `langchain-*`, `couchbase`, `streamlit`, `pypdf`). Before merging, a maintainer runs the app against a real cluster and records this evidence in the PR: + +1. **App boot**: `streamlit run chat_with_pdf_query.py` (and `streamlit run chat_with_pdf.py` if the bump touches `langchain-couchbase`, `couchbase`, or Search) starts with no errors, and the installed versions match the PR (`pip freeze | grep -iE "langchain|couchbase|streamlit|pypdf|openai"`). +2. **Couchbase / vector index state**: the cluster version, the bucket/scope/collection used, and that the index exists. For the Query app that is the Hyperscale/Composite vector index from the README's "Index Configuration" section. For the Search app it is the Search index named by `INDEX_NAME` (see "Create the Search Index"). +3. **Provider credentials**: confirm that `OPENAI_API_KEY` was available and that the configured models were accessible. State only that it was set, never the value. +4. **Sample input / query**: the PDF you uploaded (name and approximate size) and the question you asked. +5. **Observed response**: the RAG answer and the pure-LLM answer. Ask the same question a second time to confirm the cached response is returned. +6. **Screenshots or traces** of the upload confirmation and the chat answers, when practical. + +## Required settings / secret names + +Set these in `.streamlit/secrets.toml` (copy `.streamlit/secrets.example.toml`) or as environment variables. Never commit real values, and never paste them into PRs, issues, or logs. + +| Name | Used by | +| --- | --- | +| `OPENAI_API_KEY` | both apps | +| `DB_CONN_STR`, `DB_USERNAME`, `DB_PASSWORD` | both apps | +| `DB_BUCKET`, `DB_SCOPE`, `DB_COLLECTION`, `CACHE_COLLECTION` | both apps | +| `INDEX_NAME` | `chat_with_pdf.py` only | +| `AUTH_ENABLED`, `LOGIN_PASSWORD` | optional app password gate | + +The required CI smoke job needs **no** repository secrets. The optional `openai-provider-smoke` job uses the `OPENAI_API_KEY` Actions secret when it is available. diff --git a/README.md b/README.md index 3041d2d..6f163b2 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,24 @@ All LLM responses are cached in the collection specified. If the same exact ques `pip install -r requirements.txt` +### Smoke tests + +Streamlit + Playwright smoke tests for both apps run in CI without any credentials: + +```bash +pip install -r requirements-dev.txt +python -m playwright install --with-deps chromium +python -m pytest tests/smoke -v +``` + +An optional live OpenAI provider check (no Couchbase) runs when `OPENAI_API_KEY` is set and is skipped otherwise: + +```bash +python -m pytest tests/provider -v -rs +``` + +See [AGENTS.md](AGENTS.md) for what each tier covers and the manual live-validation checklist for dependency updates. Neither test suite replaces a full Couchbase-backed RAG run. + ### Set the environment secrets Copy the `secrets.example.toml` file in `.streamlit` folder and rename it to `secrets.toml` and replace the placeholders with the actual values for your environment. diff --git a/chat_with_pdf.py b/chat_with_pdf.py index 28dc50f..6dac1e8 100644 --- a/chat_with_pdf.py +++ b/chat_with_pdf.py @@ -40,8 +40,9 @@ def save_to_vector_store(uploaded_file, vector_store): with open(temp_file_path, "wb") as f: f.write(uploaded_file.getvalue()) - loader = PyPDFLoader(temp_file_path) - docs = loader.load() + + loader = PyPDFLoader(temp_file_path) + docs = loader.load() text_splitter = RecursiveCharacterTextSplitter( chunk_size=1500, chunk_overlap=150 diff --git a/chat_with_pdf_query.py b/chat_with_pdf_query.py index c8b2e40..a4ca271 100644 --- a/chat_with_pdf_query.py +++ b/chat_with_pdf_query.py @@ -41,8 +41,9 @@ def save_to_vector_store(uploaded_file, vector_store): with open(temp_file_path, "wb") as f: f.write(uploaded_file.getvalue()) - loader = PyPDFLoader(temp_file_path) - docs = loader.load() + + loader = PyPDFLoader(temp_file_path) + docs = loader.load() text_splitter = RecursiveCharacterTextSplitter( chunk_size=1500, chunk_overlap=150 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..608a544 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,3 @@ +-r requirements.txt +pytest==9.1.1 +playwright==1.63.0 diff --git a/tests/provider/test_openai_provider.py b/tests/provider/test_openai_provider.py new file mode 100644 index 0000000..03ab1a9 --- /dev/null +++ b/tests/provider/test_openai_provider.py @@ -0,0 +1,62 @@ +"""Optional live OpenAI provider smoke test. Needs OPENAI_API_KEY; no Couchbase. + +The embedding and chat model settings are read from both app entrypoints, so +this checks the exact models and ChatOpenAI arguments the apps use. +""" + +import ast +import math +import os +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +APPS = ["chat_with_pdf_query.py", "chat_with_pdf.py"] + +# Both README index definitions use a 1536-dim vector field. +EMBEDDING_DIMS = 1536 + +pytestmark = pytest.mark.skipif( + not os.environ.get("OPENAI_API_KEY"), + reason="OPENAI_API_KEY is not set; skipping live OpenAI provider smoke test", +) + + +def _constructor_calls(name): + """Literal keyword arguments of every `name(...)` call in the apps.""" + calls = set() + for app in APPS: + for node in ast.walk(ast.parse((REPO_ROOT / app).read_text())): + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == name: + kwargs = {kw.arg: ast.literal_eval(kw.value) for kw in node.keywords} + calls.add(tuple(sorted(kwargs.items()))) + assert calls, f"no {name}(...) call found in {APPS}" + return [dict(c) for c in sorted(calls)] + + +def _ids(calls): + return [",".join(f"{k}={v}" for k, v in c.items()) or "defaults" for c in calls] + + +EMBEDDING_CALLS = _constructor_calls("OpenAIEmbeddings") +CHAT_CALLS = _constructor_calls("ChatOpenAI") + + +@pytest.mark.parametrize("kwargs", EMBEDDING_CALLS, ids=_ids(EMBEDDING_CALLS)) +def test_embeddings_return_numeric_vector(kwargs): + from langchain_openai import OpenAIEmbeddings + + vector = OpenAIEmbeddings(**kwargs).embed_query("Couchbase vector search") + assert len(vector) == EMBEDDING_DIMS + assert all(isinstance(x, float) and math.isfinite(x) for x in vector) + assert any(x != 0 for x in vector) + + +@pytest.mark.parametrize("kwargs", CHAT_CALLS, ids=_ids(CHAT_CALLS)) +def test_chat_model_returns_text(kwargs): + from langchain_openai import ChatOpenAI + + llm = ChatOpenAI(**kwargs, max_retries=1) + reply = llm.invoke("Reply with the single word: pong") + assert isinstance(reply.content, str) and reply.content.strip() diff --git a/tests/smoke/conftest.py b/tests/smoke/conftest.py new file mode 100644 index 0000000..ce2f027 --- /dev/null +++ b/tests/smoke/conftest.py @@ -0,0 +1,134 @@ +import os +import socket +import subprocess +import sys +import time +import urllib.request +from pathlib import Path + +import pytest +from playwright.sync_api import sync_playwright + +REPO_ROOT = Path(__file__).resolve().parents[2] +STUBBED_APP = Path(__file__).resolve().parent / "stubbed_app.py" +STARTUP_TIMEOUT_S = 60 + +# Variables the apps read. Cleared for every server so a developer's real +# credentials never leak into the smoke run. +APP_ENV_VARS = [ + "OPENAI_API_KEY", + "DB_CONN_STR", + "DB_USERNAME", + "DB_PASSWORD", + "DB_BUCKET", + "DB_SCOPE", + "DB_COLLECTION", + "CACHE_COLLECTION", + "INDEX_NAME", + "AUTH_ENABLED", + "LOGIN_PASSWORD", +] + +# Placeholder values for the stubbed run; nothing connects to them. +PLACEHOLDER_ENV = {name: f"smoke-{name.lower()}" for name in APP_ENV_VARS} +PLACEHOLDER_ENV["DB_CONN_STR"] = "couchbase://smoke.invalid" +PLACEHOLDER_ENV["AUTH_ENABLED"] = "False" + + +def _free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +class StreamlitServer: + def __init__(self, script_args, extra_env, tmp_home: Path): + self.port = _free_port() + self.url = f"http://127.0.0.1:{self.port}" + self.log_path = tmp_home / "streamlit.log" + env = {k: v for k, v in os.environ.items() if k not in APP_ENV_VARS} + env.update(extra_env) + # Isolated HOME so ~/.streamlit/secrets.toml and config are not picked up. + env["HOME"] = str(tmp_home) + script, *script_argv = script_args + cmd = [ + sys.executable, "-m", "streamlit", "run", script, + "--server.headless", "true", + "--server.port", str(self.port), + "--server.address", "127.0.0.1", + "--server.fileWatcherType", "none", + "--browser.gatherUsageStats", "false", + ] + if script_argv: + cmd += ["--", *script_argv] + self._log = open(self.log_path, "w") + self.proc = subprocess.Popen( + cmd, cwd=REPO_ROOT, env=env, stdout=self._log, stderr=subprocess.STDOUT + ) + self._wait_healthy() + + def _wait_healthy(self): + deadline = time.time() + STARTUP_TIMEOUT_S + while time.time() < deadline: + if self.proc.poll() is not None: + raise RuntimeError(f"Streamlit exited early:\n{self.log()}") + try: + with urllib.request.urlopen(f"{self.url}/_stcore/health", timeout=2) as r: + if r.status == 200: + return + except OSError: + pass + time.sleep(0.5) + raise RuntimeError(f"Streamlit not healthy after {STARTUP_TIMEOUT_S}s:\n{self.log()}") + + def log(self) -> str: + self._log.flush() + return self.log_path.read_text() + + def stop(self): + self.proc.terminate() + try: + self.proc.wait(timeout=10) + except subprocess.TimeoutExpired: + self.proc.kill() + self._log.close() + + +@pytest.fixture(scope="session") +def browser(): + with sync_playwright() as p: + b = p.chromium.launch() + yield b + b.close() + + +@pytest.fixture +def page(browser): + context = browser.new_context() + pg = context.new_page() + yield pg + context.close() + + +@pytest.fixture +def start_app(tmp_path): + """Start an app: start_app("chat_with_pdf_query.py", stubbed=True).""" + servers = [] + + def _start(entrypoint: str, stubbed: bool, env=None): + if stubbed: + args = [str(STUBBED_APP), entrypoint] + extra_env = dict(PLACEHOLDER_ENV) + else: + args = [entrypoint] + extra_env = {} + extra_env.update(env or {}) + home = tmp_path / f"home{len(servers)}" + home.mkdir() + server = StreamlitServer(args, extra_env, home) + servers.append(server) + return server + + yield _start + for server in servers: + server.stop() diff --git a/tests/smoke/stubbed_app.py b/tests/smoke/stubbed_app.py new file mode 100644 index 0000000..a57ec2c --- /dev/null +++ b/tests/smoke/stubbed_app.py @@ -0,0 +1,39 @@ +"""Run a demo entrypoint with Couchbase and OpenAI replaced by in-memory fakes. + +Used only by the smoke tests, so the full UI (PDF upload, retrieval, chat) can be +exercised in CI without a Couchbase cluster or OpenAI credentials: + + streamlit run tests/smoke/stubbed_app.py -- chat_with_pdf_query.py + +Streamlit re-executes this script on every interaction, so the fakes are +re-applied before each run of the target app. +""" + +import runpy +import sys +from typing import Any + +import couchbase.cluster +import langchain_couchbase +import langchain_couchbase.cache +import langchain_couchbase.vectorstores +import langchain_openai +from langchain_core.caches import InMemoryCache +from langchain_core.embeddings import DeterministicFakeEmbedding +from langchain_core.vectorstores import InMemoryVectorStore + +from support import FakeChatModel, FakeCluster + + +def fake_vector_store(**kwargs: Any) -> InMemoryVectorStore: + return InMemoryVectorStore(embedding=kwargs["embedding"]) + + +couchbase.cluster.Cluster = FakeCluster +langchain_couchbase.CouchbaseQueryVectorStore = fake_vector_store +langchain_couchbase.vectorstores.CouchbaseSearchVectorStore = fake_vector_store +langchain_couchbase.cache.CouchbaseCache = lambda **kwargs: InMemoryCache() +langchain_openai.OpenAIEmbeddings = lambda **kwargs: DeterministicFakeEmbedding(size=64) +langchain_openai.ChatOpenAI = lambda **kwargs: FakeChatModel() + +runpy.run_path(sys.argv[1], run_name="__main__") diff --git a/tests/smoke/support.py b/tests/smoke/support.py new file mode 100644 index 0000000..b0318e1 --- /dev/null +++ b/tests/smoke/support.py @@ -0,0 +1,70 @@ +"""Shared helpers for the smoke tests (importable without side effects).""" + +from pathlib import Path +from typing import Any, List, Optional + +from langchain_core.language_models.chat_models import BaseChatModel +from langchain_core.messages import AIMessage, BaseMessage +from langchain_core.outputs import ChatGeneration, ChatResult + +# Marker text embedded in the smoke-test PDF. The fake LLM reports whether it +# saw it in the prompt, which proves the PDF went through the retriever. +PDF_MARKER = "SMOKE-MARKER-7f3a" + + +class FakeCluster: + def __init__(self, *args: Any, **kwargs: Any) -> None: + pass + + def wait_until_ready(self, *args: Any, **kwargs: Any) -> None: + pass + + +class FakeChatModel(BaseChatModel): + """Answers deterministically and reports whether the PDF context was used.""" + + @property + def _llm_type(self) -> str: + return "smoke-fake-chat" + + def _generate( + self, + messages: List[BaseMessage], + stop: Optional[List[str]] = None, + run_manager: Any = None, + **kwargs: Any, + ) -> ChatResult: + prompt = "\n".join(str(m.content) for m in messages) + if PDF_MARKER in prompt: + answer = f"Fake LLM answer using PDF context ({PDF_MARKER})." + else: + answer = "Fake LLM answer without PDF context." + return ChatResult(generations=[ChatGeneration(message=AIMessage(content=answer))]) + + +def make_pdf(path: Path, text: str) -> Path: + """Write a minimal single-page PDF containing `text`.""" + content = f"BT /F1 12 Tf 72 720 Td ({text}) Tj ET".encode() + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + b"/Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>", + b"<< /Length %d >>\nstream\n" % len(content) + content + b"\nendstream", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ] + out = bytearray(b"%PDF-1.4\n") + offsets = [] + for i, obj in enumerate(objects, start=1): + offsets.append(len(out)) + out += b"%d 0 obj\n" % i + obj + b"\nendobj\n" + xref = len(out) + out += b"xref\n0 %d\n0000000000 65535 f \n" % (len(objects) + 1) + for off in offsets: + out += b"%010d 00000 n \n" % off + out += b"trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n" % ( + len(objects) + 1, + xref, + ) + path.write_bytes(bytes(out)) + return path diff --git a/tests/smoke/test_streamlit_apps.py b/tests/smoke/test_streamlit_apps.py new file mode 100644 index 0000000..1af3b95 --- /dev/null +++ b/tests/smoke/test_streamlit_apps.py @@ -0,0 +1,96 @@ +"""Streamlit + Playwright smoke tests for both RAG demo entrypoints. + +These run without Couchbase or OpenAI credentials. Live RAG validation is +still manual; see AGENTS.md. +""" + +import re + +import pytest +from playwright.sync_api import Page, expect + +from support import PDF_MARKER, make_pdf + +APPS = { + "chat_with_pdf_query.py": "Chat with PDF (Query Vector Store)", + "chat_with_pdf.py": "Chat with PDF (Search Vector Store)", +} +TIMEOUT_MS = 30_000 + + +def wait_for_app(page: Page, url: str): + page.goto(url) + expect(page.get_by_test_id("stApp")).to_be_visible(timeout=TIMEOUT_MS) + # Streamlit shows a "Running..." status widget while the script executes. + expect(page.get_by_test_id("stStatusWidget")).to_have_count(0, timeout=TIMEOUT_MS) + + +def assert_no_errors(page: Page, server): + expect(page.get_by_test_id("stException")).to_have_count(0) + expect(page.get_by_text("Traceback (most recent call last)")).to_have_count(0) + assert "Traceback (most recent call last)" not in server.log(), server.log() + + +@pytest.mark.parametrize("entrypoint", APPS) +def test_unconfigured_app_reports_missing_settings(page, start_app, entrypoint): + """Real dependencies import and the app fails gracefully without settings.""" + server = start_app(entrypoint, stubbed=False) + wait_for_app(page, server.url) + + expect( + page.get_by_text("OPENAI_API_KEY environment variable is not set") + ).to_be_visible(timeout=TIMEOUT_MS) + assert_no_errors(page, server) + + +@pytest.mark.parametrize("entrypoint", APPS) +def test_auth_gate_renders(page, start_app, entrypoint): + server = start_app(entrypoint, stubbed=True, env={"AUTH_ENABLED": "True"}) + wait_for_app(page, server.url) + + page.get_by_label("Enter password").fill("wrong-password") + page.get_by_role("button", name="Submit").click() + expect(page.get_by_text("Incorrect password")).to_be_visible(timeout=TIMEOUT_MS) + expect(page.get_by_text(APPS[entrypoint])).to_have_count(0) + assert_no_errors(page, server) + + +@pytest.mark.parametrize("entrypoint", APPS) +def test_stubbed_app_upload_and_chat(page, start_app, entrypoint, tmp_path): + """Full UI flow with Couchbase/OpenAI replaced by in-memory fakes.""" + server = start_app(entrypoint, stubbed=True) + wait_for_app(page, server.url) + + # Expected controls and text render. + expect(page.get_by_role("heading", name=APPS[entrypoint])).to_be_visible() + expect(page.get_by_text("Upload your PDF")).to_be_visible() + expect(page.get_by_role("button", name="Upload", exact=True)).to_be_visible() + expect(page.get_by_text("How does it work?")).to_be_visible() + expect( + page.get_by_text("Hi, I'm a chatbot who can chat with the PDF.") + ).to_be_visible() + chat_input = page.get_by_placeholder("Ask a question based on the PDF") + expect(chat_input).to_be_visible() + assert_no_errors(page, server) + + # Upload a PDF: exercises pypdf loading, text splitting and indexing. + pdf = make_pdf(tmp_path / "smoke.pdf", f"The secret code is {PDF_MARKER}.") + page.locator('input[type="file"]').set_input_files(pdf) + expect(page.get_by_text("smoke.pdf")).to_be_visible(timeout=TIMEOUT_MS) + page.get_by_role("button", name="Upload", exact=True).click() + expect( + page.get_by_text(re.compile(r"PDF loaded into vector store in \d+ documents")) + ).to_be_visible(timeout=TIMEOUT_MS) + + # Ask a question: RAG answer must include retrieved PDF context, the + # pure-LLM answer must not. + chat_input.fill("What is the secret code?") + chat_input.press("Enter") + expect(page.get_by_text("What is the secret code?")).to_be_visible(timeout=TIMEOUT_MS) + expect( + page.get_by_text(f"Fake LLM answer using PDF context ({PDF_MARKER}).") + ).to_be_visible(timeout=TIMEOUT_MS) + expect(page.get_by_text("Fake LLM answer without PDF context.")).to_be_visible( + timeout=TIMEOUT_MS + ) + assert_no_errors(page, server)