diff --git a/app/platform/[tenantKey]/[mentorId]/_components/nav-bar/index.tsx b/app/platform/[tenantKey]/[mentorId]/_components/nav-bar/index.tsx index 39fb1f02..c57c2ef0 100644 --- a/app/platform/[tenantKey]/[mentorId]/_components/nav-bar/index.tsx +++ b/app/platform/[tenantKey]/[mentorId]/_components/nav-bar/index.tsx @@ -122,6 +122,7 @@ export const NEW_CHAT_NAV_ITEM = { export const ANALYTICS_NAV_ITEM: MentorSegment = { value: 'analytics', label: 'Analytics', + labelKey: 'analytics', icon: LineChart, userTypes: [UserType.FREE_TRIAL, UserType.ADMIN], rbacResource: (mentorDbId) => `/mentors/${mentorDbId}/#view_analytics`, @@ -135,6 +136,9 @@ export const ANALYTICS_NAV_ITEM: MentorSegment = { export function NavBar() { const t = useTranslations('navBarIndex'); + // Segment labels + category titles live in the shared `header` namespace + // (same keys header.tsx uses) so both nav surfaces stay in sync. + const tHeader = useTranslations('header'); const [openModal, setOpenModal] = React.useState(false); const dispatch = useAppDispatch(); const selectedAnalyticsMentor = useAppSelector(selectSelectedMentor); @@ -350,11 +354,11 @@ export function NavBar() { .filter((s) => s.navCategory) .map((s) => ({ value: s.value, - label: s.label, + label: tHeader(s.labelKey), icon: s.icon, category: s.navCategory, })); - }, [filteredSegments]); + }, [filteredSegments, tHeader]); const showForkButton = !userIsStudent && @@ -585,7 +589,10 @@ export function NavBar() { > + MENTOR_SEGMENT_NAV_CATEGORIES.map((c) => ({ + key: c.key, + title: tHeader(c.titleKey), + })) as ReadonlyArray } items={categorizedDropdownItems} onItemSelect={handleSegmentClick} diff --git a/components/modals/edit-mentor-modal/__tests__/index.test.tsx b/components/modals/edit-mentor-modal/__tests__/index.test.tsx index e4bbf55c..35a5cb8e 100644 --- a/components/modals/edit-mentor-modal/__tests__/index.test.tsx +++ b/components/modals/edit-mentor-modal/__tests__/index.test.tsx @@ -239,6 +239,7 @@ vi.mock('../tabs', () => ({ FlowTab: () =>
Flow Tab
, HistoryTab: () =>
History Tab
, DatasetsTab: () =>
Datasets Tab
, + EvaluationTab: () =>
Evaluation Tab
, ApiTab: () =>
API Tab
, EmbedTab: () =>
Embed Tab
, AccessTab: () =>
Access Tab
, @@ -499,6 +500,7 @@ describe('EditMentorModal', () => { return { value, label: value, + labelKey: value, icon: SettingsIcon, userTypes: [UserType.ADMIN, UserType.FREE_TRIAL], permissionFieldsCheck: [], @@ -692,7 +694,6 @@ describe('EditMentorModal', () => { }), makeSegment(MODALS.EDIT_MENTOR.tabs.llm, { navCategory: 'configurations', - label: 'LLM Segment', }), ]; @@ -703,8 +704,10 @@ describe('EditMentorModal', () => { , ); + // Labels render translated from the `header` namespace, so the llm + // segment shows as "LLM" regardless of the fixture's raw label. const llmTriggers = await screen.findAllByRole('tab', { - name: 'LLM Segment', + name: 'LLM', }); await user.click(llmTriggers[0]); @@ -780,9 +783,10 @@ describe('EditMentorModal', () => { // the rendered segment list: Settings (the only configurations // segment) must be present. await waitFor(() => { + // Rendered via the `header` translation namespace — "Settings", not + // the raw segment value. expect( - screen.getAllByRole('tab', { name: MODALS.EDIT_MENTOR.tabs.settings }) - .length, + screen.getAllByRole('tab', { name: 'Settings' }).length, ).toBeGreaterThan(0); }); }); diff --git a/components/modals/edit-mentor-modal/edit-mentor-modal.test.tsx b/components/modals/edit-mentor-modal/edit-mentor-modal.test.tsx index 5aea542a..10ecf0c9 100644 --- a/components/modals/edit-mentor-modal/edit-mentor-modal.test.tsx +++ b/components/modals/edit-mentor-modal/edit-mentor-modal.test.tsx @@ -193,6 +193,7 @@ vi.mock('./tabs', () => ({ TasksTab: () =>
Tasks Tab
, HistoryTab: () =>
History Tab
, DatasetsTab: () =>
Datasets Tab
, + EvaluationTab: () =>
Evaluation Tab
, ApiTab: () =>
API Tab
, EmbedTab: () =>
Embed Tab
, AccessTab: () =>
Access Tab
, diff --git a/components/modals/edit-mentor-modal/index.tsx b/components/modals/edit-mentor-modal/index.tsx index 375a93ce..8c9a45a5 100644 --- a/components/modals/edit-mentor-modal/index.tsx +++ b/components/modals/edit-mentor-modal/index.tsx @@ -24,6 +24,7 @@ import { // FlowTab, HistoryTab, DatasetsTab, + EvaluationTab, ApiTab, EmbedTab, AccessTab, @@ -79,6 +80,7 @@ export const EDIT_MENTOR_TAB_COMPONENTS: Record = { [MODALS.EDIT_MENTOR.tabs.history]: , [MODALS.EDIT_MENTOR.tabs.audit_log]: , [MODALS.EDIT_MENTOR.tabs.datasets]: , + [MODALS.EDIT_MENTOR.tabs.evaluation]: , [MODALS.EDIT_MENTOR.tabs.api]: , [MODALS.EDIT_MENTOR.tabs.embed]: , [MODALS.EDIT_MENTOR.tabs.voice]: , @@ -89,6 +91,9 @@ export const EDIT_MENTOR_TAB_COMPONENTS: Record = { export function EditMentorModal({ isOpen, onClose }: Props) { const t = useTranslations('editMentorModalIndex'); + // Segment labels + category titles live in the shared `header` namespace + // (same keys header.tsx uses) so both nav surfaces stay in sync. + const tHeader = useTranslations('header'); const { changeModalTab, getEditMentorTab } = useNavigate(); const { filteredSegments, @@ -386,7 +391,7 @@ export function EditMentorModal({ isOpen, onClose }: Props) { : 'text-gray-600 hover:bg-gray-50 dark:text-gray-300 dark:hover:bg-gray-700/50', )} > - {category.title} + {tHeader(category.titleKey)} ); })} @@ -409,7 +414,9 @@ export function EditMentorModal({ isOpen, onClose }: Props) { className="mr-3 h-4 w-4 flex-shrink-0" aria-hidden="true" /> - {tab.label} + + {tHeader(tab.labelKey)} + ))} @@ -452,7 +459,7 @@ export function EditMentorModal({ isOpen, onClose }: Props) { : 'text-gray-600 hover:bg-gray-50', )} > - {category.title} + {tHeader(category.titleKey)} ); })} @@ -476,7 +483,7 @@ export function EditMentorModal({ isOpen, onClose }: Props) { className="h-3 w-3 sm:h-4 sm:w-4" aria-hidden="true" /> - {tab.label} + {tHeader(tab.labelKey)} ))} diff --git a/components/modals/edit-mentor-modal/tabs/evaluation-tab/index.test.tsx b/components/modals/edit-mentor-modal/tabs/evaluation-tab/index.test.tsx new file mode 100644 index 00000000..409f0473 --- /dev/null +++ b/components/modals/edit-mentor-modal/tabs/evaluation-tab/index.test.tsx @@ -0,0 +1,280 @@ +import React from 'react'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { render, screen, cleanup } from '@testing-library/react'; + +// ============================================================================ +// MOCKS +// ============================================================================ +// +// The EvaluationTab is a thin wrapper that reads route params + redux state, +// resolves the mentor's `mentor_unique_id`, then renders the packaged +// `AgentEvaluationTab` inside an `AgentSettingsProvider`. The tests stub each +// dependency so we can assert the wrapper's plumbing without bringing up the +// real data layer / provider tree. + +const mockUseParams = vi.fn(); +const mockUseUsername = vi.fn(); +const mockUseAppSelector = vi.fn(); +const mockUseShowFreeTrialDialog = vi.fn(); +const mockUseGetMentorSettingsQuery = vi.fn(); +const mockEnableRBAC = vi.fn(); +const mockGetLLMProviderDetails = vi.fn(); +const mockExecuteWithTrialCheck = vi.fn((fn: () => unknown) => fn?.()); +const mockAgentSettingsProvider = vi.fn(); +const mockAgentEvaluationTab = vi.fn(); +const mockIblPagination = vi.fn(); + +vi.mock('next/navigation', () => ({ + useParams: () => mockUseParams(), +})); + +vi.mock('@/hooks/use-user', () => ({ + useUsername: () => mockUseUsername(), +})); + +vi.mock('@/lib/hooks', () => ({ + useAppSelector: (selector: unknown) => mockUseAppSelector(selector), +})); + +vi.mock('@/features/rbac/rbac-slice', () => ({ + selectRbacPermissions: vi.fn(() => 'selectRbacPermissions'), +})); + +vi.mock('@/hooks/user-user-actions', () => ({ + useShowFreeTrialDialog: () => mockUseShowFreeTrialDialog(), +})); + +vi.mock('@/lib/config', () => ({ + config: { + enableRBAC: () => mockEnableRBAC(), + }, +})); + +vi.mock('@/lib/utils', () => ({ + getLLMProviderDetails: (...args: unknown[]) => + mockGetLLMProviderDetails(...args), +})); + +vi.mock('@iblai/iblai-js/data-layer', () => ({ + useGetMentorSettingsQuery: ( + args: Record, + options: Record, + ) => mockUseGetMentorSettingsQuery(args, options), +})); + +vi.mock('@/components/ibl-pagination', () => { + const Pagination = (props: unknown) => { + mockIblPagination(props); + return
; + }; + return { __esModule: true, default: Pagination }; +}); + +vi.mock('@iblai/iblai-js/web-containers/next', () => ({ + AgentSettingsProvider: ({ + children, + ...value + }: React.PropsWithChildren>) => { + mockAgentSettingsProvider(value); + return ( +
+ {children} +
+ ); + }, + AgentEvaluationTab: (props: Record) => { + mockAgentEvaluationTab(props); + return
; + }, +})); + +// Import after mocks so the module under test picks up the stubs. +import { EvaluationTab } from './index'; + +// ============================================================================ +// HELPERS +// ============================================================================ + +const setDefaults = () => { + mockUseParams.mockReturnValue({ + tenantKey: 'tenant-x', + mentorId: 'mentor-slug', + }); + mockUseUsername.mockReturnValue('alice'); + mockUseAppSelector.mockReturnValue({ read: true }); + mockUseShowFreeTrialDialog.mockReturnValue({ + executeWithTrialCheck: mockExecuteWithTrialCheck, + }); + mockEnableRBAC.mockReturnValue(true); + mockUseGetMentorSettingsQuery.mockReturnValue({ + data: { mentor_unique_id: 'mentor-uid-123' }, + }); +}; + +// ============================================================================ +// TESTS +// ============================================================================ + +describe('EvaluationTab', () => { + beforeEach(() => { + cleanup(); + vi.clearAllMocks(); + setDefaults(); + }); + + afterEach(() => { + cleanup(); + }); + + describe('Provider plumbing', () => { + it('renders the AgentEvaluationTab inside the AgentSettingsProvider', () => { + render(); + const provider = screen.getByTestId('agent-settings-provider'); + expect(provider).toBeInTheDocument(); + expect( + provider.querySelector('[data-testid="agent-evaluation-tab"]'), + ).not.toBeNull(); + }); + + it('forwards tenantKey, username, enableRBAC, rbacPermissions, and executeGatedAction', () => { + render(); + const value = mockAgentSettingsProvider.mock.calls[0][0]; + expect(value).toMatchObject({ + tenantKey: 'tenant-x', + username: 'alice', + enableRBAC: true, + rbacPermissions: { read: true }, + }); + expect(typeof value.executeGatedAction).toBe('function'); + + // executeGatedAction should delegate to executeWithTrialCheck + const sentinel = vi.fn().mockReturnValue('return-value'); + const result = value.executeGatedAction(sentinel); + expect(mockExecuteWithTrialCheck).toHaveBeenCalledWith(sentinel); + expect(sentinel).toHaveBeenCalledTimes(1); + expect(result).toBe('return-value'); + }); + + it('passes getLLMProviderDetails and IblPagination through to AgentEvaluationTab', () => { + render(); + const props = mockAgentEvaluationTab.mock.calls[0][0]; + expect(typeof props.getLLMProviderDetails).toBe('function'); + expect(typeof props.PaginationComponent).toBe('function'); + + // The forwarded getLLMProviderDetails should delegate to the local helper. + props.getLLMProviderDetails('openai', 'gpt-4'); + expect(mockGetLLMProviderDetails).toHaveBeenCalledWith('openai', 'gpt-4'); + }); + + it('reads RBAC permissions via selectRbacPermissions', () => { + render(); + // The mock recorded whatever selector was passed to useAppSelector; + // it should be the imported selectRbacPermissions reference. + const selectorArg = mockUseAppSelector.mock.calls[0][0]; + expect(typeof selectorArg).toBe('function'); + expect((selectorArg as () => unknown)()).toBe('selectRbacPermissions'); + }); + }); + + describe('mentorUniqueId resolution', () => { + it('uses mentor_unique_id from settings when present', () => { + render(); + expect(mockAgentSettingsProvider).toHaveBeenCalledWith( + expect.objectContaining({ mentorId: 'mentor-uid-123' }), + ); + }); + + it('falls back to the route mentorId when mentor_unique_id is missing', () => { + mockUseGetMentorSettingsQuery.mockReturnValue({ + data: { mentor_unique_id: null }, + }); + render(); + expect(mockAgentSettingsProvider).toHaveBeenCalledWith( + expect.objectContaining({ mentorId: 'mentor-slug' }), + ); + }); + + it('falls back to the route mentorId when the settings query returns no data', () => { + mockUseGetMentorSettingsQuery.mockReturnValue({ data: undefined }); + render(); + expect(mockAgentSettingsProvider).toHaveBeenCalledWith( + expect.objectContaining({ mentorId: 'mentor-slug' }), + ); + }); + + it("falls back to '' when both mentor_unique_id and the route mentorId are missing", () => { + mockUseParams.mockReturnValue({ tenantKey: 'tenant-x' }); + mockUseGetMentorSettingsQuery.mockReturnValue({ data: undefined }); + render(); + expect(mockAgentSettingsProvider).toHaveBeenCalledWith( + expect.objectContaining({ mentorId: '' }), + ); + }); + }); + + describe('Mentor settings query', () => { + it('passes mentor + org + userId to useGetMentorSettingsQuery', () => { + render(); + expect(mockUseGetMentorSettingsQuery).toHaveBeenCalledWith( + { mentor: 'mentor-slug', org: 'tenant-x', userId: 'alice' }, + expect.objectContaining({ skip: false }), + ); + }); + + it('skips the query when mentorId is missing', () => { + mockUseParams.mockReturnValue({ tenantKey: 'tenant-x' }); + render(); + const opts = mockUseGetMentorSettingsQuery.mock.calls[0][1]; + expect(opts.skip).toBe(true); + }); + + it('skips the query when tenantKey is missing', () => { + mockUseParams.mockReturnValue({ mentorId: 'mentor-slug' }); + render(); + const opts = mockUseGetMentorSettingsQuery.mock.calls[0][1]; + expect(opts.skip).toBe(true); + }); + + it('skips the query when username is missing', () => { + mockUseUsername.mockReturnValue(undefined); + render(); + const opts = mockUseGetMentorSettingsQuery.mock.calls[0][1]; + expect(opts.skip).toBe(true); + // Provider still mounts but tenantKey/username fall back to ''. + expect(mockAgentSettingsProvider).toHaveBeenCalledWith( + expect.objectContaining({ username: '' }), + ); + }); + + it("passes empty string for username when it's undefined", () => { + mockUseUsername.mockReturnValue(undefined); + render(); + const args = mockUseGetMentorSettingsQuery.mock.calls[0][0]; + expect(args.userId).toBe(''); + }); + }); + + describe('Empty / fallback inputs', () => { + it('passes empty string for tenantKey when route param is missing', () => { + mockUseParams.mockReturnValue({ mentorId: 'mentor-slug' }); + render(); + expect(mockAgentSettingsProvider).toHaveBeenCalledWith( + expect.objectContaining({ tenantKey: '' }), + ); + }); + + it('reflects enableRBAC=false when config returns false', () => { + mockEnableRBAC.mockReturnValue(false); + render(); + expect(mockAgentSettingsProvider).toHaveBeenCalledWith( + expect.objectContaining({ enableRBAC: false }), + ); + }); + }); +}); diff --git a/components/modals/edit-mentor-modal/tabs/evaluation-tab/index.tsx b/components/modals/edit-mentor-modal/tabs/evaluation-tab/index.tsx new file mode 100644 index 00000000..d751cb55 --- /dev/null +++ b/components/modals/edit-mentor-modal/tabs/evaluation-tab/index.tsx @@ -0,0 +1,55 @@ +'use client'; + +import { useParams } from 'next/navigation'; + +import { + AgentEvaluationTab, + AgentSettingsProvider, +} from '@iblai/iblai-js/web-containers/next'; + +import IblPagination from '@/components/ibl-pagination'; +import { useGetMentorSettingsQuery } from '@iblai/iblai-js/data-layer'; +import { selectRbacPermissions } from '@/features/rbac/rbac-slice'; +import { useAppSelector } from '@/lib/hooks'; +import { useUsername } from '@/hooks/use-user'; +import { useShowFreeTrialDialog } from '@/hooks/user-user-actions'; +import { config } from '@/lib/config'; +import { TenantKeyMentorIdParams } from '@/lib/types'; +import { getLLMProviderDetails } from '@/lib/utils'; + +export function EvaluationTab() { + const { tenantKey, mentorId } = useParams(); + const username = useUsername(); + const rbacPermissions = useAppSelector(selectRbacPermissions); + const { executeWithTrialCheck } = useShowFreeTrialDialog(); + + const { data: mentorSettings } = useGetMentorSettingsQuery( + { + mentor: mentorId, + org: tenantKey, + // @ts-expect-error userId is not part of the typed args but the API accepts it + userId: username ?? '', + }, + { skip: !mentorId || !tenantKey || !username }, + ); + + // AgentEvaluationTab filters runs by `meta.mentor_unique_id === mentorId`, + // so the provider's mentorId must be the mentor's unique_id (not the slug). + const mentorUniqueId = mentorSettings?.mentor_unique_id ?? mentorId ?? ''; + + return ( + + + + ); +} diff --git a/components/modals/edit-mentor-modal/tabs/index.ts b/components/modals/edit-mentor-modal/tabs/index.ts index a6dc86b4..4d9cf065 100644 --- a/components/modals/edit-mentor-modal/tabs/index.ts +++ b/components/modals/edit-mentor-modal/tabs/index.ts @@ -9,6 +9,7 @@ export * from './tasks-tab'; export * from './flow-tab'; export * from './history-tab'; export * from './datasets-tab'; +export * from './evaluation-tab'; export * from './api-tab'; export * from './embed-tab'; export * from './access-tab'; diff --git a/components/modals/edit-mentor-modal/tabs/memory-tab/__tests__/memory-tab.test.tsx b/components/modals/edit-mentor-modal/tabs/memory-tab/__tests__/memory-tab.test.tsx index 1e3656a9..32bfd749 100644 --- a/components/modals/edit-mentor-modal/tabs/memory-tab/__tests__/memory-tab.test.tsx +++ b/components/modals/edit-mentor-modal/tabs/memory-tab/__tests__/memory-tab.test.tsx @@ -1,5 +1,6 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; -import { render, screen, fireEvent } from '@testing-library/react'; +import { render, screen, fireEvent, waitFor } from '@testing-library/react'; +import { toast } from 'sonner'; import { MemoryTab } from '../index'; @@ -179,4 +180,43 @@ describe('MemoryTab', () => { ); }); }); + + describe('Toggle failure & fallbacks', () => { + it('rolls the toggle back and surfaces the error toast when the PATCH fails', async () => { + mockEditMentor.mockReturnValue({ + unwrap: vi.fn().mockRejectedValue(new Error('network')), + }); + render(); + + const toggle = screen.getByTestId('memory-capability-toggle'); + fireEvent.click(toggle); + // Optimistic flip happens synchronously… + expect(toggle).toBeChecked(); + + // …then the rejected unwrap rolls it back and toasts. + await waitFor(() => { + expect(toggle).not.toBeChecked(); + }); + expect(toast.error).toHaveBeenCalled(); + expect(toast.success).not.toHaveBeenCalled(); + }); + + it('treats a settings payload without enable_memory_component as off', () => { + setupDefaults({ mentor: {} }); + render(); + + expect(screen.getByTestId('memory-capability-toggle')).not.toBeChecked(); + }); + + it('sends an empty userId when no username is available', () => { + setupDefaults({ username: null }); + render(); + + fireEvent.click(screen.getByTestId('memory-capability-toggle')); + + expect(mockEditMentor).toHaveBeenCalledWith( + expect.objectContaining({ userId: '' }), + ); + }); + }); }); diff --git a/components/modals/edit-mentor-modal/tabs/memory-tab/index.tsx b/components/modals/edit-mentor-modal/tabs/memory-tab/index.tsx index 2ad1124e..33930f2e 100644 --- a/components/modals/edit-mentor-modal/tabs/memory-tab/index.tsx +++ b/components/modals/edit-mentor-modal/tabs/memory-tab/index.tsx @@ -50,6 +50,7 @@ export function MemoryTab() { org: tenantKey, // @ts-ignore userId is accepted at runtime userId: username ?? '', + // @ts-ignore enable_memory_component exists on the API but not in the typed request formData: { enable_memory_component: checked }, }).unwrap(); toast.success(t('toggleSuccess')); diff --git a/components/modals/edit-mentor-modal/tabs/sandbox-tab.test.tsx b/components/modals/edit-mentor-modal/tabs/sandbox-tab.test.tsx index 782faa7d..c387923e 100644 --- a/components/modals/edit-mentor-modal/tabs/sandbox-tab.test.tsx +++ b/components/modals/edit-mentor-modal/tabs/sandbox-tab.test.tsx @@ -1,6 +1,13 @@ import React from 'react'; import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import { render, screen, cleanup, fireEvent } from '@testing-library/react'; +import { + render, + screen, + cleanup, + fireEvent, + waitFor, +} from '@testing-library/react'; +import { toast } from 'sonner'; import { SandboxTab } from './sandbox-tab'; @@ -190,6 +197,45 @@ describe('SandboxTab', () => { }), ); }); + + it('rolls the toggle back and surfaces the error toast when the PATCH fails', async () => { + mockEditMentor.mockReturnValue({ + unwrap: vi.fn().mockRejectedValue(new Error('network')), + }); + render(); + + const toggle = screen.getByTestId('sandbox-capability-toggle'); + fireEvent.click(toggle); + // Optimistic flip happens synchronously… + expect(toggle).toBeChecked(); + + // …then the rejected unwrap rolls it back and toasts. + await waitFor(() => { + expect(toggle).not.toBeChecked(); + }); + expect(toast.error).toHaveBeenCalled(); + expect(toast.success).not.toHaveBeenCalled(); + }); + + it('treats a settings payload without enable_claw as off', () => { + mockGetMentorSettingsQuery.mockReturnValue({ data: {} }); + + render(); + + expect(screen.getByTestId('sandbox-capability-toggle')).not.toBeChecked(); + }); + + it('sends an empty userId when no username is available', () => { + mockUsername.mockReturnValue(null); + + render(); + + fireEvent.click(screen.getByTestId('sandbox-capability-toggle')); + + expect(mockEditMentor).toHaveBeenCalledWith( + expect.objectContaining({ userId: '' }), + ); + }); }); describe('Active mentor id resolution', () => { diff --git a/components/modals/edit-mentor-modal/tabs/sandbox-tab.tsx b/components/modals/edit-mentor-modal/tabs/sandbox-tab.tsx index 5735590c..d31229d7 100644 --- a/components/modals/edit-mentor-modal/tabs/sandbox-tab.tsx +++ b/components/modals/edit-mentor-modal/tabs/sandbox-tab.tsx @@ -49,6 +49,7 @@ export function SandboxTab() { org: tenantKey, // @ts-ignore userId is accepted at runtime userId: username ?? '', + // @ts-ignore enable_claw exists on the API but not in the typed request formData: { enable_claw: checked }, }).unwrap(); toast.success(t('toggleSuccess')); diff --git a/e2e/COVERAGE.md b/e2e/COVERAGE.md index 406734a2..53f89e19 100644 --- a/e2e/COVERAGE.md +++ b/e2e/COVERAGE.md @@ -793,6 +793,32 @@ The spec drives the tab through the semantic Tasks helpers from `@iblai/iblai-js --- +## Journey 63: Mentor Evaluation Tab (17 checkpoints) — `journeys/63-mentor-evaluation-tab.spec.ts` + +**Source files:** `components/modals/edit-mentor-modal/tabs/evaluation-tab/index.tsx`, `hooks/use-mentor-segments.ts` + +Wraps the packaged `AgentEvaluationTab` from `@iblai/iblai-js/web-containers/next` with `AgentSettingsProvider`, `getLLMProviderDetails`, and `IblPagination`. All interactions go through the dedicated eval-tab Playwright helpers exported from `@iblai/iblai-js/playwright` (via the `EvaluationTab` page object — mirrors the `VoiceTab`/`TasksTab` thin-delegation pattern from journeys 47/49). Eval datasets ("benchmarks") are tenant-scoped, so the whole file runs serially against one dedicated mentor (created once in `beforeAll`, mirroring journey 14) and one throwaway benchmark created by EVAL-03. Checkpoints that depend on a run reaching `COMPLETED` (EVAL-11/EVAL-12) read the run's live status and skip gracefully instead of assuming either outcome — the real api.iblai.org backend's completion timing is not deterministic, and no test in this journey asserts that a run completes. + +- [x] EVAL-01: Evals tab heading, description, and the info box (`data-testid=evaluation-info-box`) all render when the modal is opened on the Evals tab +- [x] EVAL-02: Benchmark combobox and both toolbar actions (New Evaluation, Manage benchmarks) render; New Evaluation's disabled/enabled state correctly reflects whether a benchmark is auto-selected +- [x] EVAL-03: Admin creates a new benchmark via Manage benchmarks and selects it back on the Evals tab toolbar +- [x] EVAL-04: Admin adds a Q&A pair to a benchmark via Manage benchmarks → View items → Add Q&A (Manual sub-tab) +- [x] EVAL-05: With a fresh (zero-run) benchmark selected, the runs table renders all five canonical headers (Evaluation, Status, Initiated by, Created, Actions) and the benchmark-specific empty state +- [x] EVAL-06: New Evaluation dialog shows the benchmark readback and name input, and Cancel dismisses it without creating a run +- [x] EVAL-07: New Evaluation dialog can be dismissed via Escape without creating a run +- [x] EVAL-08: Admin starts a new evaluation and it appears in the runs table with a valid status badge +- [x] EVAL-09: A run's actions menu reflects its live status — Check status/Delete always enabled; View results/New review/Export CSV enabled once completed, otherwise disabled with the `notReadyHint` tooltip +- [x] EVAL-10: Check status triggers a fresh fetch and surfaces an outcome toast (pending, completed, or failed) +- [x] EVAL-11: Once a run has completed, admin can open View results and Export CSV _(opportunistic — skips gracefully while the run has not completed yet)_ +- [x] EVAL-12: Once a run has completed, the New review (LLM-judge) dialog keeps Evaluate disabled until criteria and score name are filled _(opportunistic — skips gracefully while the run has not completed yet)_ +- [x] EVAL-13: Admin can cancel a delete confirmation (row stays) and then delete a run for real (row is removed) +- [x] EVAL-14: Manage benchmarks lists the created benchmark, and searching for its name filters down to it +- [x] EVAL-15: Evals tab has no axe-core accessibility violations on visible dialogs +- [x] EVAL-16: Admin can leave the Evals tab and return without errors (wrapper re-mounts cleanly) +- [x] EVAL-17: Non-admin does not see the Evals tab in the Edit Mentor modal (ADMIN-only segment in `use-mentor-segments.ts`) + +--- + ## Journey 9b: Voice-to-Text Dictation (1 checkpoint) — `journeys/09b-voice-to-text.spec.ts` **Source files:** `hooks/use-voice-chat.ts`, `hooks/use-timer.tsx`, `components/chat-input-form/voice-chat-button.tsx`, `components/chat-input-form.tsx` diff --git a/e2e/coverage.json b/e2e/coverage.json index 2f57ba93..abccf233 100644 --- a/e2e/coverage.json +++ b/e2e/coverage.json @@ -2618,6 +2618,102 @@ } ] }, + { + "id": "mentor-evaluation-tab", + "name": "Mentor Evaluation Tab", + "spec": "63-mentor-evaluation-tab.spec.ts", + "sourceFiles": [ + "components/modals/edit-mentor-modal/tabs/evaluation-tab/index.tsx", + "hooks/use-mentor-segments.ts" + ], + "checkpoints": [ + { + "id": "eval-01", + "description": "Evals tab heading, description, and the info box (data-testid=evaluation-info-box) all render when the modal is opened on the Evals tab", + "status": "covered" + }, + { + "id": "eval-02", + "description": "Benchmark combobox and both toolbar actions (New Evaluation, Manage benchmarks) render; New Evaluation's disabled/enabled state correctly reflects whether a benchmark is auto-selected", + "status": "covered" + }, + { + "id": "eval-03", + "description": "Admin creates a new benchmark via Manage benchmarks and selects it back on the Evals tab toolbar", + "status": "covered" + }, + { + "id": "eval-04", + "description": "Admin adds a Q&A pair to a benchmark via Manage benchmarks -> View items -> Add Q&A (Manual sub-tab)", + "status": "covered" + }, + { + "id": "eval-05", + "description": "With a fresh (zero-run) benchmark selected, the runs table renders all five canonical headers (Evaluation, Status, Initiated by, Created, Actions) and the benchmark-specific empty state", + "status": "covered" + }, + { + "id": "eval-06", + "description": "New Evaluation dialog shows the benchmark readback and name input, and Cancel dismisses it without creating a run", + "status": "covered" + }, + { + "id": "eval-07", + "description": "New Evaluation dialog can be dismissed via Escape without creating a run", + "status": "covered" + }, + { + "id": "eval-08", + "description": "Admin starts a new evaluation and it appears in the runs table with a valid status badge", + "status": "covered" + }, + { + "id": "eval-09", + "description": "A run's actions menu reflects its live status - Check status/Delete always enabled; View results/New review/Export CSV enabled once completed, otherwise disabled with the notReadyHint tooltip", + "status": "covered" + }, + { + "id": "eval-10", + "description": "Check status triggers a fresh fetch and surfaces an outcome toast (pending, completed, or failed)", + "status": "covered" + }, + { + "id": "eval-11", + "description": "Once a run has completed, admin can open View results and Export CSV (opportunistic checkpoint - skips gracefully while the run has not completed yet)", + "status": "covered" + }, + { + "id": "eval-12", + "description": "Once a run has completed, the New review (LLM-judge) dialog keeps Evaluate disabled until criteria and score name are filled (opportunistic checkpoint - skips gracefully while the run has not completed yet)", + "status": "covered" + }, + { + "id": "eval-13", + "description": "Admin can cancel a delete confirmation (row stays) and then delete a run for real (row is removed)", + "status": "covered" + }, + { + "id": "eval-14", + "description": "Manage benchmarks lists the created benchmark, and searching for its name filters down to it", + "status": "covered" + }, + { + "id": "eval-15", + "description": "Evals tab has no axe-core accessibility violations on visible dialogs", + "status": "covered" + }, + { + "id": "eval-16", + "description": "Admin can navigate away to another tab and back to Evals without errors (wrapper re-mounts cleanly)", + "status": "covered" + }, + { + "id": "eval-17", + "description": "Non-admin does not see the Evals tab in the Edit Mentor modal (ADMIN-only segment in use-mentor-segments.ts)", + "status": "covered" + } + ] + }, { "id": "audit-log", "name": "Audit Log", diff --git a/e2e/journeys/63-mentor-evaluation-tab.spec.ts b/e2e/journeys/63-mentor-evaluation-tab.spec.ts new file mode 100644 index 00000000..9f317eda --- /dev/null +++ b/e2e/journeys/63-mentor-evaluation-tab.spec.ts @@ -0,0 +1,751 @@ +import path from 'path'; +import type { Page } from '@playwright/test'; +import { test, expect } from '../fixtures/mentor-test'; +import { navigateToMentorApp, checkAdminStatus } from '../utils/auth'; +import { parsePlatformUrl } from '../utils/navigation'; +import { waitForPageReady } from '../utils/resilient'; +import { MENTOR_NEXTJS_HOST } from '../fixtures/test-data'; +import { CreateMentorPage } from '../page-objects/create-mentor.page'; +import { EditMentorPage } from '../page-objects/edit-mentor/edit-mentor.page'; +import { EvaluationTab } from '../page-objects/edit-mentor/evaluation.tab'; +import { + closeWithEsc, + expectNoAccessibilityViolationsOnDialogs, + logger, + waitForDialogReady, +} from '@iblai/iblai-js/playwright'; + +/** + * `navigateToMentorApp`, wrapped with a single retry for a Next.js dev-server + * client-side exception ("Application error: a client-side exception has + * occurred...", the framework's default error-boundary fallback) on a cold + * navigation. Observed directly on a live run: unrelated to any Evals-tab + * state (the mentor's own settings are never touched by this journey), and + * consistent with a dev-server HMR/chunk-loading hiccup under sustained + * load rather than an app bug — a plain reload/re-navigate recovers. Scoped + * to this journey rather than the shared `navigateToMentorApp` utility, + * which every other journey also relies on. + */ +async function navigateToMentorAppWithRetry( + page: Page, + url?: string, +): Promise { + try { + await navigateToMentorApp(page, url); + return; + } catch (err) { + let hasErrorBoundary = false; + try { + await page + .getByText(/application error/i) + .waitFor({ state: 'visible', timeout: 2_000 }); + hasErrorBoundary = true; + } catch { + hasErrorBoundary = false; + } + if (!hasErrorBoundary) throw err; + logger.warn( + '[Journey 63] Client-side exception on navigation — retrying once.', + ); + await navigateToMentorApp(page, url); + } +} + +/** + * Journey 63 — Mentor Evaluation (Evals) Tab. + * + * The Evals tab is rendered by the packaged `AgentEvaluationTab` from + * `@iblai/iblai-js/web-containers/next`, wrapped locally with + * `AgentSettingsProvider` + `IblPagination` in + * `components/modals/edit-mentor-modal/tabs/evaluation-tab/index.tsx`. It + * exposes an "Evaluate an agent against a benchmark" flow plus an embedded + * tenant Benchmarks manager, and is gated ADMIN-only in + * `hooks/use-mentor-segments.ts`. + * + * All interactions go through the `EvaluationTab` page object, which + * thin-delegates to the dedicated eval-tab helpers exported from + * `@iblai/iblai-js/playwright` (mirrors the `VoiceTab` / `TasksTab` pattern + * — see journeys 47 / 49). + * + * ── Shared state strategy ────────────────────────────────────────────── + * Eval datasets ("benchmarks") are TENANT-scoped — shared across every + * mentor and worker — so mutating one from a per-test throwaway mentor + * would still race against any other worker touching the same tenant. + * `test.describe.configure({ mode: 'serial' })` pins this whole file to a + * single worker (mirrors journey 44's file-level serial isolation), and a + * SINGLE dedicated mentor is created once in `beforeAll` (mirrors journey + * 14's manual-context setup/teardown) rather than per test: + * - it keeps the suite from minting a mentor per checkpoint, and + * - it lets an evaluation run started early in the file (EVAL-08) pick up + * more wall-clock time before the later, completion-gated checkpoints + * (EVAL-11/EVAL-12) read its live status — those checkpoints branch on + * whatever status is actually showing rather than assuming either + * outcome, since the real api.iblai.org backend's completion timing is + * not deterministic (a run may stay PENDING for a long time). No test + * in this file asserts that a run completes. + * - the same wall-clock time also matters for EVAL-13's delete-confirm + * step: a run deleted only moments after it was started can 404 + * against the delete endpoint even though it's already visible in the + * runs table (observed directly — the backend hadn't finished + * materializing it into the store that endpoint reads from), so + * EVAL-13 reuses the primary run rather than starting its own. + * A single throwaway benchmark, created in EVAL-03 with a + * `e2e-eval-benchmark-` name, is reused by every later + * checkpoint. Benchmarks have no delete affordance anywhere in the UI (see + * `packages/web-containers/.../profile/benchmarks/index.tsx` — only a + * "View items" action is exposed), so cleanup for it is inherently + * best-effort/non-existent; only the evaluation runs and the dedicated + * mentor are actually deleted in `afterAll`. + */ + +test.describe.configure({ mode: 'serial' }); + +test.describe('Journey 63: Mentor Evaluation Tab', () => { + let mentorId = ''; + let platformKey = ''; + let mentorUrl = ''; + let benchmarkReady = false; + let primaryRunName = ''; + let primaryRunCompleted = false; + + const BENCHMARK_NAME = EvaluationTab.uniqueName('e2e-eval-benchmark'); + const QA_QUESTION = EvaluationTab.uniqueName('E2E eval question'); + + test.beforeAll(async ({ browser }, testInfo) => { + // Mentor creation waits out its own category-list load (with a + // close/reopen retry baked into CreateMentorPage under contention) on + // top of the initial platform navigation; give it the same generous + // margin as afterAll rather than the default test timeout. + testInfo.setTimeout(180_000); + + const browserKey = testInfo.project.name + .replace('mentor-desktop-', '') + .toLowerCase(); + const authFile = path.join( + __dirname, + `../../playwright/.auth/user-${browserKey}.json`, + ); + const setupContext = await browser.newContext({ storageState: authFile }); + const setupPage = await setupContext.newPage(); + + try { + await navigateToMentorAppWithRetry(setupPage); + const isAdmin = await checkAdminStatus(setupPage); + if (!isAdmin) { + logger.warn( + '[Journey 63] Setup account is not admin; every Evals test will self-skip.', + ); + return; + } + + const createMentorPage = new CreateMentorPage(setupPage); + await createMentorPage.openAndCreate( + EvaluationTab.uniqueName('E2E Eval Mentor'), + ); + + const parsed = parsePlatformUrl(setupPage.url()); + platformKey = parsed.platformKey; + mentorId = parsed.mentorId; + mentorUrl = `${MENTOR_NEXTJS_HOST}/platform/${platformKey}/${mentorId}`; + logger.info(`[Journey 63] Created dedicated eval mentor at ${mentorUrl}`); + } finally { + await setupPage.close(); + await setupContext.close(); + } + }); + + test.afterAll(async ({ browser }, testInfo) => { + if (!mentorUrl) return; + // Deleting the mentor is a two-modal-open round trip (Settings) against + // a real backend on top of whatever the last test in the file already + // took; give it materially more room than the default test timeout so + // a slow-but-successful cleanup never gets reported as a hook timeout. + testInfo.setTimeout(180_000); + + const browserKey = testInfo.project.name + .replace('mentor-desktop-', '') + .toLowerCase(); + const authFile = path.join( + __dirname, + `../../playwright/.auth/user-${browserKey}.json`, + ); + const cleanupContext = await browser.newContext({ storageState: authFile }); + const cleanupPage = await cleanupContext.newPage(); + + try { + await navigateToMentorAppWithRetry(cleanupPage, mentorUrl); + const editMentorPage = new EditMentorPage(cleanupPage); + + // Deleting the mentor is the real cleanup goal — once it's gone, an + // undeleted run under it becomes invisible to everyone (the runs + // table is scoped by mentor_unique_id) and is effectively harmless. + // EVAL-13 already exercises + asserts the primary run's own delete + // flow when it runs, so this hook doesn't repeat that as a second, + // unasserted round trip — it would roughly double this hook's + // duration for no coverage benefit. The benchmark itself is left + // behind on purpose — there is no delete affordance for it anywhere + // in the UI (see class comment above). + await editMentorPage.open('Settings'); + await waitForPageReady(cleanupPage); + await editMentorPage.settings.deleteMentor(); + logger.info(`[Journey 63] Deleted dedicated eval mentor ${mentorId}`); + } catch (err) { + logger.warn( + `[Journey 63] Failed to clean up eval mentor ${mentorId}: ${err}`, + ); + } finally { + await cleanupPage.close(); + await cleanupContext.close(); + } + }); + + test.beforeEach(async ({ page, editMentorPage }) => { + test.skip(!mentorUrl, 'Dedicated eval mentor was not created in beforeAll'); + + await navigateToMentorAppWithRetry(page, mentorUrl); + const isAdmin = await checkAdminStatus(page); + if (!isAdmin) { + test.skip(true, 'Evals tab is admin-only'); + return; + } + + await editMentorPage.open('Evals'); + await waitForPageReady(page); + await waitForDialogReady(page, editMentorPage.dialog); + await expect( + editMentorPage.evaluation.manageBenchmarksButton(), + ).toBeVisible({ + timeout: 15_000, + }); + + // Once EVAL-03 has created the shared throwaway benchmark, every later + // checkpoint should be looking at it rather than whatever the toolbar + // auto-selects (index.tsx auto-selects `benchmarks[0]`, which is not + // necessarily ours on a tenant that already has other benchmarks). + if (benchmarkReady) { + await editMentorPage.evaluation.selectBenchmark(BENCHMARK_NAME); + } + }); + + // EVAL-01: Tab heading, description, and the new info box all render. + test('admin opens the Evals tab and sees the heading, description, and info box', async ({ + editMentorPage, + }) => { + const { dialog } = editMentorPage; + + const evalTab = dialog.getByRole('tab', { + name: EvaluationTab.LABELS.tabName, + exact: true, + }); + await expect(evalTab).toHaveAttribute('data-state', 'active', { + timeout: 10_000, + }); + + await expect( + dialog.getByRole('heading', { + name: EvaluationTab.LABELS.header.title, + exact: true, + }), + ).toBeVisible({ timeout: 10_000 }); + await expect( + dialog.getByText(EvaluationTab.LABELS.header.description), + ).toBeVisible({ timeout: 5_000 }); + + const infoBox = dialog.getByTestId('evaluation-info-box'); + await expect(infoBox).toBeVisible({ timeout: 5_000 }); + expect((await infoBox.innerText()).trim().length).toBeGreaterThan(0); + + await editMentorPage.close(); + }); + + // EVAL-02: Toolbar (benchmark combobox + New Evaluation + Manage + // benchmarks) renders, and the New Evaluation CTA's disabled/enabled + // state correctly reflects whether a benchmark could be auto-selected. + test('admin sees the benchmark combobox and both toolbar actions, and New Evaluation reflects whether a benchmark is selected', async ({ + editMentorPage, + }) => { + const { evaluation } = editMentorPage; + + await expect(evaluation.benchmarkCombobox()).toBeVisible({ + timeout: 10_000, + }); + await expect(evaluation.newEvaluationButton()).toBeVisible({ + timeout: 5_000, + }); + await expect(evaluation.manageBenchmarksButton()).toBeVisible({ + timeout: 5_000, + }); + + // index.tsx auto-selects the first fetched benchmark the instant the + // tenant has one, so "disabled" only happens on a genuinely empty + // tenant. Read which branch this real tenant is actually in instead of + // assuming either. + let hasBenchmarks = true; + try { + await expect(evaluation.benchmarkCombobox()).toBeEnabled({ + timeout: 10_000, + }); + } catch { + hasBenchmarks = false; + } + + if (hasBenchmarks) { + await expect(evaluation.newEvaluationButton()).toBeEnabled({ + timeout: 10_000, + }); + } else { + await expect(evaluation.newEvaluationButton()).toBeDisabled({ + timeout: 5_000, + }); + await evaluation.expectNoBenchmarksNotice(); + } + + await editMentorPage.close(); + }); + + // EVAL-03: Admin creates the shared throwaway benchmark via "Manage + // benchmarks" and selects it back on the Evals tab toolbar. + test('admin creates a new benchmark via Manage benchmarks and selects it on the Evals tab', async ({ + editMentorPage, + }) => { + const { evaluation } = editMentorPage; + + await evaluation.openManageBenchmarksDialog(); + await evaluation.createBenchmark({ + name: BENCHMARK_NAME, + description: 'Throwaway benchmark created by e2e journey 63.', + }); + await evaluation.closeManageBenchmarksDialog(); + + await evaluation.selectBenchmark(BENCHMARK_NAME); + benchmarkReady = true; + + await editMentorPage.close(); + }); + + // EVAL-04: Admin adds a Q&A pair to the benchmark via Manage benchmarks → + // View items → Add Q&A (Manual sub-tab), and it's listed afterward. + test('admin adds a Q&A pair to the benchmark via Manage benchmarks', async ({ + editMentorPage, + }) => { + test.skip(!benchmarkReady, 'Benchmark was not created in EVAL-03'); + const { evaluation } = editMentorPage; + + await evaluation.openManageBenchmarksDialog(); + await evaluation.searchBenchmarks(BENCHMARK_NAME); + await evaluation.expectBenchmarkListed(BENCHMARK_NAME); + + await evaluation.openBenchmarkItems(BENCHMARK_NAME); + await evaluation.openAddItemsDialog(); + await evaluation.addQaPairsManually([ + { question: QA_QUESTION, answer: 'E2E expected answer.' }, + ]); + await evaluation.expectQaItemListed(QA_QUESTION); + + await evaluation.closeBenchmarkItemsDialog(); + await evaluation.closeManageBenchmarksDialog(); + + await editMentorPage.close(); + }); + + // EVAL-05: With the fresh (zero-run) benchmark selected, the runs table + // renders its five canonical headers and the benchmark-specific empty + // state (`{dataset}` substituted). + test('with the fresh benchmark selected, the runs table shows its headers and the benchmark-specific empty state', async ({ + editMentorPage, + }) => { + test.skip(!benchmarkReady, 'Benchmark was not created in EVAL-03'); + const { evaluation } = editMentorPage; + + for (const header of EvaluationTab.TABLE_HEADERS) { + await expect(evaluation.tableHeader(header)).toBeVisible({ + timeout: 15_000, + }); + } + await evaluation.expectRunsTableEmpty(BENCHMARK_NAME); + + await editMentorPage.close(); + }); + + // EVAL-06: New Evaluation dialog shows the benchmark readback + name + // input + Start button, and Cancel dismisses it without creating a run. + test('the New Evaluation dialog shows the benchmark readback and name input, and Cancel dismisses it', async ({ + editMentorPage, + }) => { + test.skip(!benchmarkReady, 'Benchmark was not created in EVAL-03'); + const { evaluation } = editMentorPage; + + const dialog = await evaluation.openStartEvaluationDialog(); + await expect(dialog.getByText(BENCHMARK_NAME, { exact: true })).toBeVisible( + { timeout: 5_000 }, + ); + await expect( + dialog.getByLabel(EvaluationTab.LABELS.startDialog.experimentNameLabel), + ).toBeVisible({ timeout: 5_000 }); + await expect( + dialog.getByRole('button', { + name: EvaluationTab.LABELS.startDialog.start, + }), + ).toBeVisible(); + + await evaluation.cancelStartEvaluation(); + + await editMentorPage.close(); + }); + + // EVAL-07: New Evaluation dialog can be dismissed via Escape (modal stack + // hygiene) without creating a run. + test('the New Evaluation dialog can be dismissed via Escape without creating a run', async ({ + page, + editMentorPage, + }) => { + test.skip(!benchmarkReady, 'Benchmark was not created in EVAL-03'); + const { evaluation } = editMentorPage; + + const dialog = await evaluation.openStartEvaluationDialog(); + await closeWithEsc(page); + await expect(dialog).toBeHidden({ timeout: 5_000 }); + + await editMentorPage.close(); + }); + + // EVAL-08: Admin starts a real evaluation; the dialog closes and the run + // appears in the table with a valid status badge. This is the "primary" + // run later checkpoints opportunistically re-check the status of. + test('admin starts a new evaluation and it appears in the table with a valid status', async ({ + editMentorPage, + }) => { + test.skip(!benchmarkReady, 'Benchmark was not created in EVAL-03'); + const { evaluation } = editMentorPage; + + primaryRunName = EvaluationTab.uniqueName('e2e-eval-run'); + await evaluation.startEvaluation({ name: primaryRunName }); + await evaluation.expectRunInTable(primaryRunName); + + const status = await evaluation.readRunStatus(primaryRunName); + expect(['COMPLETED', 'FAILED', 'PENDING', 'IN_PROGRESS']).toContain(status); + + await editMentorPage.close(); + }); + + // EVAL-09: The primary run's row-actions menu reflects its live status — + // Check status / Delete are always available; View results / New review / + // Export CSV are enabled once completed and disabled (with the + // `notReadyHint` tooltip) otherwise. Branches on the real status instead + // of assuming the run is still pending, since completion timing against + // the real backend is not deterministic. + test("the primary run's actions menu reflects its live status", async ({ + editMentorPage, + }) => { + test.skip(!primaryRunName, 'Primary run was not started in EVAL-08'); + const { evaluation } = editMentorPage; + + const status = await evaluation.readRunStatus(primaryRunName); + primaryRunCompleted = status === 'COMPLETED'; + + const menu = await evaluation.openRunActionsMenu(primaryRunName); + const viewResults = menu.getByRole('menuitem', { + name: EvaluationTab.LABELS.actions.viewResults, + }); + const newReview = menu.getByRole('menuitem', { + name: EvaluationTab.LABELS.actions.evaluate, + }); + const exportCsv = menu.getByRole('menuitem', { + name: EvaluationTab.LABELS.actions.exportCsv, + }); + const checkStatus = menu.getByRole('menuitem', { + name: EvaluationTab.LABELS.actions.checkStatus, + }); + const del = menu.getByRole('menuitem', { + name: EvaluationTab.LABELS.actions.delete, + }); + + // Always available regardless of run status. + await expect(checkStatus).toBeEnabled({ timeout: 5_000 }); + await expect(del).toBeEnabled({ timeout: 5_000 }); + + if (primaryRunCompleted) { + await expect(viewResults).toBeEnabled({ timeout: 5_000 }); + await expect(newReview).toBeEnabled({ timeout: 5_000 }); + await expect(exportCsv).toBeEnabled({ timeout: 5_000 }); + } else { + await expect(viewResults).toBeDisabled({ timeout: 5_000 }); + await expect(viewResults).toHaveAttribute( + 'title', + EvaluationTab.NOT_READY_HINT, + ); + await expect(newReview).toBeDisabled({ timeout: 5_000 }); + await expect(newReview).toHaveAttribute( + 'title', + EvaluationTab.NOT_READY_HINT, + ); + await expect(exportCsv).toBeDisabled({ timeout: 5_000 }); + await expect(exportCsv).toHaveAttribute( + 'title', + EvaluationTab.NOT_READY_HINT, + ); + } + + await editMentorPage.page.keyboard.press('Escape'); + await expect(menu).toBeHidden({ timeout: 5_000 }); + await editMentorPage.close(); + }); + + // EVAL-10: "Check status" triggers a fresh fetch and surfaces one of the + // three outcome toasts. + test('admin checks the primary run status and sees an outcome toast', async ({ + editMentorPage, + }) => { + test.skip(!primaryRunName, 'Primary run was not started in EVAL-08'); + const { evaluation } = editMentorPage; + + const outcome = await evaluation.checkRunStatus(primaryRunName); + expect(['pending', 'completed', 'failed']).toContain(outcome); + if (outcome === 'completed') primaryRunCompleted = true; + + await editMentorPage.close(); + }); + + // EVAL-11 (opportunistic): once the primary run has completed, admin can + // open View results and Export CSV. Re-reads the live status first since + // it may have completed since EVAL-09/10 last checked; skips gracefully + // (never fails) while the run is still not ready. + test('admin can view results and export CSV once the primary run has completed', async ({ + editMentorPage, + }) => { + test.skip(!primaryRunName, 'Primary run was not started in EVAL-08'); + const { evaluation } = editMentorPage; + + if (!primaryRunCompleted) { + primaryRunCompleted = + (await evaluation.readRunStatus(primaryRunName)) === 'COMPLETED'; + } + test.skip( + !primaryRunCompleted, + 'Primary run has not completed yet — View results / Export CSV are only reachable once it has (real backend, non-deterministic timing).', + ); + + const detail = await evaluation.openRunResults(primaryRunName); + await expect( + detail.getByRole('button', { + name: EvaluationTab.LABELS.detailDialog.refreshAriaLabel, + exact: true, + }), + ).toBeVisible({ timeout: 10_000 }); + + // Close via the dialog's own visible "Close" (X) button rather than + // `evaluation.closeEvaluationDetailDialog()` (Escape-based): this + // dialog auto-refreshes (polls the run for new traces) while open, and + // a single Escape keypress can race a poll-driven re-render and get + // dropped, leaving the dialog stuck open past the 10s wait. A direct + // click re-resolves the current DOM node instead of relying on a + // point-in-time key event, so it isn't subject to that race. + await detail.getByRole('button', { name: 'Close', exact: true }).click(); + await expect(detail).toBeHidden({ timeout: 10_000 }); + + const filename = await evaluation.exportRunCsv(primaryRunName); + expect(filename).toContain(primaryRunName); + expect(filename.endsWith('.csv')).toBe(true); + + await editMentorPage.close(); + }); + + // EVAL-12 (opportunistic): once the primary run has completed, the New + // review (LLM-judge) dialog keeps Evaluate disabled until criteria + + // score name + an LLM are all provided. Stops short of actually picking + // an LLM/submitting — that would spend a real LLM call against the real + // backend for a check that's already fully covered by the disabled-state + // assertions below, and model names are prefixes of one another (e.g. + // "gpt-4o" / "gpt-4o-mini"), which is a real substring-collision risk for + // any hardcoded model name. + test('the New review dialog keeps Evaluate disabled until criteria and score name are filled, once the primary run has completed', async ({ + editMentorPage, + }) => { + test.skip(!primaryRunName, 'Primary run was not started in EVAL-08'); + const { evaluation } = editMentorPage; + + if (!primaryRunCompleted) { + primaryRunCompleted = + (await evaluation.readRunStatus(primaryRunName)) === 'COMPLETED'; + } + test.skip( + !primaryRunCompleted, + 'New review is only reachable once the primary run has completed (real backend, non-deterministic timing).', + ); + + const judgeDialog = await evaluation.openNewReviewForRun(primaryRunName); + const submit = judgeDialog.getByRole('button', { + name: EvaluationTab.LABELS.judgeDialog.submit, + exact: true, + }); + await expect(submit).toBeDisabled({ timeout: 5_000 }); + + await judgeDialog + .getByLabel(EvaluationTab.LABELS.judgeDialog.criteriaLabel) + .fill('Score the response for clarity and accuracy.'); + await expect(submit).toBeDisabled({ timeout: 5_000 }); + + await judgeDialog + .getByLabel(EvaluationTab.LABELS.judgeDialog.scoreNameLabel) + .fill('e2e-quality'); + // LLM still unselected — submit must stay disabled even with both text + // fields filled in. + await expect(submit).toBeDisabled({ timeout: 5_000 }); + + await evaluation.cancelLlmJudge(); + await editMentorPage.close(); + }); + + // EVAL-13: Cancelling a delete confirmation leaves the row in place; + // confirming for real removes it. Reuses the primary run (several minutes + // old by this point in the file) rather than starting a fresh one — a + // run created only moments earlier can 404 on the delete endpoint even + // though it's already visible in the table (observed directly: the + // backend hadn't finished materializing a <1-minute-old run into the + // store the delete endpoint reads from, so every delete attempt against + // it failed with a 404 and the confirm dialog never closed). The primary + // run has had several other checkpoints' worth of wall-clock time to + // settle, which this same delete flow already deletes reliably in + // `afterAll` when EVAL-13 doesn't run. + test('admin can cancel a delete confirmation and then delete a run for real', async ({ + editMentorPage, + }) => { + test.skip(!primaryRunName, 'Primary run was not started in EVAL-08'); + const { evaluation } = editMentorPage; + + await evaluation.cancelDeleteEvaluation(primaryRunName); + await evaluation.expectRunInTable(primaryRunName); + + await evaluation.deleteEvaluation(primaryRunName); + await evaluation.expectRunNotInTable(primaryRunName); + + await editMentorPage.close(); + }); + + // EVAL-14: Manage benchmarks lists the created benchmark, and searching + // for its name filters down to it. + test('Manage benchmarks lists the created benchmark and search filters to it', async ({ + editMentorPage, + }) => { + test.skip(!benchmarkReady, 'Benchmark was not created in EVAL-03'); + const { evaluation } = editMentorPage; + + await evaluation.openManageBenchmarksDialog(); + await evaluation.expectBenchmarkListed(BENCHMARK_NAME); + + await evaluation.searchBenchmarks(BENCHMARK_NAME); + await evaluation.expectBenchmarkListed(BENCHMARK_NAME); + + await evaluation.closeManageBenchmarksDialog(); + await editMentorPage.close(); + }); + + // EVAL-15: No axe-core accessibility violations on visible dialogs. + // + // Excludes the benchmark combobox trigger: it's a pre-existing upstream + // defect in `@iblai/iblai-js`'s `BenchmarkCombobox` (a plain `