From 29c845844bf360847a6cd52269c76ffb83c04534 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Wed, 7 Oct 2026 15:16:44 -0400 Subject: [PATCH 1/3] Fix V2 Control Center Dashboard, Users and Groups 500s; remove Data health Cosmos DB rejected the queries behind these sections with HTTP 400. The azure-cosmos Python SDK cannot run cross-partition GROUP BY or COUNT over a DISTINCT subquery, and the Users sort used a two-property ORDER BY with no composite index on user_settings. - Dashboard: count SELECT DISTINCT VALUE results, derive uploads by subtracting VALUE COUNTs, tally status values, and aggregate two streamed projections for insights. The login heatmap now totals each weekday and hour. - Users: order one property at a time in recorded and missing populations; the export runs its first query before it starts streaming. - Groups: the inventory aggregates streamed projections. - Add a Cosmos query guard, route-level tests and an AST scan of every V2 Control Center query. - Remove the V2 Data health section and its now-unused migrate APIs. - Version 0.261.290; feature doc, guide, fix doc and release notes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/config.py | 2 +- .../single_app/functions_activity_logging.py | 31 - .../functions_control_center_groups.py | 59 +- .../route_backend_control_center.py | 958 ++++++------------ .../v2_ui/src/pages/ControlCenterPage.tsx | 139 +-- .../explanation/features/V2_CONTROL_CENTER.md | 26 +- .../ACTIVITY_LOG_MIGRATION_PROMPT_FIX.md | 4 + ...L_CENTER_COSMOS_QUERY_COMPATIBILITY_FIX.md | 70 ++ docs/explanation/release_notes.md | 18 + docs/guides/v2-control-center.md | 18 +- .../test_support/cosmos_query_guard.py | 89 ++ ...ntrol_center_cosmos_query_compatibility.py | 170 ++++ .../test_v2_control_center_dashboard.py | 376 ++++++- .../test_v2_control_center_foundation.py | 70 +- .../test_v2_control_center_groups.py | 76 +- .../test_v2_control_center_users.py | 198 +++- .../test_v2_control_center_data_health.py | 183 ---- ...t_v2_control_center_data_health_removed.py | 192 ++++ 18 files changed, 1508 insertions(+), 1171 deletions(-) create mode 100644 docs/explanation/fixes/V2_CONTROL_CENTER_COSMOS_QUERY_COMPATIBILITY_FIX.md create mode 100644 functional_tests/test_support/cosmos_query_guard.py create mode 100644 functional_tests/test_v2_control_center_cosmos_query_compatibility.py delete mode 100644 ui_tests/test_v2_control_center_data_health.py create mode 100644 ui_tests/test_v2_control_center_data_health_removed.py diff --git a/application/single_app/config.py b/application/single_app/config.py index efe40558c..638e73afe 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -101,7 +101,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.289" +VERSION = "0.261.290" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_activity_logging.py b/application/single_app/functions_activity_logging.py index 636b4baf6..61866bf4a 100644 --- a/application/single_app/functions_activity_logging.py +++ b/application/single_app/functions_activity_logging.py @@ -40,37 +40,6 @@ def _create_activity_record(record, idempotency_key=None): return record -def has_activity_log_for_resource(user_id, activity_type, resource_id, workspace_type=None): - """Check for an existing creation record within its user partition.""" - if not user_id or not resource_id: - return False - - if activity_type == 'conversation_creation': - resource_path = 'c.conversation.conversation_id' - elif activity_type == 'document_creation': - resource_path = 'c.document.document_id' - else: - raise ValueError("Unsupported activity type for resource lookup.") - - query = ( - "SELECT TOP 1 VALUE c.id FROM c " - f"WHERE c.activity_type = @activity_type AND {resource_path} = @resource_id" - ) - parameters = [ - {'name': '@activity_type', 'value': activity_type}, - {'name': '@resource_id', 'value': resource_id}, - ] - if workspace_type: - query += " AND c.workspace_type = @workspace_type" - parameters.append({'name': '@workspace_type', 'value': workspace_type}) - matches = cosmos_activity_logs_container.query_items( - query=query, - parameters=parameters, - partition_key=user_id, - ) - return next(iter(matches), None) is not None - - def coerce_activity_log_user_id(user_id: Any) -> str: """Extract a stable string user id from a scalar or session-style identity payload.""" if user_id is None: diff --git a/application/single_app/functions_control_center_groups.py b/application/single_app/functions_control_center_groups.py index 93c941ff7..460df6e81 100644 --- a/application/single_app/functions_control_center_groups.py +++ b/application/single_app/functions_control_center_groups.py @@ -2,6 +2,7 @@ """Server-side group inventory and validated selection for Control Center.""" import re +from collections import Counter, defaultdict from datetime import datetime, timezone @@ -118,30 +119,29 @@ def group_row(group, documents, tokens, last_activity): def load_group_inventory(groups_container, documents_container, activity_container): - """Four batched queries, independent of row count. Fail rather than invent totals.""" + """Four batched queries, independent of row count. Fail rather than invent totals. + + The Python Cosmos SDK cannot run cross-partition GROUP BY, so the document, token and + activity queries stream narrow projections and the totals are aggregated here. + """ groups = list(groups_container.query_items( query=("SELECT c.id, c.name, c.description, c.owner, c.users, c.admins, " "c.documentManagers, c.status, c.createdDate, c.metrics FROM c"), enable_cross_partition_query=True, )) - documents = { - row["group_id"]: int(row.get("total") or 0) - for row in documents_container.query_items( - query=("SELECT c.group_id, COUNT(1) AS total FROM c " - "WHERE c.type = 'document_metadata' AND IS_DEFINED(c.group_id) GROUP BY c.group_id"), - enable_cross_partition_query=True, - ) - } - tokens = { - row["group_id"]: int(row.get("total") or 0) - for row in activity_container.query_items( - query=("SELECT c.workspace_context.group_id AS group_id, SUM(c.usage.total_tokens) AS total " - "FROM c WHERE c.activity_type = 'token_usage' " - "AND IS_DEFINED(c.workspace_context.group_id) AND IS_NUMBER(c.usage.total_tokens) " - "GROUP BY c.workspace_context.group_id"), - enable_cross_partition_query=True, - ) - } + documents = Counter(documents_container.query_items( + query=("SELECT VALUE c.group_id FROM c " + "WHERE c.type = 'document_metadata' AND IS_STRING(c.group_id)"), + enable_cross_partition_query=True, + )) + tokens = defaultdict(int) + for row in activity_container.query_items( + query=("SELECT c.workspace_context.group_id AS group_id, c.usage.total_tokens AS tokens " + "FROM c WHERE c.activity_type = 'token_usage' " + "AND IS_STRING(c.workspace_context.group_id) AND IS_NUMBER(c.usage.total_tokens)"), + enable_cross_partition_query=True, + ): + tokens[row["group_id"]] += row["tokens"] # The writers use three group locations. One coalesced expression avoids duplicate # records and includes admin CSV and approval events that use top-level group_id. group_expression = ( @@ -149,18 +149,19 @@ def load_group_inventory(groups_container, documents_container, activity_contain "IIF(IS_STRING(c.group.group_id) AND c.group.group_id != '', " "c.group.group_id, c.workspace_context.group_id))" ) - activity_times = { - row["group_id"]: row.get("last_activity") - for row in activity_container.query_items( - query=(f"SELECT {group_expression} AS group_id, MAX(c.timestamp) AS last_activity FROM c " - f"WHERE IS_STRING({group_expression}) AND {group_expression} != '' " - f"GROUP BY {group_expression}"), - enable_cross_partition_query=True, - ) - } + activity_times = {} + for row in activity_container.query_items( + query=(f"SELECT {group_expression} AS group_id, c.timestamp FROM c " + f"WHERE IS_STRING({group_expression}) AND {group_expression} != '' " + "AND IS_STRING(c.timestamp)"), + enable_cross_partition_query=True, + ): + group_id, timestamp = row["group_id"], row["timestamp"] + if timestamp > activity_times.get(group_id, ""): + activity_times[group_id] = timestamp calculated_at = datetime.now(timezone.utc).isoformat() return { - "rows": [group_row(group, documents.get(group["id"], 0), tokens.get(group["id"], 0), + "rows": [group_row(group, documents.get(group["id"], 0), int(tokens.get(group["id"], 0)), activity_times.get(group["id"])) for group in groups], "calculated_at": calculated_at, } diff --git a/application/single_app/route_backend_control_center.py b/application/single_app/route_backend_control_center.py index 0be7d9461..c0bc7194c 100644 --- a/application/single_app/route_backend_control_center.py +++ b/application/single_app/route_backend_control_center.py @@ -5,6 +5,7 @@ import math import re import time +from collections import Counter, defaultdict from io import StringIO from flask import Response, make_response, stream_with_context @@ -85,6 +86,12 @@ CONTROL_CENTER_MANAGEMENT_MAX_PER_PAGE = 250 CONTROL_CENTER_DASHBOARD_CACHE_TTL_SECONDS = 90 CONTROL_CENTER_DASHBOARD_CACHE_MAX_ENTRIES = 128 +CONTROL_CENTER_DASHBOARD_RANKING_LIMIT = 10 +CONTROL_CENTER_DASHBOARD_RANKED_FIELDS = ( + ("users", "user_id"), + ("groups", "group_id"), + ("public_workspaces", "public_workspace_id"), +) DASHBOARD_INVALID_RANGE_ERROR = ( "Invalid dashboard date range. Use 7, 30, or 90 days or a valid custom range of up to 366 days." ) @@ -279,23 +286,26 @@ def _dashboard_query_count(container, query, parameters=None): def _dashboard_count_active_users(start_date, end_date): - """Count distinct users with recorded login activity in a UTC interval.""" - return _dashboard_query_count( - cosmos_activity_logs_container, - """ - SELECT VALUE COUNT(1) FROM ( - SELECT DISTINCT c.user_id FROM c - WHERE c.activity_type = 'user_login' - AND IS_DEFINED(c.user_id) - AND c.timestamp >= @start_date - AND c.timestamp <= @end_date - ) + """Count distinct users with recorded login activity in a UTC interval. + + The Python Cosmos SDK cannot run COUNT over a DISTINCT subquery across partitions, + so the distinct user IDs are read with SELECT DISTINCT VALUE and counted here. + """ + user_ids = cosmos_activity_logs_container.query_items( + query=""" + SELECT DISTINCT VALUE c.user_id FROM c + WHERE c.activity_type = 'user_login' + AND IS_DEFINED(c.user_id) + AND c.timestamp >= @start_date + AND c.timestamp <= @end_date """, - [ + parameters=[ {"name": "@start_date", "value": start_date.isoformat()}, {"name": "@end_date", "value": end_date.isoformat()}, ], + enable_cross_partition_query=True, ) + return len(set(user_ids)) def _dashboard_activity_count(activity_type, start_date, end_date): @@ -319,30 +329,49 @@ def _dashboard_activity_count(activity_type, start_date, end_date): def _dashboard_document_upload_counts(start_date, end_date): - """Aggregate document-creation activity by the recorded workspace type.""" - rows = list(cosmos_activity_logs_container.query_items( - query=""" - SELECT c.workspace_type AS workspace_type, COUNT(1) AS count FROM c - WHERE c.activity_type = 'document_creation' - AND ( - (c.timestamp >= @start_date AND c.timestamp <= @end_date) - OR (c.created_at >= @start_date AND c.created_at <= @end_date) - ) - GROUP BY c.workspace_type - """, - parameters=[ - {"name": "@start_date", "value": start_date.isoformat()}, - {"name": "@end_date", "value": end_date.isoformat()}, - ], - enable_cross_partition_query=True, - )) - counts = {"personal": 0, "group": 0, "public": 0} - for row in rows: - workspace_type = row.get("workspace_type") or "personal" - if workspace_type not in counts: - workspace_type = "personal" - counts[workspace_type] += int(row.get("count") or 0) - return counts + """Count document-creation activity by the recorded workspace type. + + Cross-partition GROUP BY is unavailable to the Python Cosmos SDK, so group and + public uploads are counted directly. Every other recorded or missing type is + personal, so personal is the remainder of the period total. + """ + window = """ + c.activity_type = 'document_creation' + AND ( + (c.timestamp >= @start_date AND c.timestamp <= @end_date) + OR (c.created_at >= @start_date AND c.created_at <= @end_date) + ) + """ + parameters = [ + {"name": "@start_date", "value": start_date.isoformat()}, + {"name": "@end_date", "value": end_date.isoformat()}, + ] + total = _dashboard_query_count( + cosmos_activity_logs_container, + f"SELECT VALUE COUNT(1) FROM c WHERE {window}", + parameters, + ) + counts = { + workspace_type: _dashboard_query_count( + cosmos_activity_logs_container, + f"SELECT VALUE COUNT(1) FROM c WHERE {window} AND c.workspace_type = @workspace_type", + parameters + [{"name": "@workspace_type", "value": workspace_type}], + ) + for workspace_type in ("group", "public") + } + return {"personal": max(total - counts["group"] - counts["public"], 0), **counts} + + +def _dashboard_status_rows(container): + """Tally stored status values for _dashboard_status_counts without a GROUP BY query.""" + statuses = Counter( + status if status is None or isinstance(status, str) else str(status) + for status in container.query_items( + query="SELECT VALUE c.status FROM c WHERE IS_DEFINED(c.status)", + enable_cross_partition_query=True, + ) + ) + return [{"status": status, "count": count} for status, count in statuses.items()] def _dashboard_token_total(start_date, end_date, token_filters): @@ -392,6 +421,115 @@ def _dashboard_document_failures(start_date, end_date): ) +def _dashboard_token_value(value): + """Return the numeric token count Cosmos SUM would add; other values count as zero.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return 0 + return value + + +def _dashboard_ranked(totals, value_key): + """Return the largest totals, with ties ordered by ID so the ranking is stable.""" + ranked = sorted(totals.items(), key=lambda item: (-item[1], item[0])) + return [ + {"id": entity_id, value_key: int(total)} + for entity_id, total in ranked[:CONTROL_CENTER_DASHBOARD_RANKING_LIMIT] + ] + + +def _dashboard_token_insights(start_date, end_date, token_filters): + """Total filtered token usage by day and model and rank the largest consumers. + + The Python Cosmos SDK cannot run cross-partition GROUP BY, so one narrow projection + of the filtered token records is streamed and totalled here. + """ + where_clause, parameters = build_token_usage_query_context( + start_date, + end_date, + token_filters=token_filters, + ) + by_model = defaultdict(int) + consumers = {key: defaultdict(int) for key, _ in CONTROL_CENTER_DASHBOARD_RANKED_FIELDS} + rows = cosmos_activity_logs_container.query_items( + query=f""" + SELECT c.timestamp, + c.usage.model AS model, + c.usage.total_tokens AS tokens, + c.user_id AS user_id, + c.workspace_context.group_id AS group_id, + c.workspace_context.public_workspace_id AS public_workspace_id + FROM c + WHERE {where_clause} + """, + parameters=parameters, + enable_cross_partition_query=True, + ) + for row in rows: + tokens = _dashboard_token_value(row.get("tokens")) + timestamp = row.get("timestamp") + if isinstance(timestamp, str) and "model" in row: + by_model[(timestamp[:10], str(row.get("model") or "Unknown model"))] += tokens + for key, field in CONTROL_CENTER_DASHBOARD_RANKED_FIELDS: + entity_id = row.get(field) + if isinstance(entity_id, str) and entity_id: + consumers[key][entity_id] += tokens + token_usage_by_model = [ + {"date": date, "model": model, "tokens": int(tokens)} + for (date, model), tokens in sorted(by_model.items()) + ] + return token_usage_by_model, { + key: _dashboard_ranked(totals, "tokens") for key, totals in consumers.items() + } + + +def _dashboard_activity_insights(start_date, end_date): + """Rank recorded activity and build the UTC login heatmap from one projection. + + Each heatmap cell totals every login in the period for one weekday and hour. + """ + actors = {key: defaultdict(int) for key, _ in CONTROL_CENTER_DASHBOARD_RANKED_FIELDS} + logins = Counter() + rows = cosmos_activity_logs_container.query_items( + query=""" + SELECT c.timestamp, + c.activity_type, + c.user_id AS user_id, + c.workspace_context.group_id AS group_id, + c.workspace_context.public_workspace_id AS public_workspace_id + FROM c + WHERE c.timestamp >= @start_date + AND c.timestamp <= @end_date + """, + parameters=[ + {"name": "@start_date", "value": start_date.isoformat()}, + {"name": "@end_date", "value": end_date.isoformat()}, + ], + enable_cross_partition_query=True, + ) + for row in rows: + for key, field in CONTROL_CENTER_DASHBOARD_RANKED_FIELDS: + entity_id = row.get(field) + if isinstance(entity_id, str) and entity_id: + actors[key][entity_id] += 1 + timestamp = row.get("timestamp") + if row.get("activity_type") != "user_login" or not isinstance(timestamp, str): + continue + try: + weekday = datetime.strptime(timestamp[:10], "%Y-%m-%d").weekday() + hour = int(timestamp[11:13]) + except ValueError: + continue + if 0 <= hour <= 23: + logins[(weekday, hour)] += 1 + login_cells = [ + {"weekday": weekday, "hour": hour, "count": count} + for (weekday, hour), count in sorted(logins.items()) + ] + return { + key: _dashboard_ranked(counts, "activity_count") for key, counts in actors.items() + }, login_cells + + def parse_control_center_management_pagination(request_args): """Parse shared Control Center management pagination query parameters.""" try: @@ -1474,6 +1612,7 @@ def enhance_user_with_activity(user, force_refresh=False): "documents": "c.settings.metrics.document_metrics.total_documents", "tokens": "c.settings.metrics.token_metrics.total_tokens", } +CONTROL_CENTER_USER_FIELDS = "c.id, c.email, c.display_name, c.settings" def _control_center_validate_user_id(user_id): @@ -1603,6 +1742,93 @@ def add_clause(clause, name, value): return (" AND ".join(clauses) if clauses else "1=1"), parameters +def _control_center_user_populations(filters): + """Split users by whether the sort property is recorded; each part orders one property. + + An ORDER BY over two properties needs a composite index that user_settings does not + have. Users with a recorded sort value are ordered by that property alone, and users + without one follow in ID order, last in either direction. + """ + field = CONTROL_CENTER_USER_SORTS[filters["sort"]] + recorded = f"(IS_DEFINED({field}) AND NOT IS_NULL({field}))" + return ( + (recorded, f"ORDER BY {field} {filters['direction'].upper()}"), + (f"NOT {recorded}", "ORDER BY c.id ASC"), + ) + + +def _control_center_count_users(container, where_clause, parameters): + """Count the users matching a parameterized WHERE clause.""" + rows = list(container.query_items( + query=f"SELECT VALUE COUNT(1) FROM c WHERE {where_clause}", + parameters=parameters, + enable_cross_partition_query=True, + )) + return int(rows[0] or 0) if rows else 0 + + +def _control_center_query_user_page(container, filters, page, per_page): + """Return one page of filtered users with the total, clamped page and page count. + + A page can end the recorded population and continue into the missing one. + """ + where_clause, parameters = _control_center_user_where(filters) + (recorded_clause, recorded_order), (missing_clause, missing_order) = ( + _control_center_user_populations(filters) + ) + total = _control_center_count_users(container, where_clause, parameters) + recorded = _control_center_count_users( + container, + f"({where_clause}) AND {recorded_clause}", + parameters, + ) + total_pages = get_control_center_total_pages(total, per_page) + page = clamp_control_center_page(page, total_pages) + offset = (page - 1) * per_page + user_docs = [] + if offset < recorded: + user_docs.extend(container.query_items( + query=( + f"SELECT {CONTROL_CENTER_USER_FIELDS} FROM c " + f"WHERE ({where_clause}) AND {recorded_clause} {recorded_order} " + "OFFSET @offset LIMIT @limit" + ), + parameters=parameters + [ + {"name": "@offset", "value": offset}, + {"name": "@limit", "value": min(per_page, recorded - offset)}, + ], + enable_cross_partition_query=True, + )) + if len(user_docs) < per_page and total > recorded: + user_docs.extend(container.query_items( + query=( + f"SELECT {CONTROL_CENTER_USER_FIELDS} FROM c " + f"WHERE ({where_clause}) AND {missing_clause} {missing_order} " + "OFFSET @offset LIMIT @limit" + ), + parameters=parameters + [ + {"name": "@offset", "value": max(0, offset - recorded)}, + {"name": "@limit", "value": per_page - len(user_docs)}, + ], + enable_cross_partition_query=True, + )) + return user_docs, total, page, total_pages + + +def _control_center_iter_users(container, filters): + """Yield every filtered user: recorded sort values in order, then the rest by ID.""" + where_clause, parameters = _control_center_user_where(filters) + for clause, order in _control_center_user_populations(filters): + yield from container.query_items( + query=( + f"SELECT {CONTROL_CENTER_USER_FIELDS} FROM c " + f"WHERE ({where_clause}) AND {clause} {order}" + ), + parameters=parameters, + enable_cross_partition_query=True, + ) + + def _control_center_effective_restriction(settings, setting_key, now=None): """Return the effective allow/deny state and any stored expiry for one user setting.""" value = settings.get(setting_key, {}) @@ -3762,31 +3988,12 @@ def api_v2_control_center_users(): try: filters = _control_center_parse_user_filters(request.args) page, per_page = parse_control_center_management_pagination(request.args) - where_clause, parameters = _control_center_user_where(filters) - - count_result = list(cosmos_user_settings_container.query_items( - query=f"SELECT VALUE COUNT(1) FROM c WHERE {where_clause}", - parameters=parameters, - enable_cross_partition_query=True, - )) - total = int(count_result[0] or 0) if count_result else 0 - total_pages = get_control_center_total_pages(total, per_page) - page = clamp_control_center_page(page, total_pages) - sort_expression = CONTROL_CENTER_USER_SORTS[filters["sort"]] - query = ( - "SELECT c.id, c.email, c.display_name, c.settings FROM c " - f"WHERE {where_clause} ORDER BY {sort_expression} {filters['direction'].upper()}, c.id ASC " - "OFFSET @offset LIMIT @limit" + user_docs, total, page, total_pages = _control_center_query_user_page( + cosmos_user_settings_container, + filters, + page, + per_page, ) - paged_parameters = parameters + [ - {"name": "@offset", "value": (page - 1) * per_page}, - {"name": "@limit", "value": per_page}, - ] - user_docs = list(cosmos_user_settings_container.query_items( - query=query, - parameters=paged_parameters, - enable_cross_partition_query=True, - )) users = [_control_center_user_row(user, filters["now"]) for user in user_docs] metric_times = sorted( user["metrics_calculated_at"] for user in users @@ -4106,17 +4313,14 @@ def api_v2_control_center_users_export(): """Stream a CSV export of users matching the same filters as the Users list.""" try: filters = _control_center_parse_user_filters(request.args) - where_clause, parameters = _control_center_user_where(filters) - sort_expression = CONTROL_CENTER_USER_SORTS[filters["sort"]] - query = ( - "SELECT c.id, c.email, c.display_name, c.settings FROM c " - f"WHERE {where_clause} ORDER BY {sort_expression} {filters['direction'].upper()}, c.id ASC" - ) - user_docs = cosmos_user_settings_container.query_items( - query=query, - parameters=parameters, - enable_cross_partition_query=True, - ) + user_docs = _control_center_iter_users(cosmos_user_settings_container, filters) + # Run the first query now so a storage failure returns an error, not a truncated CSV. + first_user = next(user_docs, None) + + def remaining_users(): + if first_user is not None: + yield first_user + yield from user_docs def stream_csv(): buffer = StringIO() @@ -4130,7 +4334,7 @@ def stream_csv(): yield buffer.getvalue() buffer.seek(0) buffer.truncate(0) - for user_doc in user_docs: + for user_doc in remaining_users(): row = _control_center_user_row(user_doc, filters["now"]) values = ( row["id"], row["display_name"], row["email"], @@ -6719,28 +6923,14 @@ def api_v2_control_center_dashboard_summary(): cosmos_groups_container, 'SELECT VALUE COUNT(1) FROM c', ) - group_statuses = list(cosmos_groups_container.query_items( - query=""" - SELECT c.status AS status, COUNT(1) AS count FROM c - WHERE IS_DEFINED(c.status) - GROUP BY c.status - """, - enable_cross_partition_query=True, - )) + group_statuses = _dashboard_status_rows(cosmos_groups_container) groups_by_status = _dashboard_status_counts(groups_total, group_statuses) workspaces_total = _dashboard_query_count( cosmos_public_workspaces_container, 'SELECT VALUE COUNT(1) FROM c', ) - workspace_statuses = list(cosmos_public_workspaces_container.query_items( - query=""" - SELECT c.status AS status, COUNT(1) AS count FROM c - WHERE IS_DEFINED(c.status) - GROUP BY c.status - """, - enable_cross_partition_query=True, - )) + workspace_statuses = _dashboard_status_rows(cosmos_public_workspaces_container) workspaces_by_status = _dashboard_status_counts( workspaces_total, workspace_statuses, @@ -6926,100 +7116,12 @@ def api_v2_control_center_dashboard_insights(): return jsonify({**cached, 'cached': True}) try: - token_where, token_parameters = build_token_usage_query_context( + token_usage_by_model, top_tokens = _dashboard_token_insights( start_date, end_date, - token_filters=token_filters, + token_filters, ) - model_rows = list(cosmos_activity_logs_container.query_items( - query=f""" - SELECT SUBSTRING(c.timestamp, 0, 10) AS date, - c.usage.model AS model, - SUM(c.usage.total_tokens) AS tokens - FROM c - WHERE {token_where} - AND IS_DEFINED(c.timestamp) - AND IS_DEFINED(c.usage.model) - GROUP BY SUBSTRING(c.timestamp, 0, 10), c.usage.model - """, - parameters=token_parameters, - enable_cross_partition_query=True, - )) - - def ranked_token_totals(field_path): - rows = list(cosmos_activity_logs_container.query_items( - query=f""" - SELECT c.{field_path} AS id, SUM(c.usage.total_tokens) AS tokens - FROM c - WHERE {token_where} AND IS_DEFINED(c.{field_path}) - GROUP BY c.{field_path} - """, - parameters=token_parameters, - enable_cross_partition_query=True, - )) - return sorted( - ( - {'id': row.get('id'), 'tokens': int(row.get('tokens') or 0)} - for row in rows if row.get('id') - ), - key=lambda item: item['tokens'], - reverse=True, - )[:10] - - def ranked_activity_totals(field_path): - rows = list(cosmos_activity_logs_container.query_items( - query=f""" - SELECT c.{field_path} AS id, COUNT(1) AS activity_count - FROM c - WHERE c.timestamp >= @start_date - AND c.timestamp <= @end_date - AND IS_DEFINED(c.{field_path}) - GROUP BY c.{field_path} - """, - parameters=[ - {'name': '@start_date', 'value': start_date.isoformat()}, - {'name': '@end_date', 'value': end_date.isoformat()}, - ], - enable_cross_partition_query=True, - )) - return sorted( - ( - {'id': row.get('id'), 'activity_count': int(row.get('activity_count') or 0)} - for row in rows if row.get('id') - ), - key=lambda item: item['activity_count'], - reverse=True, - )[:10] - - login_rows = list(cosmos_activity_logs_container.query_items( - query=""" - SELECT SUBSTRING(c.timestamp, 0, 10) AS date, - SUBSTRING(c.timestamp, 11, 2) AS hour, - COUNT(1) AS count - FROM c - WHERE c.activity_type = 'user_login' - AND c.timestamp >= @start_date - AND c.timestamp <= @end_date - GROUP BY SUBSTRING(c.timestamp, 0, 10), SUBSTRING(c.timestamp, 11, 2) - """, - parameters=[ - {'name': '@start_date', 'value': start_date.isoformat()}, - {'name': '@end_date', 'value': end_date.isoformat()}, - ], - enable_cross_partition_query=True, - )) - login_heatmap = [] - for row in login_rows: - try: - weekday = datetime.strptime(row['date'], '%Y-%m-%d').weekday() - hour = int(row['hour']) - login_heatmap.append({ - 'weekday': weekday, - 'hour': hour, - 'count': int(row.get('count') or 0), - }) - except (KeyError, TypeError, ValueError): - continue + top_activity, login_cells = _dashboard_activity_insights(start_date, end_date) response_data = { 'period': { @@ -7028,32 +7130,13 @@ def ranked_activity_totals(field_path): 'days': period_days, 'timezone': 'UTC', }, - 'token_usage_by_model': [ - { - 'date': row.get('date'), - 'model': row.get('model') or 'Unknown model', - 'tokens': int(row.get('tokens') or 0), - } - for row in model_rows - ], - 'top_tokens': { - 'users': ranked_token_totals('user_id'), - 'groups': ranked_token_totals('workspace_context.group_id'), - 'public_workspaces': ranked_token_totals( - 'workspace_context.public_workspace_id' - ), - }, - 'top_activity': { - 'users': ranked_activity_totals('user_id'), - 'groups': ranked_activity_totals('workspace_context.group_id'), - 'public_workspaces': ranked_activity_totals( - 'workspace_context.public_workspace_id' - ), - }, + 'token_usage_by_model': token_usage_by_model, + 'top_tokens': top_tokens, + 'top_activity': top_activity, 'login_heatmap': { 'weekday_convention': 'Monday=0 through Sunday=6', 'timezone': 'UTC', - 'cells': login_heatmap, + 'cells': login_cells, }, } _dashboard_cache_set(cache_key, response_data) @@ -7758,477 +7841,6 @@ def api_get_refresh_status(): debug_print(f"Error getting refresh status: {e}") return jsonify({'error': 'Failed to get refresh status'}), 500 - # Activity Log Migration APIs - @bp.route('/api/admin/control-center/migrate/status', methods=['GET']) - @swagger_route(security=get_auth_security()) - @login_required - @control_center_required('admin') - def api_get_migration_status(): - """ - Check if there are conversations and documents that need to be migrated to activity logs. - Returns counts of records without the 'added_to_activity_log' flag. - """ - try: - migration_status = { - 'conversations_without_logs': 0, - 'personal_documents_without_logs': 0, - 'group_documents_without_logs': 0, - 'public_documents_without_logs': 0, - 'total_documents_without_logs': 0, - 'migration_needed': False, - 'estimated_total_records': 0 - } - - # Check conversations without the flag - try: - conversations_query = """ - SELECT VALUE COUNT(1) - FROM c - WHERE NOT IS_DEFINED(c.added_to_activity_log) OR c.added_to_activity_log = false - """ - conversations_result = list(cosmos_conversations_container.query_items( - query=conversations_query, - enable_cross_partition_query=True - )) - migration_status['conversations_without_logs'] = conversations_result[0] if conversations_result else 0 - except Exception as e: - debug_print(f"Error checking conversations migration status: {e}") - - # Check personal documents without the flag - try: - personal_docs_query = """ - SELECT VALUE COUNT(1) - FROM c - WHERE NOT IS_DEFINED(c.added_to_activity_log) OR c.added_to_activity_log = false - """ - personal_docs_result = list(cosmos_user_documents_container.query_items( - query=personal_docs_query, - enable_cross_partition_query=True - )) - migration_status['personal_documents_without_logs'] = personal_docs_result[0] if personal_docs_result else 0 - except Exception as e: - debug_print(f"Error checking personal documents migration status: {e}") - - # Check group documents without the flag - try: - group_docs_query = """ - SELECT VALUE COUNT(1) - FROM c - WHERE NOT IS_DEFINED(c.added_to_activity_log) OR c.added_to_activity_log = false - """ - group_docs_result = list(cosmos_group_documents_container.query_items( - query=group_docs_query, - enable_cross_partition_query=True - )) - migration_status['group_documents_without_logs'] = group_docs_result[0] if group_docs_result else 0 - except Exception as e: - debug_print(f"Error checking group documents migration status: {e}") - - # Check public documents without the flag - try: - public_docs_query = """ - SELECT VALUE COUNT(1) - FROM c - WHERE NOT IS_DEFINED(c.added_to_activity_log) OR c.added_to_activity_log = false - """ - public_docs_result = list(cosmos_public_documents_container.query_items( - query=public_docs_query, - enable_cross_partition_query=True - )) - migration_status['public_documents_without_logs'] = public_docs_result[0] if public_docs_result else 0 - except Exception as e: - debug_print(f"Error checking public documents migration status: {e}") - - # Calculate totals - migration_status['total_documents_without_logs'] = ( - migration_status['personal_documents_without_logs'] + - migration_status['group_documents_without_logs'] + - migration_status['public_documents_without_logs'] - ) - - migration_status['estimated_total_records'] = ( - migration_status['conversations_without_logs'] + - migration_status['total_documents_without_logs'] - ) - - migration_status['migration_needed'] = migration_status['estimated_total_records'] > 0 - - return jsonify(migration_status), 200 - - except Exception as e: - debug_print(f"Error getting migration status: {e}") - return jsonify({'error': 'Failed to get migration status'}), 500 - - @bp.route('/api/admin/control-center/migrate/all', methods=['POST']) - @swagger_route(security=get_auth_security()) - @login_required - @control_center_required('admin') - def api_migrate_to_activity_logs(): - """ - Migrate all conversations and documents without activity logs. - This adds activity log records and sets the 'added_to_activity_log' flag. - - WARNING: This may take a while for large datasets and could impact performance. - Recommended to run during off-peak hours. - """ - try: - from functions_activity_logging import log_conversation_creation, log_document_creation_transaction - - results = { - 'conversations_migrated': 0, - 'conversations_failed': 0, - 'conversations_skipped_existing': 0, - 'personal_documents_migrated': 0, - 'personal_documents_failed': 0, - 'personal_documents_skipped_existing': 0, - 'group_documents_migrated': 0, - 'group_documents_failed': 0, - 'group_documents_skipped_existing': 0, - 'public_documents_migrated': 0, - 'public_documents_failed': 0, - 'public_documents_skipped_existing': 0, - 'total_migrated': 0, - 'total_skipped_existing': 0, - 'total_failed': 0, - 'errors': [] - } - - # Migrate conversations - debug_print("Starting conversation migration...") - try: - conversations_query = """ - SELECT * - FROM c - WHERE NOT IS_DEFINED(c.added_to_activity_log) OR c.added_to_activity_log = false - """ - conversations = list(cosmos_conversations_container.query_items( - query=conversations_query, - enable_cross_partition_query=True - )) - - debug_print(f"Found {len(conversations)} conversations to migrate") - - for conv in conversations: - try: - if has_activity_log_for_resource( - conv.get('user_id'), 'conversation_creation', conv.get('id') - ): - conv['added_to_activity_log'] = True - cosmos_conversations_container.upsert_item(conv) - results['conversations_skipped_existing'] += 1 - continue - - # Create activity log directly to preserve original timestamp - activity_log = { - 'id': build_activity_log_id( - 'conversation_creation', - conv.get('user_id'), - f"backfill:{conv.get('id')}", - ), - 'activity_type': 'conversation_creation', - 'user_id': conv.get('user_id'), - 'timestamp': conv.get('created_at') or conv.get('last_updated') or datetime.utcnow().isoformat(), - 'created_at': conv.get('created_at') or conv.get('last_updated') or datetime.utcnow().isoformat(), - 'conversation': { - 'conversation_id': conv.get('id'), - 'title': conv.get('title', 'Untitled'), - 'context': conv.get('context', []), - 'tags': conv.get('tags', []) - }, - 'workspace_type': 'personal', - 'workspace_context': {} - } - - # Save to activity logs container - cosmos_activity_logs_container.upsert_item(activity_log) - - # Add flag to conversation - conv['added_to_activity_log'] = True - cosmos_conversations_container.upsert_item(conv) - - results['conversations_migrated'] += 1 - - except Exception as conv_error: - results['conversations_failed'] += 1 - error_msg = f"Failed to migrate conversation {conv.get('id')}: {str(conv_error)}" - debug_print(error_msg) - results['errors'].append(error_msg) - - except Exception as e: - error_msg = f"Error during conversation migration: {str(e)}" - debug_print(error_msg) - results['errors'].append(error_msg) - - # Migrate personal documents - debug_print("Starting personal documents migration...") - try: - personal_docs_query = """ - SELECT * - FROM c - WHERE NOT IS_DEFINED(c.added_to_activity_log) OR c.added_to_activity_log = false - """ - personal_docs = list(cosmos_user_documents_container.query_items( - query=personal_docs_query, - enable_cross_partition_query=True - )) - - for doc in personal_docs: - try: - if has_activity_log_for_resource( - doc.get('user_id'), 'document_creation', doc.get('id'), 'personal' - ): - doc['added_to_activity_log'] = True - cosmos_user_documents_container.upsert_item(doc) - results['personal_documents_skipped_existing'] += 1 - continue - - # Create activity log directly to preserve original timestamp - activity_log = { - 'id': build_activity_log_id( - 'document_creation', - doc.get('user_id'), - f"backfill:personal:{doc.get('id')}", - ), - 'user_id': doc.get('user_id'), - 'activity_type': 'document_creation', - 'workspace_type': 'personal', - 'timestamp': doc.get('upload_date') or datetime.utcnow().isoformat(), - 'created_at': doc.get('upload_date') or datetime.utcnow().isoformat(), - 'document': { - 'document_id': doc.get('id'), - 'file_name': doc.get('file_name', 'Unknown'), - 'file_type': doc.get('file_type', 'unknown'), - 'file_size_bytes': doc.get('file_size', 0), - 'page_count': doc.get('number_of_pages', 0), - 'version': doc.get('version', 1) - }, - 'embedding_usage': { - 'total_tokens': doc.get('embedding_tokens', 0), - 'model_deployment_name': doc.get('embedding_model_deployment_name', 'unknown') - }, - 'document_metadata': { - 'author': doc.get('author'), - 'title': doc.get('title'), - 'subject': doc.get('subject'), - 'publication_date': doc.get('publication_date'), - 'keywords': doc.get('keywords', []), - 'abstract': doc.get('abstract') - }, - 'workspace_context': {} - } - - # Save to activity logs container - cosmos_activity_logs_container.upsert_item(activity_log) - - # Add flag to document - doc['added_to_activity_log'] = True - cosmos_user_documents_container.upsert_item(doc) - - results['personal_documents_migrated'] += 1 - - except Exception as doc_error: - results['personal_documents_failed'] += 1 - error_msg = f"Failed to migrate personal document {doc.get('id')}: {str(doc_error)}" - debug_print(error_msg) - results['errors'].append(error_msg) - - except Exception as e: - error_msg = f"Error during personal documents migration: {str(e)}" - debug_print(error_msg) - results['errors'].append(error_msg) - - # Migrate group documents - debug_print("Starting group documents migration...") - try: - group_docs_query = """ - SELECT * - FROM c - WHERE NOT IS_DEFINED(c.added_to_activity_log) OR c.added_to_activity_log = false - """ - group_docs = list(cosmos_group_documents_container.query_items( - query=group_docs_query, - enable_cross_partition_query=True - )) - - for doc in group_docs: - try: - if has_activity_log_for_resource( - doc.get('user_id'), 'document_creation', doc.get('id'), 'group' - ): - doc['added_to_activity_log'] = True - cosmos_group_documents_container.upsert_item(doc) - results['group_documents_skipped_existing'] += 1 - continue - - # Create activity log directly to preserve original timestamp - activity_log = { - 'id': build_activity_log_id( - 'document_creation', - doc.get('user_id'), - f"backfill:group:{doc.get('id')}", - ), - 'user_id': doc.get('user_id'), - 'activity_type': 'document_creation', - 'workspace_type': 'group', - 'timestamp': doc.get('upload_date') or datetime.utcnow().isoformat(), - 'created_at': doc.get('upload_date') or datetime.utcnow().isoformat(), - 'document': { - 'document_id': doc.get('id'), - 'file_name': doc.get('file_name', 'Unknown'), - 'file_type': doc.get('file_type', 'unknown'), - 'file_size_bytes': doc.get('file_size', 0), - 'page_count': doc.get('number_of_pages', 0), - 'version': doc.get('version', 1) - }, - 'embedding_usage': { - 'total_tokens': doc.get('embedding_tokens', 0), - 'model_deployment_name': doc.get('embedding_model_deployment_name', 'unknown') - }, - 'document_metadata': { - 'author': doc.get('author'), - 'title': doc.get('title'), - 'subject': doc.get('subject'), - 'publication_date': doc.get('publication_date'), - 'keywords': doc.get('keywords', []), - 'abstract': doc.get('abstract') - }, - 'workspace_context': { - 'group_id': doc.get('group_id') - } - } - - # Save to activity logs container - cosmos_activity_logs_container.upsert_item(activity_log) - - # Add flag to document - doc['added_to_activity_log'] = True - cosmos_group_documents_container.upsert_item(doc) - - results['group_documents_migrated'] += 1 - - except Exception as doc_error: - results['group_documents_failed'] += 1 - error_msg = f"Failed to migrate group document {doc.get('id')}: {str(doc_error)}" - debug_print(error_msg) - results['errors'].append(error_msg) - - except Exception as e: - error_msg = f"Error during group documents migration: {str(e)}" - debug_print(error_msg) - results['errors'].append(error_msg) - - # Migrate public documents - debug_print("Starting public documents migration...") - try: - public_docs_query = """ - SELECT * - FROM c - WHERE NOT IS_DEFINED(c.added_to_activity_log) OR c.added_to_activity_log = false - """ - public_docs = list(cosmos_public_documents_container.query_items( - query=public_docs_query, - enable_cross_partition_query=True - )) - - for doc in public_docs: - try: - if has_activity_log_for_resource( - doc.get('user_id'), 'document_creation', doc.get('id'), 'public' - ): - doc['added_to_activity_log'] = True - cosmos_public_documents_container.upsert_item(doc) - results['public_documents_skipped_existing'] += 1 - continue - - # Create activity log directly to preserve original timestamp - activity_log = { - 'id': build_activity_log_id( - 'document_creation', - doc.get('user_id'), - f"backfill:public:{doc.get('id')}", - ), - 'user_id': doc.get('user_id'), - 'activity_type': 'document_creation', - 'workspace_type': 'public', - 'timestamp': doc.get('upload_date') or datetime.utcnow().isoformat(), - 'created_at': doc.get('upload_date') or datetime.utcnow().isoformat(), - 'document': { - 'document_id': doc.get('id'), - 'file_name': doc.get('file_name', 'Unknown'), - 'file_type': doc.get('file_type', 'unknown'), - 'file_size_bytes': doc.get('file_size', 0), - 'page_count': doc.get('number_of_pages', 0), - 'version': doc.get('version', 1) - }, - 'embedding_usage': { - 'total_tokens': doc.get('embedding_tokens', 0), - 'model_deployment_name': doc.get('embedding_model_deployment_name', 'unknown') - }, - 'document_metadata': { - 'author': doc.get('author'), - 'title': doc.get('title'), - 'subject': doc.get('subject'), - 'publication_date': doc.get('publication_date'), - 'keywords': doc.get('keywords', []), - 'abstract': doc.get('abstract') - }, - 'workspace_context': { - 'public_workspace_id': doc.get('public_workspace_id') - } - } - - # Save to activity logs container - cosmos_activity_logs_container.upsert_item(activity_log) - - # Add flag to document - doc['added_to_activity_log'] = True - cosmos_public_documents_container.upsert_item(doc) - - results['public_documents_migrated'] += 1 - - except Exception as doc_error: - results['public_documents_failed'] += 1 - error_msg = f"Failed to migrate public document {doc.get('id')}: {str(doc_error)}" - debug_print(error_msg) - results['errors'].append(error_msg) - - except Exception as e: - error_msg = f"Error during public documents migration: {str(e)}" - debug_print(error_msg) - results['errors'].append(error_msg) - - # Calculate totals - results['total_migrated'] = ( - results['conversations_migrated'] + - results['personal_documents_migrated'] + - results['group_documents_migrated'] + - results['public_documents_migrated'] - ) - - results['total_skipped_existing'] = ( - results['conversations_skipped_existing'] + - results['personal_documents_skipped_existing'] + - results['group_documents_skipped_existing'] + - results['public_documents_skipped_existing'] - ) - - results['total_failed'] = ( - results['conversations_failed'] + - results['personal_documents_failed'] + - results['group_documents_failed'] + - results['public_documents_failed'] - ) - - debug_print(f"Migration complete: {results['total_migrated']} migrated, {results['total_failed']} failed") - - return jsonify(results), 200 - - except Exception as e: - debug_print(f"Error during migration: {e}") - import traceback - traceback.print_exc() - return jsonify({'error': f'Migration failed: {str(e)}'}), 500 - @bp.route('/api/admin/control-center/activity-logs', methods=['GET']) @swagger_route(security=get_auth_security()) @login_required diff --git a/application/v2_ui/src/pages/ControlCenterPage.tsx b/application/v2_ui/src/pages/ControlCenterPage.tsx index cbe6dcdb0..5742d39cf 100644 --- a/application/v2_ui/src/pages/ControlCenterPage.tsx +++ b/application/v2_ui/src/pages/ControlCenterPage.tsx @@ -1,18 +1,15 @@ // ControlCenterPage.tsx // THESIS: Put governed administration into its own pane, not the user workspace rail. // OWN-WORLD: Inherit SimpleChat V2's semantic glass surfaces, compact workhorse type, and blue accent. -// STORY: Administrators see which areas are available; Data health is explicitly on-demand. +// STORY: Administrators see only the management areas their Control Center capabilities allow. // FIRST VIEWPORT: A labeled section rail sits beside a broad page header and one focused work area. // FORM: Operate; extend the established Admin Settings pane without changing its visual system. -import { useState } from 'react'; import { NavLink, useNavigate, useParams } from 'react-router-dom'; -import { Activity, BarChart3, Database, FolderOpen, PanelLeftClose, PanelLeftOpen, Users } from 'lucide-react'; +import { Activity, BarChart3, FolderOpen, PanelLeftClose, PanelLeftOpen, Users } from 'lucide-react'; import { clsx } from 'clsx'; -import { api } from '../lib/apiClient'; import { PageHeader } from '../components/layout/PageHeader'; -import { ConfirmDialog } from '../components/ui/ConfirmDialog'; -import { GlassButton, GlassPanel } from '../components/ui/primitives'; +import { GlassPanel } from '../components/ui/primitives'; import { DashboardSection } from '../components/controlCenter/DashboardSection'; import { ActivityLogsSection } from '../components/controlCenter/ActivityLogsSection'; import type { ControlCenterCapabilities } from '../lib/types'; @@ -22,31 +19,7 @@ import { UsersSection } from '../components/controlCenter/UsersSection'; import { GroupsSection } from '../components/controlCenter/GroupsSection'; import { PublicWorkspacesSection } from '../components/controlCenter/PublicWorkspacesSection'; -type SectionId = 'dashboard' | 'users' | 'groups' | 'public-workspaces' | 'activity-logs' | 'data-health'; - -interface MigrationStatus { - conversations_without_logs: number; - personal_documents_without_logs: number; - group_documents_without_logs: number; - public_documents_without_logs: number; - total_documents_without_logs: number; - migration_needed: boolean; - estimated_total_records: number; -} - -interface MigrationResult { - conversations_migrated: number; - conversations_skipped_existing: number; - personal_documents_migrated: number; - personal_documents_skipped_existing: number; - group_documents_migrated: number; - group_documents_skipped_existing: number; - public_documents_migrated: number; - public_documents_skipped_existing: number; - total_migrated: number; - total_skipped_existing: number; - total_failed: number; -} +type SectionId = 'dashboard' | 'users' | 'groups' | 'public-workspaces' | 'activity-logs'; const SECTIONS: { id: SectionId; @@ -59,109 +32,8 @@ const SECTIONS: { { id: 'groups', label: 'Groups', Icon: Users, capability: 'can_manage_groups' }, { id: 'public-workspaces', label: 'Public Workspaces', Icon: FolderOpen, capability: 'can_manage_workspaces' }, { id: 'activity-logs', label: 'Activity Logs', Icon: Activity, capability: 'can_view_activity_logs' }, - { id: 'data-health', label: 'Data health', Icon: Database, capability: 'can_run_maintenance' }, ]; -function MigrationDataHealth() { - const [checking, setChecking] = useState(false); - const [running, setRunning] = useState(false); - const [status, setStatus] = useState(null); - const [result, setResult] = useState(null); - const [error, setError] = useState(null); - const [confirmOpen, setConfirmOpen] = useState(false); - - const check = async () => { - setChecking(true); - setError(null); - setResult(null); - try { - setStatus(await api.get('/api/admin/control-center/migrate/status')); - } catch (requestError) { - setError(requestError instanceof Error ? requestError.message : 'Unable to check activity-log status.'); - } finally { - setChecking(false); - } - }; - - const runBackfill = async () => { - setConfirmOpen(false); - setRunning(true); - setError(null); - setResult(null); - try { - setResult(await api.post('/api/admin/control-center/migrate/all')); - } catch (requestError) { - setError(requestError instanceof Error ? requestError.message : 'Unable to run the activity-log backfill.'); - } finally { - setRunning(false); - } - }; - - return ( -
-
-

Activity-log data health

-

- The legacy backfill adds conversation and document creation records to activity logs when those - records are missing. Normal application workflows already write activity logs, so this maintenance - operation is usually unnecessary. Check the counts before deciding to run it. -

-
-
- void check()}> - {checking ? 'Checking…' : 'Check activity-log status'} - - setConfirmOpen(true)}> - {running ? 'Running backfill…' : 'Run backfill'} - -
- {error ?

{error}

: null} - {status ? ( - -

Check results

-
- {[ - ['Conversations without the legacy flag', status.conversations_without_logs], - ['Personal documents without the legacy flag', status.personal_documents_without_logs], - ['Group documents without the legacy flag', status.group_documents_without_logs], - ['Public documents without the legacy flag', status.public_documents_without_logs], - ['Estimated records', status.estimated_total_records], - ].map(([label, value]) => ( -
-
{label}
-
{Number(value).toLocaleString()}
-
- ))} -
-

- {status.migration_needed - ? 'These counts identify records missing the legacy flag, not necessarily missing activity logs.' - : 'No records are missing the legacy flag.'} -

-
- ) : null} - {result ? ( - -

Backfill complete

-

- {result.total_migrated.toLocaleString()} records added; - {' '}{result.total_skipped_existing.toLocaleString()} existing activity records skipped; - {' '}{result.total_failed.toLocaleString()} failures. -

-
- ) : null} - {confirmOpen ? ( - setConfirmOpen(false)} - onConfirm={() => void runBackfill()}> -

The operation can take time on large datasets. It is safe to run again; records already present will be skipped.

-
- ) : null} -
- ); -} - function SectionPlaceholder({ label }: { label: string }) { return ( @@ -182,7 +54,7 @@ export function ControlCenterPage() { const navigate = useNavigate(); const railCollapsed = useUserSettingsStore((state) => state.settings.v2ControlCenterRailCollapsed === true); const updateUserSettings = useUserSettingsStore((state) => state.update); - const sections: SectionId[] = ['dashboard', 'users', 'groups', 'public-workspaces', 'activity-logs', 'data-health']; + const sections: SectionId[] = ['dashboard', 'users', 'groups', 'public-workspaces', 'activity-logs']; const section = (sections.includes(requestedSection as SectionId) ? requestedSection : 'dashboard') as SectionId; const current = SECTIONS.find((item) => item.id === section) ?? SECTIONS[0]; const allowed = Boolean(capabilities?.[current.capability]); @@ -233,7 +105,6 @@ export function ControlCenterPage() { : section === 'groups' ? : section === 'activity-logs' ? : section === 'public-workspaces' ? - : section === 'data-health' ? : } diff --git a/docs/explanation/features/V2_CONTROL_CENTER.md b/docs/explanation/features/V2_CONTROL_CENTER.md index 63648c68a..66b031fbf 100644 --- a/docs/explanation/features/V2_CONTROL_CENTER.md +++ b/docs/explanation/features/V2_CONTROL_CENTER.md @@ -1,6 +1,6 @@ # V2 Control Center -The V2 Control Center is a permission-aware administration pane for managing SimpleChat. It provides a usage dashboard, user/group/public-workspace management, activity investigations, and an explicitly invoked activity-log data-health tool. +The V2 Control Center is a permission-aware administration pane for managing SimpleChat. It provides a usage dashboard, user/group/public-workspace management, and activity investigations. **Dashboard implemented in version:** 0.261.279 **Foundation implemented in version:** 0.261.278 @@ -8,7 +8,7 @@ The V2 Control Center is a permission-aware administration pane for managing Sim **Groups implemented in version:** 0.261.282 **Activity Logs implemented in version:** 0.261.284 **Public Workspaces implemented in version:** 0.261.283 -**Current version:** 0.261.286 (Public Workspace validation-safety fixes above the integrated management phases) +**Current version:** 0.261.290 (Dashboard, Users and Groups query fixes for the Python Cosmos SDK; Data health removed) **Dependencies:** React 18, TypeScript, Vite, Flask session authentication, and the existing Control Center APIs. @@ -16,7 +16,7 @@ The V2 Control Center is a permission-aware administration pane for managing Sim The Control Center is a distinct React route (`/control-center` and `/control-center/
`), reached from the account menu when the signed-in user has at least one Control Center capability. It is not a primary workspace-navigation item. The internal section rail has a separate per-user collapsed-state preference. -`get_control_center_capabilities()` in `functions_authentication.py` is the shared permission decision for both `control_center_required()` and the `/api/v2/bootstrap` response. When the ControlCenterAdmin role requirement is enabled, that role grants all capabilities; otherwise the regular Admin role grants them. The optional ControlCenterDashboardReader role grants dashboard viewing only when its setting is enabled. The bootstrap payload exposes `can_view_dashboard`, `can_manage_users`, `can_manage_groups`, `can_manage_workspaces`, `can_view_activity_logs`, and `can_run_maintenance`. +`get_control_center_capabilities()` in `functions_authentication.py` is the shared permission decision for both `control_center_required()` and the `/api/v2/bootstrap` response. When the ControlCenterAdmin role requirement is enabled, that role grants all capabilities; otherwise the regular Admin role grants them. The optional ControlCenterDashboardReader role grants dashboard viewing only when its setting is enabled. The bootstrap payload exposes `can_view_dashboard`, `can_manage_users`, `can_manage_groups`, `can_manage_workspaces`, `can_view_activity_logs`, and `can_run_maintenance`. Since Data health was removed in 0.261.290, no V2 section uses `can_run_maintenance`; it remains part of the shared capability contract. ## Dashboard @@ -24,6 +24,8 @@ Users, Groups and Public Workspaces management are available with `can_manage_us The Dashboard is available to users with `can_view_dashboard`, including users assigned the configured ControlCenterDashboardReader role. `GET /api/v2/control-center/dashboard/summary` returns counts and period comparisons; `GET /api/v2/control-center/dashboard/insights` returns grouped chart data. Both use a 90-second in-process cache keyed by the date range and token filters. Pass `force_refresh=1` to bypass it. +Neither endpoint runs cross-partition `GROUP BY` or `COUNT` over `DISTINCT` values: the azure-cosmos Python SDK cannot run either, and Cosmos rejects such a query with HTTP 400. Active-user, DAU, WAU and MAU counts count the results of `SELECT DISTINCT VALUE c.user_id`. Uploads by workspace type subtract the `group` and `public` counts from the period total; any other or missing type is personal. Group and public-workspace status counts tally a `SELECT VALUE c.status` projection. Insights stream two narrow projections, the filtered token records and the activity window, and aggregate them in the application. Each login heatmap cell totals one UTC weekday and hour across the whole period, and rankings with equal totals are ordered by ID. See [V2 Control Center Cosmos Query Compatibility Fix](../fixes/V2_CONTROL_CENTER_COSMOS_QUERY_COMPATIBILITY_FIX.md). + The date presets are 7, 30, and 90 UTC calendar days. Custom ranges use `start_date` and `end_date` in `YYYY-MM-DD` format and are limited to 366 days. The trend charts reuse the existing activity-trends and token-filter APIs, CSV export, and “Chat with these trends” endpoint. Chart data is grouped from the fields written by `functions_activity_logging.py`: `user_login.timestamp`, creation activity types and `workspace_type`, and `token_usage.usage.model`, `usage.total_tokens`, `token_type`, `user_id`, and `workspace_context` IDs. The login heatmap uses UTC and Monday=0. Invalid dashboard date ranges return a generic validation error rather than exposing exception details. @@ -49,13 +51,11 @@ The Users section supports server-side search by email or display name, access a `GET /api/v2/control-center/users` returns cached login, conversation, document, and token metrics with their calculation timestamps. Its response includes the oldest and newest metric timestamps and the count of users on the current page without a cached metric snapshot so administrators can judge freshness. `GET /api/v2/control-center/users/` returns the profile and current access/upload restrictions, usage summary, the most recent activity records, and group/public-workspace memberships and ownership. -Administrators can change access or upload restrictions for one user, or select explicit users and users matching the current filters across pages. Filter-based bulk selection supports exclusions and is capped at 500 accounts. Bulk changes use the existing user settings update path so established activity and audit behavior remains in effect. Deleting a user's documents creates an approval request; it does not directly delete the documents. - -`GET /api/v2/control-center/users/export.csv` exports all users matching the current filters. CSV cells beginning with `=`, `+`, `-`, or `@` are prefixed to prevent spreadsheet formula execution. The Activity tab's link carries `user_id` into the Activity Logs section. +Each Users query orders a single property, because a two-property `ORDER BY` needs a composite index that `user_settings` does not have. Users with a recorded value for the sort column come first, in the chosen direction. Users without one follow in ID order, so missing values are listed last in either direction, and a page can continue from one population into the other. Equal recorded values have no guaranteed secondary order. The automatic index serves both queries, so existing deployments need no index change. -## Data health +Administrators can change access or upload restrictions for one user, or select explicit users and users matching the current filters across pages. Filter-based bulk selection supports exclusions and is capped at 500 accounts. Bulk changes use the existing user settings update path so established activity and audit behavior remains in effect. Deleting a user's documents creates an approval request; it does not directly delete the documents. -Data health is available to users with `can_run_maintenance`. Its Check button calls `GET /api/admin/control-center/migrate/status` only when requested. Run backfill requires confirmation and calls `POST /api/admin/control-center/migrate/all`. The legacy-flag counts do not prove that activity logs are missing: normal application writers already record activity, so the backfill is normally unnecessary. The backfill checks for a matching resource creation record in the user's activity-log partition and uses stable per-resource IDs to avoid duplicate writes on repeated or concurrent runs. +`GET /api/v2/control-center/users/export.csv` exports all users matching the current filters, in the list's order. It runs its first query before streaming, so a storage failure returns an error rather than a header-only file. CSV cells beginning with `=`, `+`, `-`, or `@` are prefixed to prevent spreadsheet formula execution. The Activity tab's link carries `user_id` into the Activity Logs section. ## Groups @@ -80,9 +80,9 @@ The Groups section helps administrators find shared workspaces that need attenti | `page`, `per_page` | Server paging, default 25 and maximum 250 rows | | `force_refresh=1` | Rebuild the server inventory instead of using its 90-second cache | -`functions_control_center_groups.py` builds an inventory with four batched Cosmos queries: projected groups, document metadata counts, all-time token totals, and latest activity timestamps. Activity includes the top-level `group_id`, nested `group.group_id`, and `workspace_context.group_id` writer shapes. Filtering, sorting and paging occur on the server after aggregation; neither the browser nor per-row enrichment performs filtering or counting. This deliberately avoids N+1 queries and Cosmos ordering on undefined/computed fields. Tied sort values use stable ID ordering and missing activity sorts last in either direction. Failed aggregates fail the request rather than silently reporting zero. +`functions_control_center_groups.py` builds an inventory with four batched Cosmos queries. One projects the groups. The other three stream narrow projections that the application aggregates into document counts, all-time token totals and latest activity timestamps: document metadata group IDs, group token records, and a coalesced group ID with timestamp. The Python Cosmos SDK cannot run cross-partition `GROUP BY`. Activity includes the top-level `group_id`, nested `group.group_id`, and `workspace_context.group_id` writer shapes. Filtering, sorting and paging occur on the server after aggregation; neither the browser nor per-row enrichment performs filtering or counting. This deliberately avoids N+1 queries and Cosmos ordering on undefined/computed fields. Tied sort values use stable ID ordering and missing activity sorts last in either direction. Failed aggregates fail the request rather than silently reporting zero. -The response includes `groups`, `pagination`, and `metrics_freshness` with the snapshot's `calculated_at`, source and TTL. List/CSV totals can lag external writes by up to 90 seconds; **Refresh groups** bypasses the cache. Memory and rebuild cost scale with the group inventory and activity aggregates; this is not continuation-token pagination. Legacy storage-size estimates retain their own older refresh timestamp and are not represented as current counts. +The response includes `groups`, `pagination`, and `metrics_freshness` with the snapshot's `calculated_at`, source and TTL. List/CSV totals can lag external writes by up to 90 seconds; **Refresh groups** bypasses the cache. Rebuild cost scales with the number of group documents and all-time group token and activity records, while memory holds only per-group totals. This is not continuation-token pagination. Legacy storage-size estimates retain their own older refresh timestamp and are not represented as current counts. `POST /api/v2/control-center/groups/bulk-status` accepts either `group_ids` or a `filter` object, plus `status` and `reason`. Filter selection accepts the same filter fields, supports `exclude_ids`, and resolves against a fresh server inventory. The full population is capped at 500 before any writes occur. Locked/inactive status requires a nonblank reason (maximum 2,000 characters). `PUT /api/v2/control-center/groups//status` has the same reason rule. Both call the classic single-status writer, preserving etag conflict handling, status history, activity logging and App Insights audit events. Partial results report `success_count`, `failed_count`, and individual `failed_groups`; unchanged groups succeed without duplicate audit records. The classic status route keeps its existing optional-reason behavior. @@ -172,12 +172,12 @@ An existing executor limitation remains: document deletion catches individual do ## Usage -Open **Account → Control Center**, then select a section in its internal rail. A bookmarked section URL opens that section directly. On the Dashboard, select a date range and optional token filters; charts include accessible data tables, and chart selections link to the corresponding filtered activity view. Export downloads the trend data as CSV. “Chat with these trends” creates a conversation containing the selected trend data. The Data health page is intended for explicit diagnosis or a known recovery scenario; check first, and run the backfill only when the result and operational context justify it. +Open **Account → Control Center**, then select a section in its internal rail. A bookmarked section URL opens that section directly; an unknown section, such as a bookmark to the removed Data health page, opens the Dashboard. On the Dashboard, select a date range and optional token filters; charts include accessible data tables, and chart selections link to the corresponding filtered activity view. Export downloads the trend data as CSV. “Chat with these trends” creates a conversation containing the selected trend data. ## Testing and limitations -Functional checks cover dashboard status normalization, period deltas, cache expiry and refresh, dashboard-reader access, capability parity, bootstrap exposure, manual-only migration checks, and route wiring. Playwright coverage verifies the Data health interaction and capability visibility. Top activity and token rankings are limited to entities present in the recorded `user_id` and workspace-context fields; unlogged historical status snapshots and model/provider details are not inferred. +Functional checks cover dashboard status normalization, period deltas, cache expiry and refresh, dashboard-reader access, capability parity, bootstrap exposure, and route wiring. Route-level tests run the real Dashboard and Users handlers, and the Groups inventory, against fake containers that reject query shapes the Python Cosmos SDK cannot run. `functional_tests/test_v2_control_center_cosmos_query_compatibility.py` scans every V2 Control Center SQL string for cross-partition `GROUP BY`, `COUNT` over `DISTINCT` values, and multi-property `ORDER BY` without a declared composite index. Playwright coverage verifies capability-based section visibility and that the removed Data health section stays absent, including for its old URL. Top activity and token rankings are limited to entities present in the recorded `user_id` and workspace-context fields; unlogged historical status snapshots and model/provider details are not inferred. ## Version tracking -The application version is defined by `VERSION` in `application/single_app/config.py`. The foundation was added in **0.261.278**, the dashboard in **0.261.279**, user management in **0.261.280**, group management in **0.261.282**, public workspace management in **0.261.283**, and Activity Logs in **0.261.284**. +The application version is defined by `VERSION` in `application/single_app/config.py`. The foundation was added in **0.261.278**, the dashboard in **0.261.279**, user management in **0.261.280**, group management in **0.261.282**, public workspace management in **0.261.283**, and Activity Logs in **0.261.284**. In **0.261.290**, the Dashboard, Users and Groups queries were made compatible with the Python Cosmos SDK. The same release removed the Data health section and its `GET /api/admin/control-center/migrate/status` and `POST /api/admin/control-center/migrate/all` backfill APIs; the classic Control Center had already stopped using them. diff --git a/docs/explanation/fixes/ACTIVITY_LOG_MIGRATION_PROMPT_FIX.md b/docs/explanation/fixes/ACTIVITY_LOG_MIGRATION_PROMPT_FIX.md index 5261719ea..dfb49f8d6 100644 --- a/docs/explanation/fixes/ACTIVITY_LOG_MIGRATION_PROMPT_FIX.md +++ b/docs/explanation/fixes/ACTIVITY_LOG_MIGRATION_PROMPT_FIX.md @@ -24,3 +24,7 @@ Implemented in version **0.261.278** (`application/single_app/config.py`): ## Impact Opening the classic Control Center no longer triggers four cross-partition count scans or a misleading migration prompt. Administrators retain an explicit diagnostic and recovery path in V2. Existing logs are preserved; the manual backfill skips records that already have matching creation events. + +## Update in 0.261.290 + +The V2 Data health section was removed, and with it the only callers of `GET /api/admin/control-center/migrate/status` and `POST /api/admin/control-center/migrate/all`. Both APIs and the backfill-only `has_activity_log_for_resource` helper were deleted, so neither Control Center offers the backfill any longer. Normal application workflows continue to write activity records, and `build_activity_log_id` still gives idempotent writers stable record IDs. `ui_tests/test_v2_control_center_data_health.py` was replaced by `ui_tests/test_v2_control_center_data_health_removed.py`, and `functional_tests/test_v2_control_center_foundation.py` now checks that the section and APIs stay removed. See [V2 Control Center Cosmos Query Compatibility Fix](V2_CONTROL_CENTER_COSMOS_QUERY_COMPATIBILITY_FIX.md). diff --git a/docs/explanation/fixes/V2_CONTROL_CENTER_COSMOS_QUERY_COMPATIBILITY_FIX.md b/docs/explanation/fixes/V2_CONTROL_CENTER_COSMOS_QUERY_COMPATIBILITY_FIX.md new file mode 100644 index 000000000..689034a6d --- /dev/null +++ b/docs/explanation/fixes/V2_CONTROL_CENTER_COSMOS_QUERY_COMPATIBILITY_FIX.md @@ -0,0 +1,70 @@ +# V2 Control Center Cosmos Query Compatibility Fix + +**Fixed in version:** 0.261.290 + +## Issue + +In the V2 Control Center, the Dashboard showed "Failed to retrieve dashboard insights." Users showed "Unable to retrieve users." Groups showed "Unable to retrieve groups." Every call to these endpoints returned HTTP 500: + +- `GET /api/v2/control-center/dashboard/summary` +- `GET /api/v2/control-center/dashboard/insights` +- `GET /api/v2/control-center/users` (and `users/export.csv`) +- `GET /api/v2/control-center/groups` (and group detail, `groups/export.csv` and filter-based bulk status) + +Public Workspaces and Activity Logs loaded normally. + +## Root cause + +For every failed request, Application Insights recorded the route's `[CONTROL_CENTER] ... failed` event with `error_type=CosmosHttpResponseError`. Cosmos DB answered those queries with HTTP 400 because they used three shapes that the azure-cosmos Python SDK cannot run across partitions. + +1. **Cross-partition `GROUP BY`.** Before it runs a cross-partition query, the Python SDK asks the gateway for a query plan and lists the query features it can execute: Aggregate, CompositeAggregate, Distinct, MultipleOrderBy, OffsetAndLimit, OrderBy, Top and a few others. `GroupBy` is not on that list, even in the newest SDK, so the gateway rejects any query that needs it. Upgrading the SDK does not help. The request traces for the groups list show that the query-plan request itself returned 400. These queries used it: + - Dashboard summary: group and public-workspace status counts, document uploads by workspace type. + - Dashboard insights: tokens by day and model, top token consumers, top activity, login heatmap. + - Groups inventory (`load_group_inventory`): document counts, all-time token totals, latest activity. +2. **`COUNT` over a `DISTINCT` subquery.** `SELECT VALUE COUNT(1) FROM (SELECT DISTINCT c.user_id ...)` needs the `DCount` feature, which the Python SDK also does not list. The dashboard active-user, DAU, WAU and MAU counts used it. +3. **Two-property `ORDER BY` without a composite index.** The Users list and export ordered by `, c.id`. A multi-property `ORDER BY` needs a matching composite index, and the `user_settings` container has none. Expected composite indexes are applied only when an administrator enables App Maintenance indexing. The query plan succeeded, but the partition rejected the query. + +The functional tests did not catch any of these. Their fake containers accepted any SQL, and some tests asserted that the `GROUP BY` text was present. + +Activity Logs failed twice at the same time for a different, already documented reason: it needs the `activity_logs` composite index from App Maintenance. Once a manual maintenance run applied that index, Activity Logs worked, so it needs no change here. + +## Resolution + +The response contracts are unchanged. Each unsupported query is replaced with a shape that already works in production in this application. + +| Area | Before | After | +|---|---|---| +| Active users, DAU, WAU, MAU | `COUNT(1)` over a `DISTINCT` subquery | `SELECT DISTINCT VALUE c.user_id`, counted in Python | +| Document uploads by workspace type | `GROUP BY c.workspace_type` | Three `SELECT VALUE COUNT(1)` queries: all, `group` and `public`. Personal is the remainder, matching the previous rule that any other or missing type counts as personal | +| Group and public-workspace status counts | `GROUP BY c.status` | `SELECT VALUE c.status`, tallied in Python and normalized by the existing `_dashboard_status_counts` | +| Dashboard insights | Six `GROUP BY` queries | Two streamed projections aggregated in Python: one over the filtered token records and one over the activity window | +| Groups inventory | Three `GROUP BY` aggregates | Three streamed projections (document group IDs, group token records, coalesced group ID and timestamp) aggregated in Python. The inventory still uses four batched queries and the 90-second snapshot | +| Users list and export | `ORDER BY , c.id` | Users with a recorded sort value are ordered by that single property. Users without one follow, ordered by `c.id`. This is the same approach as Public Workspaces, and the automatic index serves both queries | + +### Behavior notes + +- **Login heatmap:** each cell now totals every login in the period for that UTC weekday and hour. Before, the API returned one cell per date and hour, and the UI's lookup showed only the first matching date, so a 30-day range undercounted repeated weekdays. The response shape is unchanged. +- **Ranking ties:** top token consumers and top activity rankings are ordered by ID when totals tie, so repeated loads list them the same way. +- **Users ordering:** accounts with no recorded value for the sort column are listed last in either direction, in ID order. As in Public Workspaces, equal recorded values have no guaranteed secondary order. +- **Users export:** the export runs its first query before streaming starts. A storage failure now returns the JSON error instead of a CSV that contains only the header row. +- **Token values:** dashboard token totals add numeric values only. A non-numeric value counts as zero instead of making the whole aggregate undefined. +- **Cost:** in-application aggregation reads the same documents the `GROUP BY` queries would have read, returning narrow projections. Dashboard responses keep their 90-second cache, and the groups inventory keeps its 90-second snapshot. + +## Files modified + +- `application/single_app/route_backend_control_center.py`: dashboard summary and insights helpers, and Users paging and export helpers. +- `application/single_app/functions_control_center_groups.py`: `load_group_inventory`. +- `application/single_app/config.py`: version 0.261.290. +- `functional_tests/test_support/cosmos_query_guard.py` (new): rejects `GROUP BY`, `COUNT` over `DISTINCT` values, and multi-property `ORDER BY` without a declared composite index. +- `functional_tests/test_v2_control_center_cosmos_query_compatibility.py` (new): scans every SQL string in the V2 Control Center code paths through the guard. The activity-log `ORDER BY` passes only because `config.py` declares its composite index. +- `functional_tests/test_v2_control_center_dashboard.py`, `functional_tests/test_v2_control_center_users.py` and `functional_tests/test_v2_control_center_groups.py`: the fakes now pass every query through the guard. The tests run the real routes and aggregation helpers. + +## Validation + +- `python -m pytest functional_tests/test_v2_control_center_dashboard.py functional_tests/test_v2_control_center_users.py functional_tests/test_v2_control_center_groups.py functional_tests/test_v2_control_center_cosmos_query_compatibility.py`: all pass. +- Each new test fails on the code before this fix. On the previous route module, the compatibility scan reports 13 unsupported query sites: the Users list and export, dashboard summary and dashboard insights. The dashboard and Users route tests receive HTTP 500, and the groups aggregation test rejects the `GROUP BY` document count. +- Route-level dashboard and Users tests drive the real Flask handlers with fake containers that reject unsupported queries. The Users test checks a page that ends the recorded population and continues into the missing one, both sort directions, page clamping and export order. + +## Related changes in this release + +The V2 Data health section was removed in the same release, together with the activity-log backfill APIs that only it used: `GET /api/admin/control-center/migrate/status` and `POST /api/admin/control-center/migrate/all`. See [V2 Control Center](../features/V2_CONTROL_CENTER.md) and [Activity Log Migration Prompt Fix](ACTIVITY_LOG_MIGRATION_PROMPT_FIX.md). diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 7e69e1ecc..6040db169 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,24 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.290)** + +#### Bug Fixes + +* **V2 Control Center Dashboard, Users and Groups Load Again** + * The Dashboard showed "Failed to retrieve dashboard insights.", and Users and Groups showed "Unable to retrieve users." and "Unable to retrieve groups." Cosmos DB rejected their queries with HTTP 400. The azure-cosmos Python SDK cannot run cross-partition `GROUP BY` or `COUNT` over `DISTINCT` values, and the Users sort used a two-property `ORDER BY` that needs a composite index the `user_settings` container does not have. + * The affected queries now use shapes the SDK supports. Distinct values and counts are aggregated in the application, the Groups inventory streams narrow projections, and Users orders by one property at a time, listing accounts without a recorded sort value last. Existing deployments need no index change or App Maintenance step. + * The login heatmap now totals each UTC weekday and hour across the whole period instead of showing a single date's count. The Users CSV export now reports a storage failure as an error instead of returning a file with only the header row. + * A compatibility test scans every V2 Control Center query so these shapes cannot return. + * (Ref: `route_backend_control_center.py`, `functions_control_center_groups.py`, `test_v2_control_center_cosmos_query_compatibility.py`, [V2 Control Center Cosmos Query Compatibility Fix](fixes/V2_CONTROL_CENTER_COSMOS_QUERY_COMPATIBILITY_FIX.md)) + +#### Breaking Changes + +* **V2 Control Center Data Health Removed** + * The Data health section has been removed, together with the activity-log backfill APIs that only it used: `GET /api/admin/control-center/migrate/status` and `POST /api/admin/control-center/migrate/all`. The classic Control Center had already stopped calling them, and normal application workflows record activity themselves. + * **Migration**: None required. A bookmark to the old Data health page now opens the Dashboard. + * (Ref: `ControlCenterPage.tsx`, `route_backend_control_center.py`, `functions_activity_logging.py`, [Activity Log Migration Prompt Fix](fixes/ACTIVITY_LOG_MIGRATION_PROMPT_FIX.md)) + ### **(v0.261.288)** #### New Features diff --git a/docs/guides/v2-control-center.md b/docs/guides/v2-control-center.md index 4990272fe..6838fa062 100644 --- a/docs/guides/v2-control-center.md +++ b/docs/guides/v2-control-center.md @@ -1,7 +1,7 @@ --- layout: page title: "Use the V2 Control Center" -description: "Review usage trends in the V2 Control Center and check activity-log data only when needed." +description: "Review usage trends and manage users, groups, public workspaces and activity in the V2 Control Center." section: "Guides" audience: admin --- @@ -14,7 +14,7 @@ The section rail is filtered to your permissions. The Dashboard is available to ## Manage users -Open **Users** to find accounts by email or display name, filter by access, upload permission, recent login, or document ownership, and sort usage columns. Filters and sorting are reflected in the URL, so a filtered view can be bookmarked or opened from a Dashboard drill-through. The list reports when its cached usage metrics were calculated and how many accounts on the current page have not yet received a metrics refresh. +Open **Users** to find accounts by email or display name, filter by access, upload permission, recent login, or document ownership, and sort usage columns. Accounts without a recorded value for the sort column, such as users whose metrics have not been refreshed yet, are listed after the others in either direction. Filters and sorting are reflected in the URL, so a filtered view can be bookmarked or opened from a Dashboard drill-through. The list reports when its cached usage metrics were calculated and how many accounts on the current page have not yet received a metrics refresh. Select rows to allow or deny access or uploads. Selection can include all users matching the current filters across pages, with an option to exclude individual accounts. Restrictions may have an optional expiry. Open a user to review the account, recent activity, group and public-workspace memberships, and ownership. Changes reconcile against the server and report failures rather than leaving the list in an optimistic-only state. @@ -26,7 +26,7 @@ Choose a 7-, 30-, or 90-day period, or set an inclusive custom UTC date range of The charts use recorded logins, conversation creation, document creation by workspace type, token usage type, token models, and workspace activity. Select chart points or KPI cards to open the related section with query filters. Use **Export** to download trend data as CSV, or **Chat with these trends** to start a conversation containing the selected trend data. Token filters apply to token totals and token charts; they do not change login, conversation, or upload counts. -The login heatmap reports UTC hours with Monday as weekday zero. Charts include data tables for screen-reader and text-based access. Dashboard summaries are cached for 90 seconds; choose **Refresh** to bypass the cache. +The login heatmap totals the logins for each UTC weekday and hour across the selected period, with Monday as weekday zero, so it shows recurring busy times rather than a single day. Charts include data tables for screen-reader and text-based access. Dashboard summaries are cached for 90 seconds; choose **Refresh** to bypass the cache. ## Manage groups @@ -68,19 +68,9 @@ Individual document deletion, workspace deletion, take-ownership and transfer-to Activity exports contain only the 20 recent projected records displayed in the drawer. Activity Logs links open the broader investigation with public workspace scope. -## Check activity-log data health - -The **Data health** section is available to users with maintenance access. Its backfill is a legacy repair operation for conversation and document creation events; ordinary application workflows already write activity logs, so the backfill is normally unnecessary. - -1. Select **Data health** and choose **Check activity-log status** to request the current legacy-flag counts. The check is not performed automatically when the page opens. -2. Read the results carefully: they count records missing a legacy flag and do not establish that their activity events are absent. -3. Use **Run backfill** only when you have a known need. Confirm the operation in the dialog. The backfill checks the relevant user's activity-log partition and skips existing creation records. - -The status and backfill APIs remain protected by the Control Center maintenance permission. Running a backfill may take time on large datasets. - ## Permission model -The V2 pane uses the same server-side role and setting rules as the existing Control Center endpoints. A dashboard reader can see only the dashboard section when the dashboard-reader role setting is enabled. Management, activity-log, and maintenance sections require full Control Center access. Hiding a section in the browser is not an authorization boundary; the APIs enforce their own access checks. +The V2 pane uses the same server-side role and setting rules as the existing Control Center endpoints. A dashboard reader can see only the dashboard section when the dashboard-reader role setting is enabled. Management and activity-log sections require full Control Center access. Hiding a section in the browser is not an authorization boundary; the APIs enforce their own access checks. ## Related diff --git a/functional_tests/test_support/cosmos_query_guard.py b/functional_tests/test_support/cosmos_query_guard.py new file mode 100644 index 000000000..f8091bc43 --- /dev/null +++ b/functional_tests/test_support/cosmos_query_guard.py @@ -0,0 +1,89 @@ +# cosmos_query_guard.py +"""Reject Cosmos SQL shapes that the azure-cosmos Python SDK cannot run across partitions. + +The SDK asks the gateway for a query plan and lists the features it can execute +(Aggregate, CompositeAggregate, Distinct, MultipleOrderBy, OffsetAndLimit, OrderBy, Top +and a few others). GroupBy and DCount are not on that list, so a cross-partition query +that needs them fails with HTTP 400 before it runs. An ORDER BY over more than one +property also fails with HTTP 400 unless the container declares a matching composite +index. Fake containers accept any SQL, so tests run their queries through this guard. +""" + +import re + + +GROUP_BY_PATTERN = re.compile(r"\bGROUP\s+BY\b", re.IGNORECASE) +DISTINCT_COUNT_PATTERN = re.compile( + r"\bCOUNT\s*\(\s*DISTINCT\b|\bCOUNT\s*\([^()]*\)\s*FROM\s*\(\s*SELECT\s+DISTINCT\b", + re.IGNORECASE, +) +ORDER_BY_PATTERN = re.compile( + r"\bORDER\s+BY\b(?P.*?)(?=\bOFFSET\b|\bLIMIT\b|\)|$)", + re.IGNORECASE | re.DOTALL, +) +ORDER_ITEM_PATTERN = re.compile(r"^(?P.+?)(?:\s+(?PASC|DESC))?$", re.IGNORECASE | re.DOTALL) +PROPERTY_PATH_PATTERN = re.compile(r"^c((?:\.[A-Za-z_][A-Za-z0-9_]*)+)$") + + +def _split_order_items(items): + parts, depth, current = [], 0, [] + for character in items: + if character in "([{": + depth += 1 + elif character in ")]}": + depth -= 1 + if character == "," and depth == 0: + parts.append("".join(current).strip()) + current = [] + else: + current.append(character) + parts.append("".join(current).strip()) + return [part for part in parts if part] + + +def _order_item(item): + match = ORDER_ITEM_PATTERN.match(item.strip()) + expression = match.group("expression").strip() + direction = "descending" if (match.group("direction") or "").upper() == "DESC" else "ascending" + path_match = PROPERTY_PATH_PATTERN.match(expression) + path = path_match.group(1).replace(".", "/") if path_match else None + return path, direction + + +def _served_by_composite_index(order_items, composite_indexes): + """A composite index serves its exact path order in its own or fully reversed directions.""" + requested = [_order_item(item) for item in order_items] + if any(path is None for path, _ in requested): + return False + for index in composite_indexes: + declared = [(entry[0], entry[1]) for entry in index] + if [path for path, _ in declared] != [path for path, _ in requested]: + continue + same = all(want == have for (_, want), (_, have) in zip(requested, declared)) + reversed_ = all(want != have for (_, want), (_, have) in zip(requested, declared)) + if same or reversed_: + return True + return False + + +def cosmos_query_problems(query, composite_indexes=()): + """Return why a cross-partition query would be rejected, or an empty list.""" + problems = [] + if GROUP_BY_PATTERN.search(query): + problems.append("cross-partition GROUP BY is not supported by the Python SDK") + if DISTINCT_COUNT_PATTERN.search(query): + problems.append("COUNT over DISTINCT values (DCount) is not supported by the Python SDK") + for match in ORDER_BY_PATTERN.finditer(query): + items = _split_order_items(match.group("items")) + if len(items) > 1 and not _served_by_composite_index(items, composite_indexes): + problems.append( + f"ORDER BY on {len(items)} properties needs a declared composite index: {', '.join(items)}" + ) + return problems + + +def assert_cosmos_query_supported(query, composite_indexes=()): + """Fail the calling test when a query shape cannot run through the Python SDK.""" + problems = cosmos_query_problems(query, composite_indexes) + if problems: + raise AssertionError(f"Unsupported Cosmos query ({'; '.join(problems)}): {' '.join(query.split())}") diff --git a/functional_tests/test_v2_control_center_cosmos_query_compatibility.py b/functional_tests/test_v2_control_center_cosmos_query_compatibility.py new file mode 100644 index 000000000..1834037bc --- /dev/null +++ b/functional_tests/test_v2_control_center_cosmos_query_compatibility.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +# test_v2_control_center_cosmos_query_compatibility.py +""" +Functional test for V2 Control Center Cosmos query compatibility. +Version: 0.261.290 +Implemented in: 0.261.290 + +The Dashboard, Users and Groups sections returned HTTP 500 because Cosmos DB rejected +their queries with HTTP 400. The azure-cosmos Python SDK cannot run cross-partition +GROUP BY or COUNT over DISTINCT values, and a two-property ORDER BY needs a composite +index that user_settings does not have. This test scans every SQL string in the V2 +Control Center code paths and fails if one of those query shapes returns. +""" + +import ast +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +APP = ROOT / "application" / "single_app" +ROUTE = APP / "route_backend_control_center.py" +CONFIG = APP / "config.py" +sys.path.insert(0, str(ROOT / "functional_tests")) + +from test_support.cosmos_query_guard import assert_cosmos_query_supported, cosmos_query_problems +from test_support.versioning import assert_app_version_at_least + + +SQL_PATTERN = re.compile(r"\b(SELECT|ORDER\s+BY|GROUP\s+BY)\b", re.IGNORECASE) +V2_ROUTE_PREFIXES = ("api_v2_control_center_", "_dashboard_", "_control_center_") +SHARED_ROUTE_HELPERS = {"build_token_usage_query_context", "append_token_usage_filters"} + + +def _activity_log_composite_indexes(): + """Read the composite index that config.py declares for the activity_logs container.""" + tree = ast.parse(CONFIG.read_text(encoding="utf-8")) + for node in tree.body: + if (isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == "ACTIVITY_LOGS_INDEXING_POLICY" + for target in node.targets)): + policy = ast.literal_eval(node.value) + return [ + [(entry["path"], entry["order"]) for entry in index] + for index in policy["compositeIndexes"] + ] + raise AssertionError("ACTIVITY_LOGS_INDEXING_POLICY is missing from config.py.") + + +def _docstring_ids(tree): + """Return the AST node IDs of docstrings, which describe queries rather than run them.""" + ids = set() + for node in ast.walk(tree): + if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + first = node.body[0] if node.body else None + if isinstance(first, ast.Expr) and isinstance(first.value, ast.Constant) and isinstance(first.value.value, str): + ids.add(id(first.value)) + return ids + + +def _sql_strings(node, skipped_ids): + """Yield SQL-bearing string literals and f-strings, with f-string values as placeholders.""" + for child in ast.walk(node): + if id(child) in skipped_ids: + continue + if isinstance(child, ast.Constant) and isinstance(child.value, str): + text = child.value + elif isinstance(child, ast.JoinedStr): + text = "".join( + part.value if isinstance(part, ast.Constant) else "{expr}" for part in child.values + ) + else: + continue + if SQL_PATTERN.search(text): + yield child.lineno, text + + +def _scanned_sql(): + """Return (location, sql, composite indexes) for every V2 Control Center query string.""" + found = [] + route_tree = ast.parse(ROUTE.read_text(encoding="utf-8")) + route_docstrings = _docstring_ids(route_tree) + for node in ast.walk(route_tree): + if isinstance(node, ast.FunctionDef) and ( + node.name.startswith(V2_ROUTE_PREFIXES) or node.name in SHARED_ROUTE_HELPERS + ): + found.extend( + (f"{ROUTE.name}:{line} ({node.name})", text, ()) + for line, text in _sql_strings(node, route_docstrings) + ) + for module, indexes in ( + (APP / "functions_control_center_groups.py", ()), + (APP / "functions_control_center_public_workspaces.py", ()), + (APP / "functions_control_center_activity.py", _activity_log_composite_indexes()), + ): + tree = ast.parse(module.read_text(encoding="utf-8")) + found.extend( + (f"{module.name}:{line}", text, indexes) + for line, text in _sql_strings(tree, _docstring_ids(tree)) + ) + return found + + +def test_guard_rejects_query_shapes_the_python_sdk_cannot_run(): + for query in ( + "SELECT c.group_id, COUNT(1) AS total FROM c WHERE c.type = 'document_metadata' GROUP BY c.group_id", + "SELECT VALUE COUNT(1) FROM ( SELECT DISTINCT c.user_id FROM c WHERE c.activity_type = 'user_login' )", + "SELECT VALUE COUNT(DISTINCT c.user_id) FROM c", + "SELECT c.id FROM c WHERE 1=1 ORDER BY c.display_name ASC, c.id ASC OFFSET @offset LIMIT @limit", + "SELECT c.id FROM c ORDER BY {expr} {expr}, c.id ASC", + ): + assert cosmos_query_problems(query), f"Guard accepted an unsupported query: {query}" + + +def test_guard_allows_supported_query_shapes(): + activity_indexes = _activity_log_composite_indexes() + for query in ( + "SELECT DISTINCT VALUE c.user_id FROM c WHERE c.activity_type = 'user_login'", + "SELECT VALUE COUNT(1) FROM c WHERE c.workspace_type = @workspace_type", + "SELECT VALUE c.status FROM c WHERE IS_DEFINED(c.status)", + "SELECT c.id FROM c WHERE (1=1) AND (IS_DEFINED(c.email)) ORDER BY c.email DESC OFFSET @offset LIMIT @limit", + "SELECT TOP 20 c.id FROM c WHERE c.group_id = @group_id ORDER BY c.timestamp DESC", + ): + assert_cosmos_query_supported(query) + assert_cosmos_query_supported( + "SELECT * FROM c ORDER BY c.timestamp DESC, c.id DESC, c.user_id DESC", activity_indexes, + ) + assert cosmos_query_problems("SELECT * FROM c ORDER BY c.timestamp DESC, c.id DESC, c.user_id DESC") + + +def test_every_v2_control_center_query_is_supported_by_the_python_sdk(): + scanned = _scanned_sql() + assert len(scanned) >= 25, f"Expected to scan the V2 Control Center queries, found {len(scanned)}." + failures = [ + f"{location}: {'; '.join(problems)}" + for location, text, indexes in scanned + for problems in [cosmos_query_problems(text, indexes)] + if problems + ] + assert not failures, "Unsupported Cosmos query shapes:\n" + "\n".join(failures) + + +def test_scan_covers_the_previously_failing_sections(): + locations = " ".join(location for location, _, _ in _scanned_sql()) + for expected in ( + "_dashboard_count_active_users", "_dashboard_document_upload_counts", "_dashboard_status_rows", + "_dashboard_token_insights", "_dashboard_activity_insights", "_control_center_query_user_page", + "_control_center_iter_users", "functions_control_center_groups.py", + ): + assert expected in locations, f"The compatibility scan no longer reaches {expected}." + + +def test_version_is_at_least_the_implementation_version(): + assert_app_version_at_least("0.261.290") + + +TESTS = [ + test_guard_rejects_query_shapes_the_python_sdk_cannot_run, + test_guard_allows_supported_query_shapes, + test_every_v2_control_center_query_is_supported_by_the_python_sdk, + test_scan_covers_the_previously_failing_sections, + test_version_is_at_least_the_implementation_version, +] + + +if __name__ == "__main__": + for test in TESTS: + test() + print(f"PASS {test.__name__}") + print(f"{len(TESTS)}/{len(TESTS)} Cosmos query compatibility checks passed") diff --git a/functional_tests/test_v2_control_center_dashboard.py b/functional_tests/test_v2_control_center_dashboard.py index 8afe17c38..1764d42c2 100644 --- a/functional_tests/test_v2_control_center_dashboard.py +++ b/functional_tests/test_v2_control_center_dashboard.py @@ -2,18 +2,28 @@ # test_v2_control_center_dashboard.py """ Functional test for the V2 Control Center dashboard. -Version: 0.261.280 +Version: 0.261.290 Implemented in: 0.261.280 This test validates status aggregation, period comparisons, bounded cache behavior, dashboard-reader authorization, and the summary and insights route contracts. +Since 0.261.290 every dashboard query also passes the Cosmos query guard, because the +Python Cosmos SDK cannot run cross-partition GROUP BY or COUNT over DISTINCT values. """ import ast +import json +import logging import sys import time -from datetime import datetime, timedelta +from collections import Counter, defaultdict +from datetime import datetime, timedelta, timezone +from importlib.metadata import version as package_version from pathlib import Path +from unittest.mock import patch + +import werkzeug +from flask import Blueprint, Flask, jsonify, request ROOT = Path(__file__).resolve().parents[1] APP = ROOT / "application" / "single_app" @@ -21,9 +31,27 @@ AUTH = APP / "functions_authentication.py" sys.path.insert(0, str(ROOT / "functional_tests")) +from test_support.cosmos_query_guard import cosmos_query_problems from test_support.versioning import assert_app_version_at_least +DASHBOARD_HELPERS = { + "_dashboard_cache_get", "_dashboard_cache_set", "_dashboard_metric", "_dashboard_status_counts", + "_dashboard_parse_period", "_dashboard_query_count", "_dashboard_count_active_users", + "_dashboard_activity_count", "_dashboard_document_upload_counts", "_dashboard_status_rows", + "_dashboard_token_total", "_dashboard_document_failure_count", "_dashboard_document_failures", + "_dashboard_token_value", "_dashboard_ranked", "_dashboard_token_insights", + "_dashboard_activity_insights", "normalize_token_filter_value", "extract_token_filters", + "append_token_usage_filters", "build_token_usage_query_context", +} +DASHBOARD_CONSTANTS = { + "CONTROL_CENTER_DASHBOARD_CACHE_TTL_SECONDS", "CONTROL_CENTER_DASHBOARD_CACHE_MAX_ENTRIES", + "CONTROL_CENTER_DASHBOARD_RANKING_LIMIT", "CONTROL_CENTER_DASHBOARD_RANKED_FIELDS", + "DASHBOARD_INVALID_RANGE_ERROR", "_control_center_dashboard_cache", +} +DASHBOARD_ROUTES = {"api_v2_control_center_dashboard_summary", "api_v2_control_center_dashboard_insights"} + + def _function_namespace(path, names, initial=None): tree = ast.parse(path.read_text(encoding="utf-8")) wanted = [ @@ -36,6 +64,67 @@ def _function_namespace(path, names, initial=None): return namespace +def _dashboard_namespace(containers, extra=None): + """Execute the real dashboard helpers, constants and routes against fake containers.""" + tree = ast.parse(ROUTE.read_text(encoding="utf-8")) + constants = [ + node for node in tree.body + if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id in DASHBOARD_CONSTANTS for target in node.targets) + ] + helpers = [ + node for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name in DASHBOARD_HELPERS + ] + routes = [ + node for node in ast.walk(tree) + if isinstance(node, ast.FunctionDef) and node.name in DASHBOARD_ROUTES + ] + assert {node.name for node in helpers} == DASHBOARD_HELPERS + assert {node.name for node in routes} == DASHBOARD_ROUTES + blueprint = Blueprint("v2_dashboard_test", __name__) + events = [] + + def passthrough(*_args, **_kwargs): + return lambda function: function + + namespace = { + "Counter": Counter, "defaultdict": defaultdict, "datetime": datetime, "timedelta": timedelta, + "timezone": timezone, "json": json, "logging": logging, "time": time, + "jsonify": jsonify, "request": request, "bp": blueprint, + "swagger_route": passthrough, "get_auth_security": lambda: None, + "login_required": lambda function: function, "control_center_required": passthrough, + "log_event": lambda *args, **kwargs: events.append((args, kwargs)), + **containers, **(extra or {}), + } + exec(compile(ast.Module(body=constants + helpers + routes, type_ignores=[]), str(ROUTE), "exec"), namespace) + app = Flask("v2_dashboard_test") + app.config.update(TESTING=True) + app.register_blueprint(blueprint) + # Flask 2.x test clients read werkzeug.__version__, which Werkzeug 3 no longer defines. + with patch.object(werkzeug, "__version__", package_version("werkzeug"), create=True): + namespace["client"] = app.test_client() + namespace["events"] = events + return namespace + + +class GuardedContainer: + """Answer queries by shape after proving the Python SDK could run each one.""" + + def __init__(self, respond): + self.respond = respond + self.queries = [] + self.problems = [] + + def query_items(self, query, parameters=None, **kwargs): + self.queries.append({"query": query, "parameters": parameters or [], **kwargs}) + problems = cosmos_query_problems(query) + if problems: + self.problems.extend(problems) + raise RuntimeError("Cosmos rejected the query shape.") + return iter(self.respond(query, {item["name"]: item["value"] for item in parameters or []})) + + def test_group_and_workspace_status_counts_use_real_statuses(): functions = _function_namespace(ROUTE, {"_dashboard_status_counts"}) counts = functions["_dashboard_status_counts"]( @@ -73,30 +162,253 @@ def test_group_and_workspace_status_counts_use_real_statuses(): } -def test_document_uploads_are_aggregated_by_recorded_workspace_type(): - class ActivityContainer: - def query_items(self, **kwargs): - self.query = kwargs - return iter([ - {"workspace_type": "personal", "count": 3}, - {"workspace_type": "group", "count": 2}, - {"workspace_type": "public", "count": 1}, - {"workspace_type": None, "count": 1}, - {"workspace_type": "legacy", "count": 1}, - ]) +def test_status_rows_are_tallied_without_group_by(): + container = GuardedContainer(lambda query, values: ["locked", "locked", None, " Inactive ", 7, "active"]) + functions = _function_namespace( + ROUTE, + {"_dashboard_status_rows", "_dashboard_status_counts"}, + {"Counter": Counter}, + ) + rows = functions["_dashboard_status_rows"](container) + assert container.problems == [] + assert "SELECT VALUE c.status" in container.queries[0]["query"] + assert container.queries[0]["enable_cross_partition_query"] is True + assert sorted(rows, key=lambda row: str(row["status"])) == sorted([ + {"status": "locked", "count": 2}, {"status": None, "count": 1}, + {"status": " Inactive ", "count": 1}, {"status": "7", "count": 1}, + {"status": "active", "count": 1}, + ], key=lambda row: str(row["status"])) + assert functions["_dashboard_status_counts"](9, rows) == { + "active": 6, "locked": 2, "upload_disabled": 0, "inactive": 1, + } + assert functions["_dashboard_status_counts"](9, rows, unknown_status="inactive") == { + "active": 5, "locked": 2, "upload_disabled": 0, "inactive": 2, + } + + +def test_document_uploads_are_counted_by_recorded_workspace_type(): + records = [{"workspace_type": value} for value in ( + "personal", "personal", "personal", "group", "group", "public", None, "legacy", + )] + [{}] + + def respond(query, values): + assert "c.activity_type = 'document_creation'" in query + assert {"@start_date", "@end_date"} <= set(values) + if "@workspace_type" in values: + return [sum(record.get("workspace_type") == values["@workspace_type"] for record in records)] + return [len(records)] - container = ActivityContainer() + container = GuardedContainer(respond) functions = _function_namespace( ROUTE, - {"_dashboard_document_upload_counts"}, + {"_dashboard_document_upload_counts", "_dashboard_query_count"}, + {"cosmos_activity_logs_container": container}, + ) + counts = functions["_dashboard_document_upload_counts"]( + datetime(2026, 9, 1), datetime(2026, 9, 7, 23, 59, 59), + ) + assert counts == {"personal": 6, "group": 2, "public": 1} + assert list(counts) == ["personal", "group", "public"] + assert container.problems == [] + assert all("SELECT VALUE COUNT(1)" in item["query"] for item in container.queries) + assert all(item["enable_cross_partition_query"] is True for item in container.queries) + + +def test_active_users_are_counted_from_distinct_values(): + container = GuardedContainer(lambda query, values: ["user-1", "user-2", "user-1", "user-3"]) + functions = _function_namespace( + ROUTE, + {"_dashboard_count_active_users"}, {"cosmos_activity_logs_container": container}, ) start = datetime(2026, 9, 1) - end = datetime(2026, 9, 7, 23, 59, 59) - counts = functions["_dashboard_document_upload_counts"](start, end) - assert counts == {"personal": 5, "group": 2, "public": 1} - assert "GROUP BY c.workspace_type" in container.query["query"] - assert container.query["enable_cross_partition_query"] is True + end = datetime(2026, 9, 30, 23, 59, 59) + assert functions["_dashboard_count_active_users"](start, end) == 3 + query = container.queries[0] + assert container.problems == [] + assert "SELECT DISTINCT VALUE c.user_id" in query["query"] + assert "c.activity_type = 'user_login'" in query["query"] + assert query["parameters"] == [ + {"name": "@start_date", "value": start.isoformat()}, + {"name": "@end_date", "value": end.isoformat()}, + ] + + +def test_token_insights_total_models_and_rank_consumers(): + rows = [ + {"timestamp": "2026-09-01T10:00:00", "model": "gpt-4o", "tokens": 100, + "user_id": "u1", "group_id": "g1"}, + {"timestamp": "2026-09-01T11:00:00", "model": "gpt-4o", "tokens": 50, + "user_id": "u2", "public_workspace_id": "p1"}, + {"timestamp": "2026-09-02T09:00:00", "model": None, "tokens": 10.5, "user_id": "u1"}, + {"timestamp": "2026-09-02T09:30:00", "tokens": 5, "user_id": "u3"}, + {"timestamp": "2026-09-03T09:00:00", "model": "gpt-4o", "tokens": "bad", "user_id": "u2"}, + {"model": "gpt-4o", "tokens": 7, "group_id": "g1"}, + {"timestamp": "2026-09-03T09:00:00", "model": "gpt-4o", "tokens": True, "user_id": ""}, + ] + container = GuardedContainer(lambda query, values: rows) + namespace = _dashboard_namespace({"cosmos_activity_logs_container": container}) + by_model, top_tokens = namespace["_dashboard_token_insights"]( + datetime(2026, 9, 1), datetime(2026, 9, 30, 23, 59, 59), {"model": "gpt-4o"}, + ) + assert container.problems == [] + query = container.queries[0] + assert "c.activity_type = 'token_usage'" in query["query"] + assert "c.usage.model = @token_model" in query["query"] + assert {"name": "@token_model", "value": "gpt-4o"} in query["parameters"] + assert by_model == [ + {"date": "2026-09-01", "model": "gpt-4o", "tokens": 150}, + {"date": "2026-09-02", "model": "Unknown model", "tokens": 10}, + {"date": "2026-09-03", "model": "gpt-4o", "tokens": 0}, + ] + assert top_tokens == { + "users": [{"id": "u1", "tokens": 110}, {"id": "u2", "tokens": 50}, {"id": "u3", "tokens": 5}], + "groups": [{"id": "g1", "tokens": 107}], + "public_workspaces": [{"id": "p1", "tokens": 50}], + } + + +def test_rankings_are_limited_and_ties_are_stable(): + namespace = _dashboard_namespace({}) + totals = {f"user-{index:02d}": 5 for index in range(12)} + totals["user-99"] = 9 + ranked = namespace["_dashboard_ranked"](totals, "tokens") + assert len(ranked) == namespace["CONTROL_CENTER_DASHBOARD_RANKING_LIMIT"] == 10 + assert ranked[0] == {"id": "user-99", "tokens": 9} + assert [item["id"] for item in ranked[1:]] == [f"user-{index:02d}" for index in range(9)] + + +def test_activity_insights_rank_actors_and_total_each_login_hour(): + rows = [ + {"timestamp": "2026-09-07T09:15:00", "activity_type": "user_login", "user_id": "u1"}, + {"timestamp": "2026-09-14T09:45:00", "activity_type": "user_login", "user_id": "u2"}, + {"timestamp": "2026-09-15T23:05:00", "activity_type": "user_login", "user_id": "u1"}, + {"timestamp": "2026-09-15", "activity_type": "user_login", "user_id": "u1"}, + {"timestamp": "2026-09-16T10:00:00", "activity_type": "token_usage", "user_id": "u1", + "group_id": "g1", "public_workspace_id": "p1"}, + {"timestamp": "2026-09-16T11:00:00", "activity_type": "document_creation", "user_id": "u3", + "group_id": "g1"}, + {"activity_type": "user_login", "user_id": "u4"}, + ] + container = GuardedContainer(lambda query, values: rows) + namespace = _dashboard_namespace({"cosmos_activity_logs_container": container}) + top_activity, cells = namespace["_dashboard_activity_insights"]( + datetime(2026, 9, 1), datetime(2026, 9, 30, 23, 59, 59), + ) + assert container.problems == [] + assert "c.timestamp >= @start_date" in container.queries[0]["query"] + assert top_activity == { + "users": [{"id": "u1", "activity_count": 4}, {"id": "u2", "activity_count": 1}, + {"id": "u3", "activity_count": 1}, {"id": "u4", "activity_count": 1}], + "groups": [{"id": "g1", "activity_count": 2}], + "public_workspaces": [{"id": "p1", "activity_count": 1}], + } + # 2026-09-07 and 2026-09-14 are both Mondays: one cell totals both logins. + assert cells == [ + {"weekday": 0, "hour": 9, "count": 2}, + {"weekday": 1, "hour": 23, "count": 1}, + ] + + +def _dashboard_store(): + statuses = {"groups": ["locked", None, "inactive"], "workspaces": ["locked", "legacy"]} + token_rows = [ + {"timestamp": "2026-09-20T10:00:00", "model": "gpt-4o", "tokens": 40, "user_id": "u1", "group_id": "g1"}, + ] + activity_rows = [ + {"timestamp": "2026-09-21T08:00:00", "activity_type": "user_login", "user_id": "u1"}, + {"timestamp": "2026-09-21T08:30:00", "activity_type": "token_usage", "user_id": "u1", "group_id": "g1"}, + ] + + def users(query, values): + if "c.settings.access.status = 'deny'" in query: + return [1] + return [4] + + def statuses_for(key): + def respond(query, values): + if "SELECT VALUE c.status" in query: + return statuses[key] + return [len(statuses[key]) + 2] + return respond + + def activity(query, values): + if "SELECT DISTINCT VALUE c.user_id" in query: + return ["u1", "u2"] + if "SELECT VALUE SUM(c.usage.total_tokens)" in query: + return [1234] + if "document_creation" in query and "SELECT VALUE COUNT(1)" in query: + return [{"group": 2, "public": 1}.get(values.get("@workspace_type"), 5)] + if "SELECT VALUE COUNT(1)" in query: + return [3] + if "c.usage.total_tokens AS tokens" in query: + return token_rows + if "c.activity_type," in query: + return activity_rows + raise AssertionError(f"Unexpected activity query: {query}") + + return { + "cosmos_user_settings_container": GuardedContainer(users), + "cosmos_groups_container": GuardedContainer(statuses_for("groups")), + "cosmos_public_workspaces_container": GuardedContainer(statuses_for("workspaces")), + "cosmos_activity_logs_container": GuardedContainer(activity), + "cosmos_user_documents_container": GuardedContainer(lambda query, values: [1]), + "cosmos_group_documents_container": GuardedContainer(lambda query, values: [0]), + "cosmos_public_documents_container": GuardedContainer(lambda query, values: [0]), + "cosmos_approvals_container": GuardedContainer(lambda query, values: [2]), + } + + +def test_summary_and_insights_routes_succeed_with_supported_queries(): + containers = _dashboard_store() + namespace = _dashboard_namespace(containers) + client = namespace["client"] + + summary = client.get("/api/v2/control-center/dashboard/summary?days=30") + insights = client.get("/api/v2/control-center/dashboard/insights?days=30") + problems = [problem for container in containers.values() for problem in container.problems] + assert problems == [], problems + assert summary.status_code == 200, namespace["events"] + assert insights.status_code == 200, namespace["events"] + + body = summary.get_json() + assert body["users"]["total"]["value"] == 4 + assert body["users"]["blocked"]["value"] == 1 + assert body["users"]["active"]["value"] == 2 + assert body["users"]["dau"]["value"] == 2 + assert body["groups"]["total"]["value"] == 5 + assert body["groups"]["by_status"]["locked"]["value"] == 1 + assert body["groups"]["by_status"]["inactive"]["value"] == 1 + assert body["groups"]["by_status"]["active"]["value"] == 3 + assert body["public_workspaces"]["by_status"]["inactive"]["value"] == 1 + assert body["public_workspaces"]["by_status"]["active"]["value"] == 2 + assert body["document_uploads"]["by_workspace_type"]["personal"]["value"] == 2 + assert body["document_uploads"]["total"]["value"] == 5 + assert body["tokens"]["value"] == 1234 + assert body["pending_approvals"]["value"] == 2 + assert body["document_processing_failures"]["value"] == 1 + assert body["cached"] is False + + data = insights.get_json() + assert data["token_usage_by_model"] == [{"date": "2026-09-20", "model": "gpt-4o", "tokens": 40}] + assert data["top_tokens"]["groups"] == [{"id": "g1", "tokens": 40}] + assert data["top_activity"]["users"] == [{"id": "u1", "activity_count": 2}] + assert data["login_heatmap"]["cells"] == [{"weekday": 0, "hour": 8, "count": 1}] + + query_count = sum(len(container.queries) for container in containers.values()) + assert client.get("/api/v2/control-center/dashboard/summary?days=30").get_json()["cached"] is True + assert sum(len(container.queries) for container in containers.values()) == query_count + + +def test_dashboard_storage_failures_do_not_expose_details(): + containers = _dashboard_store() + groups = containers["cosmos_groups_container"] + groups.respond = lambda query, values: (_ for _ in ()).throw(RuntimeError("secret-storage-detail")) + namespace = _dashboard_namespace(containers) + response = namespace["client"].get("/api/v2/control-center/dashboard/summary?days=7&force_refresh=1") + assert response.status_code == 500 + assert "secret-storage-detail" not in response.get_data(as_text=True) + assert response.get_json() == {"error": "Failed to retrieve dashboard summary."} def test_period_deltas_and_custom_range_boundaries(): @@ -163,10 +475,7 @@ def test_dashboard_reader_can_call_summary_and_insights_routes(): for node in ast.walk(route_tree) if isinstance(node, ast.FunctionDef) } - for name in ( - "api_v2_control_center_dashboard_summary", - "api_v2_control_center_dashboard_insights", - ): + for name in DASHBOARD_ROUTES: route = route_functions[name] decorators = [ ast.unparse(item) for item in route.decorator_list @@ -200,7 +509,8 @@ def test_summary_and_insights_cover_recorded_dashboard_fields(): "document_creation", "usage.total_tokens", "cosmos_approvals_container", - "GROUP BY c.status", + "SELECT VALUE c.status", + "SELECT DISTINCT VALUE c.user_id", "workspace_context.group_id", "workspace_context.public_workspace_id", "c.usage.model", @@ -218,10 +528,7 @@ def test_invalid_period_errors_do_not_expose_exception_text(): for node in ast.walk(route_tree) if isinstance(node, ast.FunctionDef) } - for name in ( - "api_v2_control_center_dashboard_summary", - "api_v2_control_center_dashboard_insights", - ): + for name in DASHBOARD_ROUTES: handlers = [ handler for node in ast.walk(route_functions[name]) @@ -236,12 +543,19 @@ def test_invalid_period_errors_do_not_expose_exception_text(): def test_version_is_at_least_implementation_version(): - assert_app_version_at_least("0.261.280") + assert_app_version_at_least("0.261.290") TESTS = [ test_group_and_workspace_status_counts_use_real_statuses, - test_document_uploads_are_aggregated_by_recorded_workspace_type, + test_status_rows_are_tallied_without_group_by, + test_document_uploads_are_counted_by_recorded_workspace_type, + test_active_users_are_counted_from_distinct_values, + test_token_insights_total_models_and_rank_consumers, + test_rankings_are_limited_and_ties_are_stable, + test_activity_insights_rank_actors_and_total_each_login_hour, + test_summary_and_insights_routes_succeed_with_supported_queries, + test_dashboard_storage_failures_do_not_expose_details, test_period_deltas_and_custom_range_boundaries, test_dashboard_cache_expires_and_can_be_bypassed, test_dashboard_reader_can_call_summary_and_insights_routes, diff --git a/functional_tests/test_v2_control_center_foundation.py b/functional_tests/test_v2_control_center_foundation.py index 40115b7e3..66ca1660c 100644 --- a/functional_tests/test_v2_control_center_foundation.py +++ b/functional_tests/test_v2_control_center_foundation.py @@ -2,11 +2,12 @@ # test_v2_control_center_foundation.py """ Functional test for the V2 Control Center foundation. -Version: 0.261.278 +Version: 0.261.290 Implemented in: 0.261.278 -This test covers the shared access capability contract, bootstrap wiring, migration -deduplication, removal of the legacy automatic migration check, and route/pane structure. +This test covers the shared access capability contract, bootstrap wiring, stable +activity-log IDs, removal of the legacy automatic migration check, removal of the +V2 Data health section and its backfill APIs (0.261.290), and route/pane structure. """ import ast @@ -94,66 +95,47 @@ def test_bootstrap_publishes_control_center_capabilities(): assert '"control_center": get_control_center_capabilities(session_user, settings)' in source -def test_migration_is_manual_and_legacy_controls_are_removed(): - """The classic UI no longer checks or offers the migration automatically.""" +def test_activity_log_backfill_is_removed_from_both_control_centers(): + """Neither Control Center offers the legacy backfill, and its APIs no longer exist.""" js = _source(LEGACY_JS) html = _source(LEGACY_HTML) page = _source(PAGE) + route = _source(BACKFILL) assert "/api/admin/control-center/migrate/status" not in js assert "checkMigrationStatus" not in js assert "migrationBanner" not in html assert "migrationConfirmModal" not in html - assert "Check activity-log status" in page - assert "Run backfill" in page - assert "useEffect" not in page, "Data health must not check status during page load." + for removed in ("data-health", "Data health", "MigrationDataHealth", "/control-center/migrate", "Run backfill"): + assert removed not in page, f"V2 Control Center still references {removed!r}." + for removed in ("/api/admin/control-center/migrate/", "api_get_migration_status", "api_migrate_to_activity_logs"): + assert removed not in route, f"The removed backfill API is still registered: {removed!r}." -def test_backfill_checks_existing_resource_logs_and_uses_stable_ids(): - """Existing records are skipped and reruns converge on deterministic record IDs.""" +def test_activity_log_ids_are_stable_and_backfill_lookup_is_removed(): + """Idempotent writers keep deterministic IDs; the backfill-only lookup helper is gone.""" activity = _source(ACTIVITY) - route = _source(BACKFILL) helper_tree = ast.parse(activity) + helper_names = {node.name for node in helper_tree.body if isinstance(node, ast.FunctionDef)} + assert "has_activity_log_for_resource" not in helper_names helper_nodes = [ node for node in helper_tree.body - if isinstance(node, ast.FunctionDef) - and node.name in {"build_activity_log_id", "has_activity_log_for_resource"} + if isinstance(node, ast.FunctionDef) and node.name == "build_activity_log_id" ] - class ActivityContainer: - def __init__(self, found): - self.found = found - self.calls = [] - - def query_items(self, **kwargs): - self.calls.append(kwargs) - return iter(["existing-id"] if self.found else []) - import uuid - for found in (False, True): - container = ActivityContainer(found) - namespace = { - "uuid": uuid, - "cosmos_activity_logs_container": container, - } - exec(compile(ast.Module(body=helper_nodes, type_ignores=[]), str(ACTIVITY), "exec"), namespace) - exists = namespace["has_activity_log_for_resource"]( - "user-1", "document_creation", "doc-1", "group" - ) - assert exists is found - assert container.calls[0]["partition_key"] == "user-1" - assert "@workspace_type" in container.calls[0]["query"] - assert container.calls[0]["parameters"][-1]["value"] == "group" - + namespace = {"uuid": uuid} + exec(compile(ast.Module(body=helper_nodes, type_ignores=[]), str(ACTIVITY), "exec"), namespace) stable_id = namespace["build_activity_log_id"]( "document_creation", "user-1", "backfill:group:doc-1" ) assert stable_id == namespace["build_activity_log_id"]( "document_creation", "user-1", "backfill:group:doc-1" ) - assert "has_activity_log_for_resource" in route - assert "build_activity_log_id" in route - assert "total_skipped_existing" in route + assert stable_id != namespace["build_activity_log_id"]( + "document_creation", "user-2", "backfill:group:doc-1" + ) + assert "build_activity_log_id(" in activity.split("def _create_activity_record", 1)[1] def test_pane_has_section_routes_and_capability_gating(): @@ -165,7 +147,7 @@ def test_pane_has_section_routes_and_capability_gating(): assert route in app for capability in ( "can_view_dashboard", "can_manage_users", "can_manage_groups", - "can_manage_workspaces", "can_view_activity_logs", "can_run_maintenance", + "can_manage_workspaces", "can_view_activity_logs", ): assert capability in page assert "canOpenControlCenter" in sidebar @@ -173,14 +155,14 @@ def test_pane_has_section_routes_and_capability_gating(): def test_version_is_at_least_the_implementing_release(): - assert_app_version_at_least("0.261.278") + assert_app_version_at_least("0.261.290") TESTS = [ test_capability_helper_matches_legacy_access_rules, test_bootstrap_publishes_control_center_capabilities, - test_migration_is_manual_and_legacy_controls_are_removed, - test_backfill_checks_existing_resource_logs_and_uses_stable_ids, + test_activity_log_backfill_is_removed_from_both_control_centers, + test_activity_log_ids_are_stable_and_backfill_lookup_is_removed, test_pane_has_section_routes_and_capability_gating, test_version_is_at_least_the_implementing_release, ] diff --git a/functional_tests/test_v2_control_center_groups.py b/functional_tests/test_v2_control_center_groups.py index 67d6d143c..58e3b5b6c 100644 --- a/functional_tests/test_v2_control_center_groups.py +++ b/functional_tests/test_v2_control_center_groups.py @@ -1,12 +1,14 @@ # test_v2_control_center_groups.py """ Functional tests for V2 Control Center Groups. -Version: 0.261.283 +Version: 0.261.290 Implemented in: 0.261.282 Run real filters and routes over isolated Cosmos services and the real guarded group writer. Cover selection caps before writes, audit parity, detail projections, admin-only access, snapshot expiry, safe exports and approval-only actions. +Since 0.261.290 the inventory fakes reject GROUP BY, because the Python Cosmos SDK +cannot run it across partitions; the inventory aggregates streamed projections. """ import ast @@ -22,6 +24,7 @@ from flask import Blueprint, Flask, Response from test_support.control_center_group_harness import control_center_group_environment +from test_support.cosmos_query_guard import assert_cosmos_query_supported from test_support.versioning import assert_app_version_at_least @@ -44,6 +47,7 @@ def __init__(self, env): self.queries = [] def query_items(self, query, **kwargs): + assert_cosmos_query_supported(query) self.queries.append(query) assert "SELECT c.id, c.name" in query and "FROM c" in query fields = ("id", "name", "description", "owner", "users", "admins", "documentManagers", @@ -60,10 +64,11 @@ def __init__(self): self.queries = [] def query_items(self, query, **kwargs): + assert_cosmos_query_supported(query) self.queries.append(query) assert "c.type = 'document_metadata'" in query - assert "COUNT(1)" in query and "GROUP BY c.group_id" in query - return [{"group_id": "group-1", "total": 2}] + assert "SELECT VALUE c.group_id" in query + return iter(["group-1", "group-1"]) class InventoryActivity: @@ -71,14 +76,18 @@ def __init__(self): self.queries = [] def query_items(self, query, **kwargs): + assert_cosmos_query_supported(query) self.queries.append(query) - if "SUM(c.usage.total_tokens)" in query: - assert "GROUP BY c.workspace_context.group_id" in query - return [{"group_id": "group-1", "total": 120}] - if "MAX(c.timestamp)" in query: + if "c.usage.total_tokens AS tokens" in query: + assert "c.activity_type = 'token_usage'" in query + return iter([{"group_id": "group-1", "tokens": 100}, {"group_id": "group-1", "tokens": 20}]) + if "IIF(" in query: assert "IS_STRING(c.group_id) AND c.group_id != ''" in query assert "IS_STRING(c.group.group_id) AND c.group.group_id != ''" in query - return [{"group_id": "group-1", "last_activity": "2026-10-06T00:00:00Z"}] + return iter([ + {"group_id": "group-1", "timestamp": "2026-10-06T00:00:00Z"}, + {"group_id": "group-1", "timestamp": "2026-10-01T00:00:00Z"}, + ]) if "TOP 20" in query: assert "c.group_id = @group_id" in query and "c.workspace_context.group_id = @group_id" in query return [{"id": "event-1", "activity_type": "group_status_change", "timestamp": "2026-10-06T00:00:00Z"}] @@ -186,6 +195,55 @@ def fail_query(**kwargs): assert "secret-storage-credential" not in response.get_data(as_text=True) +class ProjectionContainer: + def __init__(self, responses): + self.responses = responses + self.queries = [] + + def query_items(self, query, **kwargs): + assert_cosmos_query_supported(query) + assert kwargs.get("enable_cross_partition_query") is True + self.queries.append(query) + for marker, rows in self.responses: + if marker in query: + return iter(rows) + raise AssertionError(f"Unexpected inventory query: {query}") + + +def test_inventory_aggregates_projections_without_group_by(): + groups = ProjectionContainer([("SELECT c.id, c.name", [ + {"id": "alpha", "name": "Alpha", "owner": {"id": "o1"}, "users": []}, + {"id": "beta", "name": "Beta", "owner": {"id": "o2"}, "users": []}, + {"id": "idle", "name": "Idle", "owner": {"id": "o3"}, "users": []}, + ])]) + documents = ProjectionContainer([("SELECT VALUE c.group_id", ["alpha", "beta", "alpha", "orphan"])]) + activity = ProjectionContainer([ + ("c.usage.total_tokens AS tokens", [ + {"group_id": "alpha", "tokens": 100}, {"group_id": "alpha", "tokens": 2.5}, + {"group_id": "beta", "tokens": 7}, {"group_id": "orphan", "tokens": 50}, + ]), + ("IIF(", [ + {"group_id": "alpha", "timestamp": "2026-10-01T08:00:00Z"}, + {"group_id": "alpha", "timestamp": "2026-10-06T09:00:00Z"}, + {"group_id": "beta", "timestamp": "2026-09-30T23:59:59Z"}, + {"group_id": "alpha", "timestamp": "2026-10-03T00:00:00Z"}, + ]), + ]) + inventory = inventory_module.load_group_inventory(groups, documents, activity) + rows = {row["id"]: row for row in inventory["rows"]} + assert set(rows) == {"alpha", "beta", "idle"} + assert (rows["alpha"]["documents"], rows["alpha"]["tokens"], rows["alpha"]["last_activity"]) == ( + 2, 102, "2026-10-06T09:00:00Z", + ) + assert (rows["beta"]["documents"], rows["beta"]["tokens"], rows["beta"]["last_activity"]) == ( + 1, 7, "2026-09-30T23:59:59Z", + ) + assert (rows["idle"]["documents"], rows["idle"]["tokens"], rows["idle"]["last_activity"]) == (0, 0, None) + assert isinstance(rows["alpha"]["tokens"], int) + assert len(groups.queries) + len(documents.queries) + len(activity.queries) == 4 + assert inventory["calculated_at"] + + def test_missing_status_and_dates_and_all_sorts_are_consistent(): first = inventory_module.group_row({"id": "a", "name": "Zulu", "owner": {"id": "person"}, "users": [], "status": None}, 0, 0, None) @@ -331,7 +389,7 @@ def test_routes_have_explicit_admin_and_swagger_decorators(): decorators = [ast.unparse(decorator) for decorator in node.decorator_list] assert "login_required" in decorators and "control_center_required('admin')" in decorators assert "swagger_route(security=get_auth_security())" in decorators - assert_app_version_at_least("0.261.282") + assert_app_version_at_least("0.261.290") if __name__ == "__main__": diff --git a/functional_tests/test_v2_control_center_users.py b/functional_tests/test_v2_control_center_users.py index 9a4b8307a..5371e07b3 100644 --- a/functional_tests/test_v2_control_center_users.py +++ b/functional_tests/test_v2_control_center_users.py @@ -2,18 +2,29 @@ # test_v2_control_center_users.py """ Functional test for V2 Control Center user management. -Version: 0.261.280 +Version: 0.261.290 Implemented in: 0.261.280 This test validates Users filtering, sorting, paging, detail data boundaries, bulk-action limits, admin-only authorization, cached metric freshness and CSV safety. +Since 0.261.290 the list and export order one property per query, because the +user_settings container has no composite index for a two-property ORDER BY. """ import ast +import copy +import csv +import logging import re import sys from datetime import datetime, timedelta, timezone +from importlib.metadata import version as package_version +from io import StringIO from pathlib import Path +from unittest.mock import patch + +import werkzeug +from flask import Blueprint, Flask, Response, jsonify, request, stream_with_context ROOT = Path(__file__).resolve().parents[1] APP = ROOT / "application" / "single_app" @@ -21,6 +32,7 @@ AUTH = APP / "functions_authentication.py" sys.path.insert(0, str(ROOT / "functional_tests")) +from test_support.cosmos_query_guard import assert_cosmos_query_supported from test_support.versioning import assert_app_version_at_least @@ -28,22 +40,46 @@ "_control_center_validate_user_id", "_control_center_parse_user_filters", "_control_center_user_where", + "_control_center_user_populations", + "_control_center_count_users", + "_control_center_query_user_page", + "_control_center_iter_users", "_control_center_effective_restriction", "_control_center_user_row", "_control_center_csv_safe_cell", + "parse_control_center_management_pagination", + "get_control_center_total_pages", + "clamp_control_center_page", +} +ASSIGNMENTS = { + "CONTROL_CENTER_USER_ID_PATTERN", + "CONTROL_CENTER_USER_SORTS", + "CONTROL_CENTER_USER_FIELDS", + "CONTROL_CENTER_MANAGEMENT_DEFAULT_PER_PAGE", + "CONTROL_CENTER_MANAGEMENT_MAX_PER_PAGE", } +USER_ROUTES = {"api_v2_control_center_users", "api_v2_control_center_users_export"} +POPULATION = re.compile(r"(NOT )?\(IS_DEFINED\((c\.[\w.]+)\) AND NOT IS_NULL\(\2\)\)") +ORDER = re.compile(r"ORDER BY (c\.[\w.]+) (ASC|DESC)") +USERS = [ + {"id": "u-carol", "email": "carol@example.test", "display_name": "Carol", + "settings": {"metrics": {"token_metrics": {"total_tokens": 30}}}}, + {"id": "u-alice", "email": "alice@example.test", "display_name": "Alice", + "settings": {"metrics": {"token_metrics": {"total_tokens": 10}}}}, + {"id": "u-bob", "email": "bob@example.test", "display_name": "Bob", "settings": {}}, + {"id": "u-zed", "email": "zed@example.test", + "settings": {"metrics": {"token_metrics": {"total_tokens": None}}}}, + {"id": "u-amy", "email": "=amy@example.test", "display_name": None, + "settings": {"metrics": {"token_metrics": {"total_tokens": 20}}}}, +] -def _route_helpers(): +def _route_helpers(extra=None): tree = ast.parse(ROUTE.read_text(encoding="utf-8")) assignments = [ node for node in tree.body if isinstance(node, ast.Assign) - and any( - isinstance(target, ast.Name) - and target.id in {"CONTROL_CENTER_USER_ID_PATTERN", "CONTROL_CENTER_USER_SORTS"} - for target in node.targets - ) + and any(isinstance(target, ast.Name) and target.id in ASSIGNMENTS for target in node.targets) ] functions = [ node for node in tree.body @@ -55,6 +91,7 @@ def _route_helpers(): "datetime": datetime, "timedelta": timedelta, "timezone": timezone, + **(extra or {}), } exec(compile(ast.Module(body=assignments + functions, type_ignores=[]), str(ROUTE), "exec"), namespace) return namespace @@ -69,6 +106,70 @@ def _route_functions(path): } +def _field_value(row, path): + value = row + for key in path.removeprefix("c.").split("."): + value = value.get(key) if isinstance(value, dict) else None + return value + + +class UserSettingsContainer: + """Serve the Users query shapes as Cosmos would, after the SDK query guard accepts them.""" + + def __init__(self, users, fail=False): + self.users = users + self.fail = fail + self.queries = [] + + def query_items(self, query, parameters=None, **kwargs): + assert_cosmos_query_supported(query) + assert kwargs.get("enable_cross_partition_query") is True + self.queries.append(query) + if self.fail: + raise RuntimeError("secret-storage-detail") + values = {item["name"]: item["value"] for item in parameters or []} + rows = [copy.deepcopy(row) for row in self.users] + population = POPULATION.search(query) + if population: + recorded = population.group(1) is None + rows = [row for row in rows if (_field_value(row, population.group(2)) is not None) == recorded] + if "SELECT VALUE COUNT(1)" in query: + return iter([len(rows)]) + path, direction = ORDER.search(query).groups() + rows.sort(key=lambda row: _field_value(row, path), reverse=direction == "DESC") + if "OFFSET @offset LIMIT @limit" in query: + rows = rows[values["@offset"]:values["@offset"] + values["@limit"]] + return iter(rows) + + +def _users_client(container): + """Register the real Users list and export routes against a fake user_settings container.""" + tree = ast.parse(ROUTE.read_text(encoding="utf-8")) + routes = [node for node in ast.walk(tree) if isinstance(node, ast.FunctionDef) and node.name in USER_ROUTES] + assert {node.name for node in routes} == USER_ROUTES + blueprint = Blueprint("v2_users_test", __name__) + events = [] + + def passthrough(*_args, **_kwargs): + return lambda function: function + + namespace = _route_helpers({ + "csv": csv, "StringIO": StringIO, "Response": Response, "stream_with_context": stream_with_context, + "jsonify": jsonify, "request": request, "logging": logging, "bp": blueprint, + "swagger_route": passthrough, "get_auth_security": lambda: None, + "login_required": lambda function: function, "control_center_required": passthrough, + "log_event": lambda *args, **kwargs: events.append((args, kwargs)), + "cosmos_user_settings_container": container, + }) + exec(compile(ast.Module(body=routes, type_ignores=[]), str(ROUTE), "exec"), namespace) + app = Flask("v2_users_test") + app.config.update(TESTING=True) + app.register_blueprint(blueprint) + # Flask 2.x test clients read werkzeug.__version__, which Werkzeug 3 no longer defines. + with patch.object(werkzeug, "__version__", package_version("werkzeug"), create=True): + return app.test_client(), events + + def test_filter_contract_is_parameterized_and_supports_dashboard_drillthrough(): helpers = _route_helpers() now = datetime(2026, 10, 7, tzinfo=timezone.utc) @@ -101,6 +202,80 @@ def test_filter_contract_is_parameterized_and_supports_dashboard_drillthrough(): assert "CONTROL_CENTER_USER_SORTS[filters[\"sort\"]]" in route +def test_each_population_orders_one_property_without_a_composite_index(): + helpers = _route_helpers() + for sort, field in helpers["CONTROL_CENTER_USER_SORTS"].items(): + for direction in ("asc", "desc"): + filters = helpers["_control_center_parse_user_filters"]({"sort": sort, "direction": direction}) + (recorded, recorded_order), (missing, missing_order) = ( + helpers["_control_center_user_populations"](filters) + ) + assert recorded == f"(IS_DEFINED({field}) AND NOT IS_NULL({field}))" + assert missing == f"NOT {recorded}" + assert recorded_order == f"ORDER BY {field} {direction.upper()}" + assert missing_order == "ORDER BY c.id ASC" + for clause, order in ((recorded, recorded_order), (missing, missing_order)): + assert_cosmos_query_supported(f"SELECT c.id FROM c WHERE (1=1) AND {clause} {order}") + + +def test_list_pages_cross_from_recorded_to_missing_sort_values(): + container = UserSettingsContainer(USERS) + client, events = _users_client(container) + pages = [] + for page in (1, 2, 3): + response = client.get(f"/api/v2/control-center/users?sort=name&direction=asc&per_page=2&page={page}") + assert response.status_code == 200, events + payload = response.get_json() + assert payload["pagination"]["total_items"] == 5 + assert payload["pagination"]["total_pages"] == 3 + pages.append([user["id"] for user in payload["users"]]) + assert pages == [["u-alice", "u-bob"], ["u-carol", "u-amy"], ["u-zed"]] + + descending = client.get("/api/v2/control-center/users?sort=name&direction=desc&per_page=2&page=2") + assert [user["id"] for user in descending.get_json()["users"]] == ["u-alice", "u-amy"] + + tokens = client.get("/api/v2/control-center/users?sort=tokens&direction=desc&per_page=10") + assert [user["id"] for user in tokens.get_json()["users"]] == ["u-carol", "u-amy", "u-alice", "u-bob", "u-zed"] + assert [user["tokens"] for user in tokens.get_json()["users"]] == [30, 20, 10, None, None] + + clamped = client.get("/api/v2/control-center/users?sort=name&per_page=2&page=9").get_json() + assert clamped["pagination"]["page"] == 3 + assert [user["id"] for user in clamped["users"]] == ["u-zed"] + assert all("ORDER BY c.display_name ASC, c.id" not in query for query in container.queries) + + +def test_empty_user_list_returns_one_empty_page(): + client, _ = _users_client(UserSettingsContainer([])) + payload = client.get("/api/v2/control-center/users").get_json() + assert payload["users"] == [] + assert payload["pagination"] == { + "page": 1, "per_page": 25, "total_items": 0, "total_pages": 1, "has_prev": False, "has_next": False, + } + + +def test_export_streams_recorded_values_then_missing_values(): + client, events = _users_client(UserSettingsContainer(USERS)) + response = client.get("/api/v2/control-center/users/export.csv?sort=tokens&direction=asc") + assert response.status_code == 200, events + rows = list(csv.reader(StringIO(response.get_data(as_text=True)))) + assert rows[0][:3] == ["id", "display_name", "email"] + assert [row[0] for row in rows[1:]] == ["u-alice", "u-amy", "u-carol", "u-bob", "u-zed"] + assert rows[2][2] == "'=amy@example.test" + + +def test_list_and_export_failures_are_safe_and_happen_before_streaming(): + client, events = _users_client(UserSettingsContainer(USERS, fail=True)) + for path in ("/api/v2/control-center/users", "/api/v2/control-center/users/export.csv"): + response = client.get(path) + assert response.status_code == 500 + assert response.mimetype == "application/json" + assert "secret-storage-detail" not in response.get_data(as_text=True) + assert {args[0] for args, _ in events} == { + "[CONTROL_CENTER] V2 user list query failed.", + "[CONTROL_CENTER] V2 user export query failed.", + } + + def test_filter_validation_rejects_unbounded_or_unknown_fields(): parse_filters = _route_helpers()["_control_center_parse_user_filters"] for query, message in ( @@ -209,7 +384,7 @@ def test_csv_cells_neutralize_all_formula_prefixes_and_export_is_streamed(): _route_functions(ROUTE)["api_v2_control_center_users_export"], ) assert "stream_with_context(stream_csv())" in route_source - assert "_control_center_user_where(filters)" in route_source + assert "_control_center_iter_users(" in route_source assert "_control_center_csv_safe_cell(value)" in route_source @@ -243,11 +418,16 @@ def test_routes_require_full_control_center_admin_not_dashboard_reader(): def test_version_is_at_least_the_implementation_version(): - assert_app_version_at_least("0.261.280") + assert_app_version_at_least("0.261.290") TESTS = [ test_filter_contract_is_parameterized_and_supports_dashboard_drillthrough, + test_each_population_orders_one_property_without_a_composite_index, + test_list_pages_cross_from_recorded_to_missing_sort_values, + test_empty_user_list_returns_one_empty_page, + test_export_streams_recorded_values_then_missing_values, + test_list_and_export_failures_are_safe_and_happen_before_streaming, test_filter_validation_rejects_unbounded_or_unknown_fields, test_user_ids_expiry_and_cached_row_projection_are_validated, test_allow_filters_exclude_active_denials_with_null_or_future_expiry, diff --git a/ui_tests/test_v2_control_center_data_health.py b/ui_tests/test_v2_control_center_data_health.py deleted file mode 100644 index 4d08593c1..000000000 --- a/ui_tests/test_v2_control_center_data_health.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python3 -# test_v2_control_center_data_health.py -""" -Browser coverage for the V2 Control Center data-health flow. -Version: 0.261.278 -Implemented in: 0.261.278 - -Validates capability-based discovery, on-demand status checks and the explicit confirmation -before the activity-log backfill, without contacting a live SimpleChat deployment. -""" - -import json -import sys -from pathlib import Path -from urllib.parse import urlsplit - -import pytest -from playwright.sync_api import Page, Route, expect - -sys.path.insert(0, str(Path(__file__).resolve().parent / "fixtures")) - -from playwright_connection import connect_options # noqa: F401 -from v2_notification_stubs import is_notification_count, notification_count_payload - - -ROOT = Path(__file__).resolve().parents[1] -STATIC = ROOT / "application" / "single_app" / "static" -SPA_INDEX = STATIC / "v2" / "index.html" -ORIGIN = "http://simplechat.test" - - -class ControlCenterFixture: - def __init__(self, page: Page, capabilities=None): - self.page = page - self.capabilities = capabilities or { - "can_view_dashboard": True, - "can_manage_users": True, - "can_manage_groups": True, - "can_manage_workspaces": True, - "can_view_activity_logs": True, - "can_run_maintenance": True, - } - self.requests = [] - self.unexpected = [] - page.route("**/*", self._route) - - def _route(self, route: Route): - request = route.request - parsed = urlsplit(request.url) - path = parsed.path - if f"{parsed.scheme}://{parsed.netloc}" != ORIGIN: - self.unexpected.append(request.url) - route.abort() - return - if request.method == "GET" and path.startswith("/v2"): - route.fulfill(path=str(SPA_INDEX), content_type="text/html") - elif request.method == "GET" and path.startswith("/static/"): - asset = (STATIC / path.removeprefix("/static/")).resolve() - if asset.is_relative_to(STATIC.resolve()) and asset.is_file(): - route.fulfill(path=str(asset)) - else: - self.unexpected.append(path) - route.fulfill(status=404, body="Fixture asset not found") - elif path == "/api/v2/bootstrap" and request.method == "GET": - route.fulfill(json={ - "version": "0.261.278", - "user": {"id": "test-admin", "display_name": "Test Admin", "is_admin": True, "roles": ["Admin"]}, - "branding": {"app_title": "SimpleChat", "show_logo": False, "hide_app_title": False}, - "features": {}, - "control_center": self.capabilities, - "catalogs": {"models": [], "agents": [], "prompts": [], "initial_model_selection": None}, - "scope": {"groups": [], "public_workspaces": []}, - "navigation": { - "custom_pages": {"enabled": False, "items": []}, - "external_links": {"enabled": False, "items": []}, - }, - "workspace": {"sections": {}}, - "admin_nav": [], - "notices": {"ai": {}, "web_search": {}}, - "settings": {}, - }) - elif path == "/api/user/settings" and request.method == "GET": - route.fulfill(json={"settings": {}}) - elif is_notification_count(request.method, path): - route.fulfill(json=notification_count_payload()) - elif path == "/api/admin/control-center/migrate/status" and request.method == "GET": - self.requests.append(("GET", path)) - route.fulfill(json={ - "conversations_without_logs": 2, - "personal_documents_without_logs": 1, - "group_documents_without_logs": 0, - "public_documents_without_logs": 3, - "total_documents_without_logs": 4, - "migration_needed": True, - "estimated_total_records": 6, - }) - elif path == "/api/admin/control-center/migrate/all" and request.method == "POST": - self.requests.append(("POST", path)) - route.fulfill(json={ - "conversations_migrated": 0, - "conversations_skipped_existing": 2, - "personal_documents_migrated": 1, - "personal_documents_skipped_existing": 0, - "group_documents_migrated": 0, - "group_documents_skipped_existing": 0, - "public_documents_migrated": 0, - "public_documents_skipped_existing": 3, - "total_migrated": 1, - "total_skipped_existing": 5, - "total_failed": 0, - }) - else: - self.unexpected.append(f"{request.method} {path}") - route.fulfill(status=404, json={"error": "Unexpected fixture request."}) - - def open(self, *, width=1440): - if not SPA_INDEX.is_file(): - pytest.fail("Build the V2 SPA first: npm --prefix application/v2_ui run build") - self.page.set_viewport_size({"width": width, "height": 900}) - self.page.goto(f"{ORIGIN}/v2/control-center/data-health", wait_until="networkidle") - - def assert_clean(self): - assert not self.unexpected, self.unexpected - - -@pytest.fixture -def control_center_ui(page): - fixture = ControlCenterFixture(page) - yield fixture - fixture.assert_clean() - - -pytestmark = pytest.mark.ui - - -def test_data_health_checks_only_on_demand(control_center_ui): - control_center_ui.open() - page = control_center_ui.page - - expect(page.get_by_role("heading", name="Activity-log data health")).to_be_visible() - assert control_center_ui.requests == [] - page.get_by_role("button", name="Check activity-log status").click() - expect(page.get_by_text("Estimated records", exact=True)).to_be_visible() - expect(page.get_by_text("6", exact=True)).to_be_visible() - assert control_center_ui.requests == [ - ("GET", "/api/admin/control-center/migrate/status"), - ] - - -def test_backfill_waits_for_confirmation(control_center_ui): - control_center_ui.open() - page = control_center_ui.page - - page.get_by_role("button", name="Run backfill").click() - dialog = page.get_by_role("dialog", name="Run activity-log backfill?") - expect(dialog).to_be_visible() - assert control_center_ui.requests == [] - dialog.get_by_role("button", name="Run backfill").click() - expect(page.get_by_text("1 records added; 5 existing activity records skipped; 0 failures.")).to_be_visible() - assert control_center_ui.requests == [ - ("POST", "/api/admin/control-center/migrate/all"), - ] - - -def test_control_center_menu_and_sections_follow_capabilities(page): - fixture = ControlCenterFixture(page, capabilities={ - "can_view_dashboard": True, - "can_manage_users": False, - "can_manage_groups": False, - "can_manage_workspaces": False, - "can_view_activity_logs": False, - "can_run_maintenance": False, - }) - fixture.open() - page.locator('button[aria-controls="sidebar-account-menu"]').click() - expect(page.get_by_role("link", name="Control Center")).to_be_visible() - page.get_by_role("link", name="Control Center").click() - expect(page.get_by_role("complementary", name="Control Center sections").get_by_role("link")).to_have_count(1) - expect(page.get_by_role("link", name="Dashboard")).to_be_visible() - page.goto(f"{ORIGIN}/v2/control-center/data-health", wait_until="networkidle") - expect(page.get_by_role("alert")).to_contain_text("permissions do not include this section") - assert fixture.requests == [] - fixture.assert_clean() diff --git a/ui_tests/test_v2_control_center_data_health_removed.py b/ui_tests/test_v2_control_center_data_health_removed.py new file mode 100644 index 000000000..a65f457ef --- /dev/null +++ b/ui_tests/test_v2_control_center_data_health_removed.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +# test_v2_control_center_data_health_removed.py +""" +Browser coverage for the V2 Control Center section rail after Data health was removed. +Version: 0.261.290 +Implemented in: 0.261.290 + +Validates that the rail offers no Data health section even to maintenance-capable +administrators, that an old data-health bookmark opens the Dashboard without calling the +removed activity-log backfill APIs, and that sections still follow capabilities. +""" + +import sys +from pathlib import Path +from urllib.parse import urlsplit + +import pytest +from playwright.sync_api import Page, Route, expect + +sys.path.insert(0, str(Path(__file__).resolve().parent / "fixtures")) + +from playwright_connection import connect_options # noqa: F401 +from v2_notification_stubs import is_notification_count, notification_count_payload + + +ROOT = Path(__file__).resolve().parents[1] +STATIC = ROOT / "application" / "single_app" / "static" +SPA_INDEX = STATIC / "v2" / "index.html" +ORIGIN = "http://simplechat.test" +ALL_CAPABILITIES = { + "can_view_dashboard": True, + "can_manage_users": True, + "can_manage_groups": True, + "can_manage_workspaces": True, + "can_view_activity_logs": True, + "can_run_maintenance": True, +} +SECTION_LABELS = ["Dashboard", "Users", "Groups", "Public Workspaces", "Activity Logs"] + + +def metric(value): + return {"value": value, "delta": None, "previous": None, "percent_change": None} + + +def dashboard_summary(): + statuses = {status: metric(0) for status in ("active", "locked", "upload_disabled", "inactive")} + return { + "period": {"start_date": "2026-09-08", "end_date": "2026-10-07", "days": 30, "timezone": "UTC"}, + "refreshed_at": "2026-10-07T12:00:00Z", + "users": {key: metric(1) for key in ("total", "active", "dau", "wau", "mau", "blocked")}, + "groups": {"total": metric(0), "by_status": statuses}, + "public_workspaces": {"total": metric(0), "by_status": statuses}, + "conversations": metric(0), + "document_uploads": { + "total": metric(0), + "by_workspace_type": {key: metric(0) for key in ("personal", "group", "public")}, + }, + "document_processing_failures": {**metric(0), "available": True}, + "tokens": metric(0), + "pending_approvals": metric(0), + "status_history_available": False, + "cached": False, + } + + +def dashboard_insights(): + return { + "period": {"start_date": "2026-09-08", "end_date": "2026-10-07", "days": 30, "timezone": "UTC"}, + "token_usage_by_model": [], + "top_tokens": {"users": [], "groups": [], "public_workspaces": []}, + "top_activity": {"users": [], "groups": [], "public_workspaces": []}, + "login_heatmap": {"weekday_convention": "Monday=0 through Sunday=6", "timezone": "UTC", "cells": []}, + } + + +class ControlCenterFixture: + def __init__(self, page: Page, capabilities=None): + self.page = page + self.capabilities = capabilities or ALL_CAPABILITIES + self.removed_api_requests = [] + self.unexpected = [] + page.route("**/*", self._route) + + def _route(self, route: Route): + request = route.request + parsed = urlsplit(request.url) + path = parsed.path + if f"{parsed.scheme}://{parsed.netloc}" != ORIGIN: + self.unexpected.append(request.url) + route.abort() + return + if path.startswith("/api/admin/control-center/migrate"): + self.removed_api_requests.append(f"{request.method} {path}") + route.fulfill(status=404, json={"error": "Not found."}) + elif request.method == "GET" and path.startswith("/v2"): + route.fulfill(path=str(SPA_INDEX), content_type="text/html") + elif request.method == "GET" and path.startswith("/static/"): + asset = (STATIC / path.removeprefix("/static/")).resolve() + if asset.is_relative_to(STATIC.resolve()) and asset.is_file(): + route.fulfill(path=str(asset)) + else: + self.unexpected.append(path) + route.fulfill(status=404, body="Fixture asset not found") + elif path == "/api/v2/bootstrap" and request.method == "GET": + route.fulfill(json={ + "version": "0.261.290", + "user": {"id": "test-admin", "display_name": "Test Admin", "is_admin": True, "roles": ["Admin"]}, + "branding": {"app_title": "SimpleChat", "show_logo": False, "hide_app_title": False}, + "features": {}, + "control_center": self.capabilities, + "catalogs": {"models": [], "agents": [], "prompts": [], "initial_model_selection": None}, + "scope": {"groups": [], "public_workspaces": []}, + "navigation": { + "custom_pages": {"enabled": False, "items": []}, + "external_links": {"enabled": False, "items": []}, + }, + "workspace": {"sections": {}}, + "admin_nav": [], + "notices": {"ai": {}, "web_search": {}}, + "settings": {}, + }) + elif path == "/api/user/settings" and request.method == "GET": + route.fulfill(json={"settings": {}}) + elif is_notification_count(request.method, path): + route.fulfill(json=notification_count_payload()) + elif path == "/api/admin/control-center/token-filters" and request.method == "GET": + route.fulfill(json={"success": True, "filters": { + "users": [], "groups": [], "public_workspaces": [], "models": [], + "workspace_types": [], "token_types": [], + }}) + elif path == "/api/v2/control-center/dashboard/summary" and request.method == "GET": + route.fulfill(json=dashboard_summary()) + elif path == "/api/v2/control-center/dashboard/insights" and request.method == "GET": + route.fulfill(json=dashboard_insights()) + elif path == "/api/admin/control-center/activity-trends" and request.method == "GET": + route.fulfill(json={"success": True, "activity_data": {}}) + else: + self.unexpected.append(f"{request.method} {path}") + route.fulfill(status=404, json={"error": "Unexpected fixture request."}) + + def open(self, section, *, width=1440): + if not SPA_INDEX.is_file(): + pytest.fail("Build the V2 SPA first: npm --prefix application/v2_ui run build") + self.page.set_viewport_size({"width": width, "height": 900}) + self.page.goto(f"{ORIGIN}/v2/control-center/{section}", wait_until="networkidle") + + def assert_clean(self): + assert not self.unexpected, self.unexpected + assert not self.removed_api_requests, self.removed_api_requests + + +pytestmark = pytest.mark.ui + + +def test_data_health_bookmark_opens_dashboard_without_backfill_calls(page): + fixture = ControlCenterFixture(page) + fixture.open("data-health") + + rail = page.get_by_role("complementary", name="Control Center sections") + expect(rail.get_by_role("link")).to_have_text(SECTION_LABELS) + expect(rail.get_by_role("link", name="Data health")).to_have_count(0) + expect(page.get_by_role("heading", name="Dashboard")).to_be_visible() + expect(page.get_by_text("Activity-log data health")).to_have_count(0) + expect(page.get_by_role("button", name="Run backfill")).to_have_count(0) + fixture.assert_clean() + + +def test_mobile_section_picker_has_no_data_health_option(page): + fixture = ControlCenterFixture(page) + fixture.open("data-health", width=390) + + picker = page.get_by_label("Control Center section", exact=True) + expect(picker).to_be_visible() + expect(picker.locator("option")).to_have_text(SECTION_LABELS) + expect(picker).to_have_value("dashboard") + fixture.assert_clean() + + +def test_control_center_sections_follow_capabilities(page): + fixture = ControlCenterFixture(page, capabilities={ + **{key: False for key in ALL_CAPABILITIES}, + "can_view_dashboard": True, + "can_run_maintenance": True, + }) + fixture.open("users") + + expect(page.get_by_role("alert")).to_contain_text("permissions do not include this section") + rail = page.get_by_role("complementary", name="Control Center sections") + expect(rail.get_by_role("link")).to_have_text(["Dashboard"]) + rail.get_by_role("link", name="Dashboard").click() + expect(page.get_by_role("heading", name="Dashboard")).to_be_visible() + fixture.assert_clean() From b531d57c04f7a62d942b41962f6a8316dac82eef Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Wed, 7 Oct 2026 15:42:54 -0400 Subject: [PATCH 2/3] Move the Control Center query fix to 0.261.291 The merged base already uses 0.261.290 for the V2 sidebar links fix, so the fix doc, feature doc, Data health removal note, test headers and version assertions now name 0.261.291. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/explanation/features/V2_CONTROL_CENTER.md | 6 +++--- docs/explanation/fixes/ACTIVITY_LOG_MIGRATION_PROMPT_FIX.md | 2 +- .../V2_CONTROL_CENTER_COSMOS_QUERY_COMPATIBILITY_FIX.md | 4 ++-- .../test_v2_control_center_cosmos_query_compatibility.py | 6 +++--- functional_tests/test_v2_control_center_dashboard.py | 6 +++--- functional_tests/test_v2_control_center_foundation.py | 6 +++--- functional_tests/test_v2_control_center_groups.py | 6 +++--- functional_tests/test_v2_control_center_users.py | 6 +++--- ui_tests/test_v2_control_center_data_health_removed.py | 6 +++--- 9 files changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/explanation/features/V2_CONTROL_CENTER.md b/docs/explanation/features/V2_CONTROL_CENTER.md index 66b031fbf..a4e14d53f 100644 --- a/docs/explanation/features/V2_CONTROL_CENTER.md +++ b/docs/explanation/features/V2_CONTROL_CENTER.md @@ -8,7 +8,7 @@ The V2 Control Center is a permission-aware administration pane for managing Sim **Groups implemented in version:** 0.261.282 **Activity Logs implemented in version:** 0.261.284 **Public Workspaces implemented in version:** 0.261.283 -**Current version:** 0.261.290 (Dashboard, Users and Groups query fixes for the Python Cosmos SDK; Data health removed) +**Current version:** 0.261.291 (Dashboard, Users and Groups query fixes for the Python Cosmos SDK; Data health removed) **Dependencies:** React 18, TypeScript, Vite, Flask session authentication, and the existing Control Center APIs. @@ -16,7 +16,7 @@ The V2 Control Center is a permission-aware administration pane for managing Sim The Control Center is a distinct React route (`/control-center` and `/control-center/
`), reached from the account menu when the signed-in user has at least one Control Center capability. It is not a primary workspace-navigation item. The internal section rail has a separate per-user collapsed-state preference. -`get_control_center_capabilities()` in `functions_authentication.py` is the shared permission decision for both `control_center_required()` and the `/api/v2/bootstrap` response. When the ControlCenterAdmin role requirement is enabled, that role grants all capabilities; otherwise the regular Admin role grants them. The optional ControlCenterDashboardReader role grants dashboard viewing only when its setting is enabled. The bootstrap payload exposes `can_view_dashboard`, `can_manage_users`, `can_manage_groups`, `can_manage_workspaces`, `can_view_activity_logs`, and `can_run_maintenance`. Since Data health was removed in 0.261.290, no V2 section uses `can_run_maintenance`; it remains part of the shared capability contract. +`get_control_center_capabilities()` in `functions_authentication.py` is the shared permission decision for both `control_center_required()` and the `/api/v2/bootstrap` response. When the ControlCenterAdmin role requirement is enabled, that role grants all capabilities; otherwise the regular Admin role grants them. The optional ControlCenterDashboardReader role grants dashboard viewing only when its setting is enabled. The bootstrap payload exposes `can_view_dashboard`, `can_manage_users`, `can_manage_groups`, `can_manage_workspaces`, `can_view_activity_logs`, and `can_run_maintenance`. Since Data health was removed in 0.261.291, no V2 section uses `can_run_maintenance`; it remains part of the shared capability contract. ## Dashboard @@ -180,4 +180,4 @@ Functional checks cover dashboard status normalization, period deltas, cache exp ## Version tracking -The application version is defined by `VERSION` in `application/single_app/config.py`. The foundation was added in **0.261.278**, the dashboard in **0.261.279**, user management in **0.261.280**, group management in **0.261.282**, public workspace management in **0.261.283**, and Activity Logs in **0.261.284**. In **0.261.290**, the Dashboard, Users and Groups queries were made compatible with the Python Cosmos SDK. The same release removed the Data health section and its `GET /api/admin/control-center/migrate/status` and `POST /api/admin/control-center/migrate/all` backfill APIs; the classic Control Center had already stopped using them. +The application version is defined by `VERSION` in `application/single_app/config.py`. The foundation was added in **0.261.278**, the dashboard in **0.261.279**, user management in **0.261.280**, group management in **0.261.282**, public workspace management in **0.261.283**, and Activity Logs in **0.261.284**. In **0.261.291**, the Dashboard, Users and Groups queries were made compatible with the Python Cosmos SDK. The same release removed the Data health section and its `GET /api/admin/control-center/migrate/status` and `POST /api/admin/control-center/migrate/all` backfill APIs; the classic Control Center had already stopped using them. diff --git a/docs/explanation/fixes/ACTIVITY_LOG_MIGRATION_PROMPT_FIX.md b/docs/explanation/fixes/ACTIVITY_LOG_MIGRATION_PROMPT_FIX.md index dfb49f8d6..e0c0426d3 100644 --- a/docs/explanation/fixes/ACTIVITY_LOG_MIGRATION_PROMPT_FIX.md +++ b/docs/explanation/fixes/ACTIVITY_LOG_MIGRATION_PROMPT_FIX.md @@ -25,6 +25,6 @@ Implemented in version **0.261.278** (`application/single_app/config.py`): Opening the classic Control Center no longer triggers four cross-partition count scans or a misleading migration prompt. Administrators retain an explicit diagnostic and recovery path in V2. Existing logs are preserved; the manual backfill skips records that already have matching creation events. -## Update in 0.261.290 +## Update in 0.261.291 The V2 Data health section was removed, and with it the only callers of `GET /api/admin/control-center/migrate/status` and `POST /api/admin/control-center/migrate/all`. Both APIs and the backfill-only `has_activity_log_for_resource` helper were deleted, so neither Control Center offers the backfill any longer. Normal application workflows continue to write activity records, and `build_activity_log_id` still gives idempotent writers stable record IDs. `ui_tests/test_v2_control_center_data_health.py` was replaced by `ui_tests/test_v2_control_center_data_health_removed.py`, and `functional_tests/test_v2_control_center_foundation.py` now checks that the section and APIs stay removed. See [V2 Control Center Cosmos Query Compatibility Fix](V2_CONTROL_CENTER_COSMOS_QUERY_COMPATIBILITY_FIX.md). diff --git a/docs/explanation/fixes/V2_CONTROL_CENTER_COSMOS_QUERY_COMPATIBILITY_FIX.md b/docs/explanation/fixes/V2_CONTROL_CENTER_COSMOS_QUERY_COMPATIBILITY_FIX.md index 689034a6d..72dfa44bf 100644 --- a/docs/explanation/fixes/V2_CONTROL_CENTER_COSMOS_QUERY_COMPATIBILITY_FIX.md +++ b/docs/explanation/fixes/V2_CONTROL_CENTER_COSMOS_QUERY_COMPATIBILITY_FIX.md @@ -1,6 +1,6 @@ # V2 Control Center Cosmos Query Compatibility Fix -**Fixed in version:** 0.261.290 +**Fixed in version:** 0.261.291 ## Issue @@ -54,7 +54,7 @@ The response contracts are unchanged. Each unsupported query is replaced with a - `application/single_app/route_backend_control_center.py`: dashboard summary and insights helpers, and Users paging and export helpers. - `application/single_app/functions_control_center_groups.py`: `load_group_inventory`. -- `application/single_app/config.py`: version 0.261.290. +- `application/single_app/config.py`: version 0.261.291. - `functional_tests/test_support/cosmos_query_guard.py` (new): rejects `GROUP BY`, `COUNT` over `DISTINCT` values, and multi-property `ORDER BY` without a declared composite index. - `functional_tests/test_v2_control_center_cosmos_query_compatibility.py` (new): scans every SQL string in the V2 Control Center code paths through the guard. The activity-log `ORDER BY` passes only because `config.py` declares its composite index. - `functional_tests/test_v2_control_center_dashboard.py`, `functional_tests/test_v2_control_center_users.py` and `functional_tests/test_v2_control_center_groups.py`: the fakes now pass every query through the guard. The tests run the real routes and aggregation helpers. diff --git a/functional_tests/test_v2_control_center_cosmos_query_compatibility.py b/functional_tests/test_v2_control_center_cosmos_query_compatibility.py index 1834037bc..45ccde3dc 100644 --- a/functional_tests/test_v2_control_center_cosmos_query_compatibility.py +++ b/functional_tests/test_v2_control_center_cosmos_query_compatibility.py @@ -2,8 +2,8 @@ # test_v2_control_center_cosmos_query_compatibility.py """ Functional test for V2 Control Center Cosmos query compatibility. -Version: 0.261.290 -Implemented in: 0.261.290 +Version: 0.261.291 +Implemented in: 0.261.291 The Dashboard, Users and Groups sections returned HTTP 500 because Cosmos DB rejected their queries with HTTP 400. The azure-cosmos Python SDK cannot run cross-partition @@ -151,7 +151,7 @@ def test_scan_covers_the_previously_failing_sections(): def test_version_is_at_least_the_implementation_version(): - assert_app_version_at_least("0.261.290") + assert_app_version_at_least("0.261.291") TESTS = [ diff --git a/functional_tests/test_v2_control_center_dashboard.py b/functional_tests/test_v2_control_center_dashboard.py index 1764d42c2..90d2af3c6 100644 --- a/functional_tests/test_v2_control_center_dashboard.py +++ b/functional_tests/test_v2_control_center_dashboard.py @@ -2,12 +2,12 @@ # test_v2_control_center_dashboard.py """ Functional test for the V2 Control Center dashboard. -Version: 0.261.290 +Version: 0.261.291 Implemented in: 0.261.280 This test validates status aggregation, period comparisons, bounded cache behavior, dashboard-reader authorization, and the summary and insights route contracts. -Since 0.261.290 every dashboard query also passes the Cosmos query guard, because the +Since 0.261.291 every dashboard query also passes the Cosmos query guard, because the Python Cosmos SDK cannot run cross-partition GROUP BY or COUNT over DISTINCT values. """ @@ -543,7 +543,7 @@ def test_invalid_period_errors_do_not_expose_exception_text(): def test_version_is_at_least_implementation_version(): - assert_app_version_at_least("0.261.290") + assert_app_version_at_least("0.261.291") TESTS = [ diff --git a/functional_tests/test_v2_control_center_foundation.py b/functional_tests/test_v2_control_center_foundation.py index 66ca1660c..09e8c88d8 100644 --- a/functional_tests/test_v2_control_center_foundation.py +++ b/functional_tests/test_v2_control_center_foundation.py @@ -2,12 +2,12 @@ # test_v2_control_center_foundation.py """ Functional test for the V2 Control Center foundation. -Version: 0.261.290 +Version: 0.261.291 Implemented in: 0.261.278 This test covers the shared access capability contract, bootstrap wiring, stable activity-log IDs, removal of the legacy automatic migration check, removal of the -V2 Data health section and its backfill APIs (0.261.290), and route/pane structure. +V2 Data health section and its backfill APIs (0.261.291), and route/pane structure. """ import ast @@ -155,7 +155,7 @@ def test_pane_has_section_routes_and_capability_gating(): def test_version_is_at_least_the_implementing_release(): - assert_app_version_at_least("0.261.290") + assert_app_version_at_least("0.261.291") TESTS = [ diff --git a/functional_tests/test_v2_control_center_groups.py b/functional_tests/test_v2_control_center_groups.py index 58e3b5b6c..fae415fbd 100644 --- a/functional_tests/test_v2_control_center_groups.py +++ b/functional_tests/test_v2_control_center_groups.py @@ -1,13 +1,13 @@ # test_v2_control_center_groups.py """ Functional tests for V2 Control Center Groups. -Version: 0.261.290 +Version: 0.261.291 Implemented in: 0.261.282 Run real filters and routes over isolated Cosmos services and the real guarded group writer. Cover selection caps before writes, audit parity, detail projections, admin-only access, snapshot expiry, safe exports and approval-only actions. -Since 0.261.290 the inventory fakes reject GROUP BY, because the Python Cosmos SDK +Since 0.261.291 the inventory fakes reject GROUP BY, because the Python Cosmos SDK cannot run it across partitions; the inventory aggregates streamed projections. """ @@ -389,7 +389,7 @@ def test_routes_have_explicit_admin_and_swagger_decorators(): decorators = [ast.unparse(decorator) for decorator in node.decorator_list] assert "login_required" in decorators and "control_center_required('admin')" in decorators assert "swagger_route(security=get_auth_security())" in decorators - assert_app_version_at_least("0.261.290") + assert_app_version_at_least("0.261.291") if __name__ == "__main__": diff --git a/functional_tests/test_v2_control_center_users.py b/functional_tests/test_v2_control_center_users.py index 5371e07b3..624295059 100644 --- a/functional_tests/test_v2_control_center_users.py +++ b/functional_tests/test_v2_control_center_users.py @@ -2,12 +2,12 @@ # test_v2_control_center_users.py """ Functional test for V2 Control Center user management. -Version: 0.261.290 +Version: 0.261.291 Implemented in: 0.261.280 This test validates Users filtering, sorting, paging, detail data boundaries, bulk-action limits, admin-only authorization, cached metric freshness and CSV safety. -Since 0.261.290 the list and export order one property per query, because the +Since 0.261.291 the list and export order one property per query, because the user_settings container has no composite index for a two-property ORDER BY. """ @@ -418,7 +418,7 @@ def test_routes_require_full_control_center_admin_not_dashboard_reader(): def test_version_is_at_least_the_implementation_version(): - assert_app_version_at_least("0.261.290") + assert_app_version_at_least("0.261.291") TESTS = [ diff --git a/ui_tests/test_v2_control_center_data_health_removed.py b/ui_tests/test_v2_control_center_data_health_removed.py index a65f457ef..16081b7e4 100644 --- a/ui_tests/test_v2_control_center_data_health_removed.py +++ b/ui_tests/test_v2_control_center_data_health_removed.py @@ -2,8 +2,8 @@ # test_v2_control_center_data_health_removed.py """ Browser coverage for the V2 Control Center section rail after Data health was removed. -Version: 0.261.290 -Implemented in: 0.261.290 +Version: 0.261.291 +Implemented in: 0.261.291 Validates that the rail offers no Data health section even to maintenance-capable administrators, that an old data-health bookmark opens the Dashboard without calling the @@ -103,7 +103,7 @@ def _route(self, route: Route): route.fulfill(status=404, body="Fixture asset not found") elif path == "/api/v2/bootstrap" and request.method == "GET": route.fulfill(json={ - "version": "0.261.290", + "version": "0.261.291", "user": {"id": "test-admin", "display_name": "Test Admin", "is_admin": True, "roles": ["Admin"]}, "branding": {"app_title": "SimpleChat", "show_logo": False, "hide_app_title": False}, "features": {}, From 1dc32ea5d539f7514e2a97b512161c242c038d69 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Wed, 7 Oct 2026 15:49:43 -0400 Subject: [PATCH 3/3] Move the Control Center query fix to 0.261.292 The merged base now uses 0.261.291 for the orchestrated Ask an agent fix, so the fix doc, feature doc, Data health removal note, test headers and version assertions now name 0.261.292. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/explanation/features/V2_CONTROL_CENTER.md | 6 +++--- docs/explanation/fixes/ACTIVITY_LOG_MIGRATION_PROMPT_FIX.md | 2 +- .../V2_CONTROL_CENTER_COSMOS_QUERY_COMPATIBILITY_FIX.md | 4 ++-- .../test_v2_control_center_cosmos_query_compatibility.py | 6 +++--- functional_tests/test_v2_control_center_dashboard.py | 6 +++--- functional_tests/test_v2_control_center_foundation.py | 6 +++--- functional_tests/test_v2_control_center_groups.py | 6 +++--- functional_tests/test_v2_control_center_users.py | 6 +++--- ui_tests/test_v2_control_center_data_health_removed.py | 6 +++--- 9 files changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/explanation/features/V2_CONTROL_CENTER.md b/docs/explanation/features/V2_CONTROL_CENTER.md index a4e14d53f..d68b41831 100644 --- a/docs/explanation/features/V2_CONTROL_CENTER.md +++ b/docs/explanation/features/V2_CONTROL_CENTER.md @@ -8,7 +8,7 @@ The V2 Control Center is a permission-aware administration pane for managing Sim **Groups implemented in version:** 0.261.282 **Activity Logs implemented in version:** 0.261.284 **Public Workspaces implemented in version:** 0.261.283 -**Current version:** 0.261.291 (Dashboard, Users and Groups query fixes for the Python Cosmos SDK; Data health removed) +**Current version:** 0.261.292 (Dashboard, Users and Groups query fixes for the Python Cosmos SDK; Data health removed) **Dependencies:** React 18, TypeScript, Vite, Flask session authentication, and the existing Control Center APIs. @@ -16,7 +16,7 @@ The V2 Control Center is a permission-aware administration pane for managing Sim The Control Center is a distinct React route (`/control-center` and `/control-center/
`), reached from the account menu when the signed-in user has at least one Control Center capability. It is not a primary workspace-navigation item. The internal section rail has a separate per-user collapsed-state preference. -`get_control_center_capabilities()` in `functions_authentication.py` is the shared permission decision for both `control_center_required()` and the `/api/v2/bootstrap` response. When the ControlCenterAdmin role requirement is enabled, that role grants all capabilities; otherwise the regular Admin role grants them. The optional ControlCenterDashboardReader role grants dashboard viewing only when its setting is enabled. The bootstrap payload exposes `can_view_dashboard`, `can_manage_users`, `can_manage_groups`, `can_manage_workspaces`, `can_view_activity_logs`, and `can_run_maintenance`. Since Data health was removed in 0.261.291, no V2 section uses `can_run_maintenance`; it remains part of the shared capability contract. +`get_control_center_capabilities()` in `functions_authentication.py` is the shared permission decision for both `control_center_required()` and the `/api/v2/bootstrap` response. When the ControlCenterAdmin role requirement is enabled, that role grants all capabilities; otherwise the regular Admin role grants them. The optional ControlCenterDashboardReader role grants dashboard viewing only when its setting is enabled. The bootstrap payload exposes `can_view_dashboard`, `can_manage_users`, `can_manage_groups`, `can_manage_workspaces`, `can_view_activity_logs`, and `can_run_maintenance`. Since Data health was removed in 0.261.292, no V2 section uses `can_run_maintenance`; it remains part of the shared capability contract. ## Dashboard @@ -180,4 +180,4 @@ Functional checks cover dashboard status normalization, period deltas, cache exp ## Version tracking -The application version is defined by `VERSION` in `application/single_app/config.py`. The foundation was added in **0.261.278**, the dashboard in **0.261.279**, user management in **0.261.280**, group management in **0.261.282**, public workspace management in **0.261.283**, and Activity Logs in **0.261.284**. In **0.261.291**, the Dashboard, Users and Groups queries were made compatible with the Python Cosmos SDK. The same release removed the Data health section and its `GET /api/admin/control-center/migrate/status` and `POST /api/admin/control-center/migrate/all` backfill APIs; the classic Control Center had already stopped using them. +The application version is defined by `VERSION` in `application/single_app/config.py`. The foundation was added in **0.261.278**, the dashboard in **0.261.279**, user management in **0.261.280**, group management in **0.261.282**, public workspace management in **0.261.283**, and Activity Logs in **0.261.284**. In **0.261.292**, the Dashboard, Users and Groups queries were made compatible with the Python Cosmos SDK. The same release removed the Data health section and its `GET /api/admin/control-center/migrate/status` and `POST /api/admin/control-center/migrate/all` backfill APIs; the classic Control Center had already stopped using them. diff --git a/docs/explanation/fixes/ACTIVITY_LOG_MIGRATION_PROMPT_FIX.md b/docs/explanation/fixes/ACTIVITY_LOG_MIGRATION_PROMPT_FIX.md index e0c0426d3..5307cc57a 100644 --- a/docs/explanation/fixes/ACTIVITY_LOG_MIGRATION_PROMPT_FIX.md +++ b/docs/explanation/fixes/ACTIVITY_LOG_MIGRATION_PROMPT_FIX.md @@ -25,6 +25,6 @@ Implemented in version **0.261.278** (`application/single_app/config.py`): Opening the classic Control Center no longer triggers four cross-partition count scans or a misleading migration prompt. Administrators retain an explicit diagnostic and recovery path in V2. Existing logs are preserved; the manual backfill skips records that already have matching creation events. -## Update in 0.261.291 +## Update in 0.261.292 The V2 Data health section was removed, and with it the only callers of `GET /api/admin/control-center/migrate/status` and `POST /api/admin/control-center/migrate/all`. Both APIs and the backfill-only `has_activity_log_for_resource` helper were deleted, so neither Control Center offers the backfill any longer. Normal application workflows continue to write activity records, and `build_activity_log_id` still gives idempotent writers stable record IDs. `ui_tests/test_v2_control_center_data_health.py` was replaced by `ui_tests/test_v2_control_center_data_health_removed.py`, and `functional_tests/test_v2_control_center_foundation.py` now checks that the section and APIs stay removed. See [V2 Control Center Cosmos Query Compatibility Fix](V2_CONTROL_CENTER_COSMOS_QUERY_COMPATIBILITY_FIX.md). diff --git a/docs/explanation/fixes/V2_CONTROL_CENTER_COSMOS_QUERY_COMPATIBILITY_FIX.md b/docs/explanation/fixes/V2_CONTROL_CENTER_COSMOS_QUERY_COMPATIBILITY_FIX.md index 72dfa44bf..1a0f4483b 100644 --- a/docs/explanation/fixes/V2_CONTROL_CENTER_COSMOS_QUERY_COMPATIBILITY_FIX.md +++ b/docs/explanation/fixes/V2_CONTROL_CENTER_COSMOS_QUERY_COMPATIBILITY_FIX.md @@ -1,6 +1,6 @@ # V2 Control Center Cosmos Query Compatibility Fix -**Fixed in version:** 0.261.291 +**Fixed in version:** 0.261.292 ## Issue @@ -54,7 +54,7 @@ The response contracts are unchanged. Each unsupported query is replaced with a - `application/single_app/route_backend_control_center.py`: dashboard summary and insights helpers, and Users paging and export helpers. - `application/single_app/functions_control_center_groups.py`: `load_group_inventory`. -- `application/single_app/config.py`: version 0.261.291. +- `application/single_app/config.py`: version 0.261.292. - `functional_tests/test_support/cosmos_query_guard.py` (new): rejects `GROUP BY`, `COUNT` over `DISTINCT` values, and multi-property `ORDER BY` without a declared composite index. - `functional_tests/test_v2_control_center_cosmos_query_compatibility.py` (new): scans every SQL string in the V2 Control Center code paths through the guard. The activity-log `ORDER BY` passes only because `config.py` declares its composite index. - `functional_tests/test_v2_control_center_dashboard.py`, `functional_tests/test_v2_control_center_users.py` and `functional_tests/test_v2_control_center_groups.py`: the fakes now pass every query through the guard. The tests run the real routes and aggregation helpers. diff --git a/functional_tests/test_v2_control_center_cosmos_query_compatibility.py b/functional_tests/test_v2_control_center_cosmos_query_compatibility.py index 45ccde3dc..11ca849fd 100644 --- a/functional_tests/test_v2_control_center_cosmos_query_compatibility.py +++ b/functional_tests/test_v2_control_center_cosmos_query_compatibility.py @@ -2,8 +2,8 @@ # test_v2_control_center_cosmos_query_compatibility.py """ Functional test for V2 Control Center Cosmos query compatibility. -Version: 0.261.291 -Implemented in: 0.261.291 +Version: 0.261.292 +Implemented in: 0.261.292 The Dashboard, Users and Groups sections returned HTTP 500 because Cosmos DB rejected their queries with HTTP 400. The azure-cosmos Python SDK cannot run cross-partition @@ -151,7 +151,7 @@ def test_scan_covers_the_previously_failing_sections(): def test_version_is_at_least_the_implementation_version(): - assert_app_version_at_least("0.261.291") + assert_app_version_at_least("0.261.292") TESTS = [ diff --git a/functional_tests/test_v2_control_center_dashboard.py b/functional_tests/test_v2_control_center_dashboard.py index 90d2af3c6..2fa8a27cd 100644 --- a/functional_tests/test_v2_control_center_dashboard.py +++ b/functional_tests/test_v2_control_center_dashboard.py @@ -2,12 +2,12 @@ # test_v2_control_center_dashboard.py """ Functional test for the V2 Control Center dashboard. -Version: 0.261.291 +Version: 0.261.292 Implemented in: 0.261.280 This test validates status aggregation, period comparisons, bounded cache behavior, dashboard-reader authorization, and the summary and insights route contracts. -Since 0.261.291 every dashboard query also passes the Cosmos query guard, because the +Since 0.261.292 every dashboard query also passes the Cosmos query guard, because the Python Cosmos SDK cannot run cross-partition GROUP BY or COUNT over DISTINCT values. """ @@ -543,7 +543,7 @@ def test_invalid_period_errors_do_not_expose_exception_text(): def test_version_is_at_least_implementation_version(): - assert_app_version_at_least("0.261.291") + assert_app_version_at_least("0.261.292") TESTS = [ diff --git a/functional_tests/test_v2_control_center_foundation.py b/functional_tests/test_v2_control_center_foundation.py index 09e8c88d8..203abf32c 100644 --- a/functional_tests/test_v2_control_center_foundation.py +++ b/functional_tests/test_v2_control_center_foundation.py @@ -2,12 +2,12 @@ # test_v2_control_center_foundation.py """ Functional test for the V2 Control Center foundation. -Version: 0.261.291 +Version: 0.261.292 Implemented in: 0.261.278 This test covers the shared access capability contract, bootstrap wiring, stable activity-log IDs, removal of the legacy automatic migration check, removal of the -V2 Data health section and its backfill APIs (0.261.291), and route/pane structure. +V2 Data health section and its backfill APIs (0.261.292), and route/pane structure. """ import ast @@ -155,7 +155,7 @@ def test_pane_has_section_routes_and_capability_gating(): def test_version_is_at_least_the_implementing_release(): - assert_app_version_at_least("0.261.291") + assert_app_version_at_least("0.261.292") TESTS = [ diff --git a/functional_tests/test_v2_control_center_groups.py b/functional_tests/test_v2_control_center_groups.py index fae415fbd..4df9502eb 100644 --- a/functional_tests/test_v2_control_center_groups.py +++ b/functional_tests/test_v2_control_center_groups.py @@ -1,13 +1,13 @@ # test_v2_control_center_groups.py """ Functional tests for V2 Control Center Groups. -Version: 0.261.291 +Version: 0.261.292 Implemented in: 0.261.282 Run real filters and routes over isolated Cosmos services and the real guarded group writer. Cover selection caps before writes, audit parity, detail projections, admin-only access, snapshot expiry, safe exports and approval-only actions. -Since 0.261.291 the inventory fakes reject GROUP BY, because the Python Cosmos SDK +Since 0.261.292 the inventory fakes reject GROUP BY, because the Python Cosmos SDK cannot run it across partitions; the inventory aggregates streamed projections. """ @@ -389,7 +389,7 @@ def test_routes_have_explicit_admin_and_swagger_decorators(): decorators = [ast.unparse(decorator) for decorator in node.decorator_list] assert "login_required" in decorators and "control_center_required('admin')" in decorators assert "swagger_route(security=get_auth_security())" in decorators - assert_app_version_at_least("0.261.291") + assert_app_version_at_least("0.261.292") if __name__ == "__main__": diff --git a/functional_tests/test_v2_control_center_users.py b/functional_tests/test_v2_control_center_users.py index 624295059..55c4eafb1 100644 --- a/functional_tests/test_v2_control_center_users.py +++ b/functional_tests/test_v2_control_center_users.py @@ -2,12 +2,12 @@ # test_v2_control_center_users.py """ Functional test for V2 Control Center user management. -Version: 0.261.291 +Version: 0.261.292 Implemented in: 0.261.280 This test validates Users filtering, sorting, paging, detail data boundaries, bulk-action limits, admin-only authorization, cached metric freshness and CSV safety. -Since 0.261.291 the list and export order one property per query, because the +Since 0.261.292 the list and export order one property per query, because the user_settings container has no composite index for a two-property ORDER BY. """ @@ -418,7 +418,7 @@ def test_routes_require_full_control_center_admin_not_dashboard_reader(): def test_version_is_at_least_the_implementation_version(): - assert_app_version_at_least("0.261.291") + assert_app_version_at_least("0.261.292") TESTS = [ diff --git a/ui_tests/test_v2_control_center_data_health_removed.py b/ui_tests/test_v2_control_center_data_health_removed.py index 16081b7e4..9bd6606f2 100644 --- a/ui_tests/test_v2_control_center_data_health_removed.py +++ b/ui_tests/test_v2_control_center_data_health_removed.py @@ -2,8 +2,8 @@ # test_v2_control_center_data_health_removed.py """ Browser coverage for the V2 Control Center section rail after Data health was removed. -Version: 0.261.291 -Implemented in: 0.261.291 +Version: 0.261.292 +Implemented in: 0.261.292 Validates that the rail offers no Data health section even to maintenance-capable administrators, that an old data-health bookmark opens the Dashboard without calling the @@ -103,7 +103,7 @@ def _route(self, route: Route): route.fulfill(status=404, body="Fixture asset not found") elif path == "/api/v2/bootstrap" and request.method == "GET": route.fulfill(json={ - "version": "0.261.291", + "version": "0.261.292", "user": {"id": "test-admin", "display_name": "Test Admin", "is_admin": True, "roles": ["Admin"]}, "branding": {"app_title": "SimpleChat", "show_logo": False, "hide_app_title": False}, "features": {},