From 8ab37eac4a79da6d69f1dc4dce07fb6b62087331 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Mon, 17 Aug 2026 23:12:18 -0700 Subject: [PATCH 1/5] various updates --- docs/desktop-portable.md | 37 -- py-src/data_formulator/data_connector.py | 11 +- .../data_loader/databricks_data_loader.py | 3 + .../data_loader/external_data_loader.py | 62 +++ .../data_loader/s3_data_loader.py | 2 + .../data_loader/sample_datasets_loader.py | 2 + .../datalake/azure_blob_workspace_manager.py | 24 +- .../datalake/workspace_manager.py | 66 +++- py-src/data_formulator/routes/sessions.py | 1 + src/app/App.tsx | 7 +- src/app/dfSlice.tsx | 9 +- src/app/useAutoSave.tsx | 65 +-- src/app/workspaceService.ts | 1 + src/components/ComponentType.tsx | 2 + src/i18n/locales/en/common.json | 10 + src/i18n/locales/en/messages.json | 9 +- src/i18n/locales/en/upload.json | 7 +- src/i18n/locales/zh/common.json | 10 + src/i18n/locales/zh/messages.json | 3 +- src/i18n/locales/zh/upload.json | 5 +- src/views/DataFormulator.tsx | 45 ++- src/views/DataLoadingChat.tsx | 53 ++- src/views/DataSourceSidebar.tsx | 175 +++++++-- src/views/MessageSnackbar.tsx | 371 ++++++++++++------ src/views/UnifiedDataUploadDialog.tsx | 104 +++-- src/views/dataLoadingSuggestions.ts | 4 +- tests/backend/data/test_workspace_manager.py | 46 +++ .../routes/test_session_routes_migration.py | 21 + tests/frontend/unit/app/useAutoSave.test.tsx | 49 ++- .../unit/views/DataSourceSidebar.test.tsx | 49 ++- 30 files changed, 915 insertions(+), 338 deletions(-) delete mode 100644 docs/desktop-portable.md diff --git a/docs/desktop-portable.md b/docs/desktop-portable.md deleted file mode 100644 index 473660530..000000000 --- a/docs/desktop-portable.md +++ /dev/null @@ -1,37 +0,0 @@ -# Portable desktop build - -Data Formulator's desktop bundle runs the existing Flask application on a -random loopback port and displays it in a native pywebview window. It is built -as a PyInstaller `onedir` bundle so users can unzip it and launch it without -installing Python or Node.js. - -## Build - -Build on each target operating system; PyInstaller does not cross-compile. - -```bash -yarn install --frozen-lockfile -yarn build # frontend -> py-src/data_formulator/dist -uv sync --extra desktop -uv run pyinstaller --noconfirm --clean packaging/data_formulator_desktop.spec -``` - -On Windows and Linux, the output is `dist/Data Formulator/`; distribute the -complete directory as a zip archive. On macOS, distribute -`dist/Data Formulator.app`. Code signing and macOS notarization should be added -before a public release. - -## Azure CLI authentication - -Kusto and other Entra-enabled connectors reuse the user's Azure CLI identity. -The desktop app does not request delegated `user_impersonation` permission for -its own app registration. - -Azure CLI remains an external prerequisite for Azure connections. Users can -sign in from Data Formulator's connector UI; the backend runs `az login` and -then Azure Identity obtains tokens from the CLI cache. Other features remain -usable when Azure CLI is absent. - -The launcher adds common Azure CLI install locations to `PATH`, including -Homebrew locations that are normally missing when a macOS app is opened from -Finder. \ No newline at end of file diff --git a/py-src/data_formulator/data_connector.py b/py-src/data_formulator/data_connector.py index 3fb3f837e..49fd208d0 100644 --- a/py-src/data_formulator/data_connector.py +++ b/py-src/data_formulator/data_connector.py @@ -466,6 +466,7 @@ def get_frontend_config(self, include_pinned_in_form: bool = False) -> dict[str, "icon": self._icon, "params_form": form_fields, "pinned_params": pinned_params, + "connection_identity": self._loader_class.connection_identity(self._default_params), "hierarchy": _hierarchy_dicts(full_hierarchy), "effective_hierarchy": _hierarchy_dicts(effective), "auth_instructions": self._loader_class.auth_instructions(), @@ -1314,6 +1315,7 @@ def list_connectors(): "sso_auto_connect": sso_auto, "params_form": cfg["params_form"], "pinned_params": cfg["pinned_params"], + "connection_identity": cfg["connection_identity"], "hierarchy": cfg["hierarchy"], "effective_hierarchy": cfg["effective_hierarchy"], "auth_mode": cfg["auth_mode"], @@ -1351,11 +1353,18 @@ def create_connector(): if not loader_class: raise AppError(ErrorCode.INVALID_REQUEST, f"Unknown loader type: {loader_type}") - display_name = data.get("display_name", loader_type.replace("_", " ").title()) + display_name = data.get("display_name") icon = data.get("icon", loader_type) raw_params = data.get("params", {}) default_params = _connector_config_params(loader_class, raw_params) + if not display_name: + # A connector is its type plus which instance it points at, so name it + # that way unless the user said otherwise. + type_name = loader_class.DISPLAY_NAME or loader_type.replace("_", " ").title() + identity = loader_class.connection_identity(default_params) + display_name = f"{type_name} · {identity}" if identity else type_name + try: identity = DataConnector._get_identity() except Exception as e: diff --git a/py-src/data_formulator/data_loader/databricks_data_loader.py b/py-src/data_formulator/data_loader/databricks_data_loader.py index 99490041d..7cf6aa7bd 100644 --- a/py-src/data_formulator/data_loader/databricks_data_loader.py +++ b/py-src/data_formulator/data_loader/databricks_data_loader.py @@ -43,6 +43,9 @@ class DatabricksDataLoader(ExternalDataLoader): DISPLAY_NAME = "Databricks" DESCRIPTION = "Query Databricks Unity Catalog tables through a SQL warehouse." + # http_path routes to a warehouse; the workspace host is what names the instance. + IDENTITY_PARAMS = ("server_hostname",) + @staticmethod def list_params() -> list[dict[str, Any]]: return [ diff --git a/py-src/data_formulator/data_loader/external_data_loader.py b/py-src/data_formulator/data_loader/external_data_loader.py index 96b79f8c3..4cef27673 100644 --- a/py-src/data_formulator/data_loader/external_data_loader.py +++ b/py-src/data_formulator/data_loader/external_data_loader.py @@ -33,6 +33,31 @@ def apply_import_projection( logger = logging.getLogger(__name__) +def _concise_identity(value: str) -> str: + """Reduce a connection param to the part a human recognises. + + URLs collapse to their host (``https://x.kusto.windows.net/`` -> ``x.kusto.windows.net``) + and home directories to ``~`` so identities stay short and screenshot-safe. + """ + trimmed = value.strip().rstrip("/\\") + if not trimmed: + return "" + if "://" in trimmed: + from urllib.parse import urlparse + host = urlparse(trimmed).netloc + if host: + return host + if trimmed.startswith(("/", "~")) or (len(trimmed) > 2 and trimmed[1] == ":"): + from pathlib import Path + try: + home = str(Path.home()) + if trimmed.startswith(home): + return "~" + trimmed[len(home):] + except Exception: + pass + return trimmed + + @dataclass(frozen=True) class CatalogCachePolicy: listing_ttl_seconds: int | None = 21_600 @@ -709,6 +734,43 @@ def auth_instructions(cls) -> str: #: back to ``DISPLAY_NAME``. This is NOT the verbose ``auth_instructions``. DESCRIPTION: str | None = None + #: Params naming *which* instance of this source a connector points at + #: (cluster, host, bucket…), most significant first. When ``None`` the + #: identity is derived from the required, non-advanced connection params, + #: which is right for most loaders; override where that picks up routing + #: detail rather than identity (Databricks' ``http_path``, S3's region). + IDENTITY_PARAMS: tuple[str, ...] | None = None + + @classmethod + def identity_params(cls) -> list[str]: + """Return the param names that identify this connector's instance.""" + if cls.IDENTITY_PARAMS is not None: + return list(cls.IDENTITY_PARAMS) + return [ + p["name"] for p in cls.list_params() + if p.get("tier") == "connection" + and p.get("required") + and not p.get("advanced") + and not p.get("sensitive") + ][:2] + + @classmethod + def connection_identity(cls, params: dict[str, Any]) -> str: + """Render the connection's identity, e.g. ``"mycluster.kusto.windows.net · sales"``. + + Returns an empty string when no identifying param has a value, which + is the normal case for loaders that take no connection params at all. + """ + parts: list[str] = [] + for name in cls.identity_params(): + value = params.get(name) + if value is None: + continue + concise = _concise_identity(str(value)) + if concise and concise not in parts: + parts.append(concise) + return " · ".join(parts) + @staticmethod def delegated_login_config() -> dict[str, Any] | None: """Return config for delegated (popup-based) token login, or None. diff --git a/py-src/data_formulator/data_loader/s3_data_loader.py b/py-src/data_formulator/data_loader/s3_data_loader.py index dd8403a0a..1f4c61bb5 100644 --- a/py-src/data_formulator/data_loader/s3_data_loader.py +++ b/py-src/data_formulator/data_loader/s3_data_loader.py @@ -20,6 +20,8 @@ class S3DataLoader(ExternalDataLoader): DISPLAY_NAME = "Amazon S3" DESCRIPTION = "Load CSV, JSON, or Parquet files from an Amazon S3 bucket." + IDENTITY_PARAMS = ("bucket",) + @staticmethod def list_params() -> list[dict[str, Any]]: params_list = [ diff --git a/py-src/data_formulator/data_loader/sample_datasets_loader.py b/py-src/data_formulator/data_loader/sample_datasets_loader.py index 70172fafb..59678f60a 100644 --- a/py-src/data_formulator/data_loader/sample_datasets_loader.py +++ b/py-src/data_formulator/data_loader/sample_datasets_loader.py @@ -47,6 +47,8 @@ class SampleDatasetsLoader(ExternalDataLoader): """Browse and import the built-in sample datasets.""" + DISPLAY_NAME = "Sample Datasets" + # ------------------------------------------------------------------ # Metadata # ------------------------------------------------------------------ diff --git a/py-src/data_formulator/datalake/azure_blob_workspace_manager.py b/py-src/data_formulator/datalake/azure_blob_workspace_manager.py index cd7801347..19d3b3b5d 100644 --- a/py-src/data_formulator/datalake/azure_blob_workspace_manager.py +++ b/py-src/data_formulator/datalake/azure_blob_workspace_manager.py @@ -26,6 +26,7 @@ WorkspaceManager, SESSION_STATE_FILENAME, WORKSPACE_META_FILENAME, + _session_source_ids, _strip_sensitive, ) @@ -121,6 +122,7 @@ def _upload_meta( *, table_count: Optional[int] = None, chart_count: Optional[int] = None, + source_ids: Optional[list[str]] = None, ) -> None: """Upload a lightweight ``workspace_meta.json`` blob for fast listing. @@ -133,6 +135,7 @@ def _upload_meta( # Preserve createdAt if the meta blob already exists. created_at = now_iso + existing: dict = {} if self._blob_exists(blob_name): try: existing = json.loads(self._download_blob(blob_name)) @@ -152,8 +155,16 @@ def _upload_meta( } if table_count is not None: meta["tableCount"] = table_count + elif existing.get("tableCount") is not None: + meta["tableCount"] = existing["tableCount"] if chart_count is not None: meta["chartCount"] = chart_count + elif existing.get("chartCount") is not None: + meta["chartCount"] = existing["chartCount"] + if source_ids is not None: + meta["sourceIds"] = source_ids + elif isinstance(existing.get("sourceIds"), list): + meta["sourceIds"] = existing["sourceIds"] self._upload_blob(blob_name, json.dumps(meta, ensure_ascii=False)) def _ensure_meta(self, workspace_id: str) -> dict: @@ -209,6 +220,7 @@ def list_workspaces(self) -> list[dict]: "updated_at": meta.get("updatedAt"), "table_count": meta.get("tableCount"), "chart_count": meta.get("chartCount"), + "source_ids": meta.get("sourceIds", []), }) workspaces.sort(key=lambda w: w.get("updated_at") or "", reverse=True) @@ -331,11 +343,19 @@ def save_session_state(self, workspace_id: str, state: dict) -> None: aw = clean_state.get("activeWorkspace") dn = aw["displayName"] if isinstance(aw, dict) and aw.get("displayName") else workspace_id - tables = clean_state.get("tables") + tables = clean_state.get("inputTables") + if not isinstance(tables, list): + tables = clean_state.get("tables") tc = len(tables) if isinstance(tables, list) else None charts = clean_state.get("charts") cc = len(charts) if isinstance(charts, list) else None - self._upload_meta(workspace_id, dn, table_count=tc, chart_count=cc) + self._upload_meta( + workspace_id, + dn, + table_count=tc, + chart_count=cc, + source_ids=_session_source_ids(clean_state), + ) logger.debug(f"Saved session state to blob {blob_name}") diff --git a/py-src/data_formulator/datalake/workspace_manager.py b/py-src/data_formulator/datalake/workspace_manager.py index c5af79d12..99f2dfdbc 100644 --- a/py-src/data_formulator/datalake/workspace_manager.py +++ b/py-src/data_formulator/datalake/workspace_manager.py @@ -46,6 +46,49 @@ def _strip_sensitive(state: dict) -> dict: return {k: v for k, v in state.items() if k not in _SENSITIVE_FIELDS} +def _session_source_ids(state: dict) -> list[str]: + """Summarize input-table origins for lightweight session grouping.""" + tables = state.get("inputTables") + if not isinstance(tables, list): + tables = state.get("tables") + if not isinstance(tables, list): + return [] + + source_ids: set[str] = set() + for table in tables: + if not isinstance(table, dict): + continue + source = table.get("source") + source_config = table.get("sourceConfig") + + if isinstance(source, dict) and source.get("kind") == "connector": + connector_id = source.get("connectorId") or source.get("connector_id") + if isinstance(connector_id, str) and connector_id: + source_ids.add(connector_id) + continue + + config = source_config if isinstance(source_config, dict) else source + if not isinstance(config, dict): + continue + connector_id = ( + config.get("connectorId") + or config.get("connector_id") + or config.get("sourceId") + or config.get("source_id") + ) + if isinstance(connector_id, str) and connector_id: + source_ids.add(connector_id) + continue + + source_type = config.get("type") + if source_type == "example": + source_ids.add("sample_datasets") + elif source_type in {"file", "paste", "url", "stream", "extract"}: + source_ids.add("upload") + + return sorted(source_ids) + + class WorkspaceManager: """ Manages the set of workspaces for a single user. @@ -87,6 +130,7 @@ def _write_meta( *, table_count: Optional[int] = None, chart_count: Optional[int] = None, + source_ids: Optional[list[str]] = None, provisional: Optional[bool] = None, ) -> None: """Write a lightweight ``workspace_meta.json`` used by list_workspaces. @@ -101,6 +145,7 @@ def _write_meta( # Preserve createdAt if the meta file already exists. created_at = now_iso + existing: dict = {} if meta_file.exists(): try: existing = json.loads(meta_file.read_text(encoding="utf-8")) @@ -121,8 +166,16 @@ def _write_meta( } if table_count is not None: meta["tableCount"] = table_count + elif existing.get("tableCount") is not None: + meta["tableCount"] = existing["tableCount"] if chart_count is not None: meta["chartCount"] = chart_count + elif existing.get("chartCount") is not None: + meta["chartCount"] = existing["chartCount"] + if source_ids is not None: + meta["sourceIds"] = source_ids + elif isinstance(existing.get("sourceIds"), list): + meta["sourceIds"] = existing["sourceIds"] if provisional: meta["provisional"] = True meta_file.write_text( @@ -217,6 +270,7 @@ def list_workspaces(self) -> list[dict]: "updated_at": meta.get("updatedAt"), "table_count": tc, "chart_count": cc, + "source_ids": meta.get("sourceIds", []), }) workspaces.sort(key=lambda w: w.get("updated_at") or "", reverse=True) @@ -497,12 +551,20 @@ def save_session_state(self, workspace_id: str, state: dict) -> None: aw = clean_state.get("activeWorkspace") dn = aw["displayName"] if isinstance(aw, dict) and aw.get("displayName") else workspace_id - tables = clean_state.get("tables") + tables = clean_state.get("inputTables") + if not isinstance(tables, list): + tables = clean_state.get("tables") tc = len(tables) if isinstance(tables, list) else None charts = clean_state.get("charts") cc = len(charts) if isinstance(charts, list) else None # Saving state is the moment a session stops being provisional. - self._write_meta(workspace_id, dn, table_count=tc, chart_count=cc) + self._write_meta( + workspace_id, + dn, + table_count=tc, + chart_count=cc, + source_ids=_session_source_ids(clean_state), + ) logger.debug(f"Saved session state to {state_file}") diff --git a/py-src/data_formulator/routes/sessions.py b/py-src/data_formulator/routes/sessions.py index 626afd4e0..81efcabab 100644 --- a/py-src/data_formulator/routes/sessions.py +++ b/py-src/data_formulator/routes/sessions.py @@ -126,6 +126,7 @@ def list_sessions(): entry["table_count"] = w["table_count"] if w.get("chart_count") is not None: entry["chart_count"] = w["chart_count"] + entry["source_ids"] = w.get("source_ids", []) sessions.append(entry) return json_ok({"sessions": sessions}) diff --git a/src/app/App.tsx b/src/app/App.tsx index 881fde203..4c04c76f6 100644 --- a/src/app/App.tsx +++ b/src/app/App.tsx @@ -1129,7 +1129,7 @@ const AppShell: FC = () => { - + {isCompactToolbar ? ( @@ -1137,15 +1137,16 @@ const AppShell: FC = () => { <> diff --git a/src/app/dfSlice.tsx b/src/app/dfSlice.tsx index 5c919acc9..9837857b1 100644 --- a/src/app/dfSlice.tsx +++ b/src/app/dfSlice.tsx @@ -10,13 +10,13 @@ import { getChartTemplate, getChartChannels } from "../components/ChartTemplates import { vlAdaptChart, vlRecommendEncodings } from 'flint-chart'; import { migrateState } from './stateMigrations'; import { getDataTable } from '../views/ChartUtils'; -import { getTriggers, getUrls, computeContentHash } from './utils'; +import { getUrls, computeContentHash } from './utils'; import { apiRequest, ApiRequestError } from './apiClient'; import { deleteTablesFromWorkspace } from './workspaceService'; import i18n from '../i18n'; import { Type } from '../data/types'; -import { createTableFromFromObjectArray, inferTypeFromValueArray, refineTemporalType } from '../data/utils'; -import { Identity, IdentityType, getBrowserId } from './identity'; +import { inferTypeFromValueArray, refineTemporalType } from '../data/utils'; +import { Identity, getBrowserId } from './identity'; import { REHYDRATE } from 'redux-persist'; import { setInputTablePreview } from './inputTablePreviewCache'; import { materializeInputTablePreview, materializeTables } from './tableResolution'; @@ -89,6 +89,7 @@ export interface ServerConfig { icon: string; params_form: Array<{name: string; type: string; required: boolean; default?: string; options?: string[]; advanced?: boolean; description?: string; sensitive?: boolean; tier?: 'connection' | 'auth' | 'filter'}>; pinned_params: Record; + connection_identity?: string; hierarchy: Array<{key: string; label: string}>; effective_hierarchy: Array<{key: string; label: string}>; auth_instructions: string; @@ -2681,7 +2682,7 @@ export const dfSelectors = { seen.add(cur.id); const p: string | undefined = cur.parentNodeId; if (!p) break; - const parentTurn = textTurns.find(tt => tt.id === p); + const parentTurn: TextTurn | undefined = textTurns.find(tt => tt.id === p); if (parentTurn?.dataOperation || parentTurn?.form) { return { type: 'text', textId: parentTurn.id }; } diff --git a/src/app/useAutoSave.tsx b/src/app/useAutoSave.tsx index 0289bcb86..5142fa7c8 100644 --- a/src/app/useAutoSave.tsx +++ b/src/app/useAutoSave.tsx @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { useEffect, useRef } from 'react'; +import { useCallback, useEffect, useRef } from 'react'; import { useSelector } from 'react-redux'; import { DataFormulatorState, dfSelectors } from './dfSlice'; import { saveWorkspaceState } from './workspaceService'; @@ -65,6 +65,35 @@ export function useAutoSave() { const isSavingRef = useRef(false); const pendingRef = useRef(false); const lastErrorNotifyRef = useRef(0); + const latestStateRef = useRef(state); + latestStateRef.current = state; + + const saveLatestState = useCallback(async () => { + if (isSavingRef.current) { + pendingRef.current = true; + return; + } + + isSavingRef.current = true; + try { + do { + pendingRef.current = false; + try { + await saveWorkspaceState(getSerializableState(latestStateRef.current)); + } catch (err) { + const now = Date.now(); + if (now - lastErrorNotifyRef.current >= AUTO_SAVE_ERROR_NOTIFY_MS) { + lastErrorNotifyRef.current = now; + handleApiError(err, 'Auto-save'); + } else { + console.warn('[auto-save] failed:', err); + } + } + } while (pendingRef.current); + } finally { + isSavingRef.current = false; + } + }, []); useEffect(() => { // Nothing to save while a session is loading, read-only, workspace-less, @@ -79,36 +108,8 @@ export function useAutoSave() { clearTimeout(timerRef.current); } - timerRef.current = setTimeout(async () => { - // Skip if a save is already in flight - if (isSavingRef.current) { - pendingRef.current = true; - return; - } - - isSavingRef.current = true; - try { - const serializable = getSerializableState(state); - await saveWorkspaceState(serializable); - } catch (err) { - const now = Date.now(); - if (now - lastErrorNotifyRef.current >= AUTO_SAVE_ERROR_NOTIFY_MS) { - lastErrorNotifyRef.current = now; - handleApiError(err, 'Auto-save'); - } else { - console.warn('[auto-save] failed:', err); - } - } finally { - isSavingRef.current = false; - // If state changed while we were saving, trigger another save - if (pendingRef.current) { - pendingRef.current = false; - // Re-trigger by scheduling another timeout - timerRef.current = setTimeout(() => { - // This will be picked up by the next effect cycle - }, AUTO_SAVE_DEBOUNCE_MS); - } - } + timerRef.current = setTimeout(() => { + void saveLatestState(); }, AUTO_SAVE_DEBOUNCE_MS); return () => { @@ -116,5 +117,5 @@ export function useAutoSave() { clearTimeout(timerRef.current); } }; - }, [state]); + }, [saveLatestState, state]); } diff --git a/src/app/workspaceService.ts b/src/app/workspaceService.ts index d4c9fd4d4..5e4c0d2e3 100644 --- a/src/app/workspaceService.ts +++ b/src/app/workspaceService.ts @@ -23,6 +23,7 @@ export interface WorkspaceSummary { saved_at: string | null; table_count?: number | null; chart_count?: number | null; + source_ids?: string[]; read_only?: boolean; } diff --git a/src/components/ComponentType.tsx b/src/components/ComponentType.tsx index 598c5dac4..d6528f950 100644 --- a/src/components/ComponentType.tsx +++ b/src/components/ComponentType.tsx @@ -677,6 +677,8 @@ export interface ConnectorInstance { deletable?: boolean; params_form: Array<{name: string; type: string; required: boolean; default?: string | number | boolean; options?: string[]; advanced?: boolean; description?: string; sensitive?: boolean; tier?: 'connection' | 'auth' | 'filter'}>; pinned_params: Record; + /** Which instance this connector points at (cluster, host, bucket…), resolved by the loader. */ + connection_identity?: string; hierarchy: Array<{key: string; label: string}>; effective_hierarchy: Array<{key: string; label: string}>; auth_mode?: string; diff --git a/src/i18n/locales/en/common.json b/src/i18n/locales/en/common.json index 457334ffb..0761e6443 100644 --- a/src/i18n/locales/en/common.json +++ b/src/i18n/locales/en/common.json @@ -854,6 +854,7 @@ "configureConnector": "Edit connection", "linkLocalFolder": "Link local folder", "newSession": "New session", + "importSession": "Import session", "noSessions": "No saved sessions", "tableCount": "{{count}} table(s)", "chartCount": "{{count}} chart(s)", @@ -921,6 +922,15 @@ "sortRecentlyModifiedFirst": "recently modified", "sortNameAsc": "name (a–z)", "sortSessions": "Sort sessions", + "organizeSessions": "Group and sort sessions", + "groupSessions": "Group", + "groupBySource": "Data source", + "groupSourceShort": "Source", + "noGrouping": "No grouping", + "sourceUpload": "Upload", + "sourceExampleDatasets": "Example datasets", + "sourceNoData": "No data", + "sourceOther": "Other", "runCatalogSearch": "Search", "clearCatalogSearch": "Clear search", "timeJustNow": "just now", diff --git a/src/i18n/locales/en/messages.json b/src/i18n/locales/en/messages.json index d5f894050..fa212ef41 100644 --- a/src/i18n/locales/en/messages.json +++ b/src/i18n/locales/en/messages.json @@ -22,10 +22,11 @@ "changesDiscarded": "Changes discarded", "formulate": "Formulate", "formulateAndOverride": "Formulate and override", - "viewSystemMessages": "view system messages", - "systemMessagesWithCount": "system messages ({{count}})", - "clearAllMessages": "clear all messages", - "details": "[details]", + "viewSystemMessages": "View system messages", + "systemMessagesWithCount": "System messages ({{count}})", + "showingLatest": "Showing the latest {{count}}", + "clearAllMessages": "Clear all messages", + "details": "Details", "generatedCode": "[generated code]", "chatWithAgents": "Dialog with Agents", "you": "You", diff --git a/src/i18n/locales/en/upload.json b/src/i18n/locales/en/upload.json index 10323a3d1..3aad0fe29 100644 --- a/src/i18n/locales/en/upload.json +++ b/src/i18n/locales/en/upload.json @@ -43,10 +43,10 @@ "agentChatSuggestionsLabel": "Try asking", "agentChatSendTooltip": "Start chatting with the agent", "dataSourcesLabel": "Connected to:", - "addSourceLabel": "Or add data directly:", + "addSourceLabel": "Add data:", "agentChatQuickAction": { - "connect": "Help me connect to my data source", - "askConnected": "What data do we have from connected sources?" + "connect": "Help me connect my data source", + "askConnected": "What data are available from my sources?" }, "agentChatSuggestion": { "askConnected": "What datasets do we have from connected sources?", @@ -69,6 +69,7 @@ "addConnectionDesc": "Connect to a live database", "connectorConnected": "Connected", "connectorDisconnected": "Click to connect", + "connectorNotConnected": "Not connected", "pickDataSourceType": "Choose a data source type to create a new connection.", "nameYourConnection": "Name your {{type}} connection.", "connectionName": "Connection name", diff --git a/src/i18n/locales/zh/common.json b/src/i18n/locales/zh/common.json index 1d20a31d0..4fd743a6c 100644 --- a/src/i18n/locales/zh/common.json +++ b/src/i18n/locales/zh/common.json @@ -854,6 +854,7 @@ "configureConnector": "编辑连接", "linkLocalFolder": "链接本地文件夹", "newSession": "新建会话", + "importSession": "导入会话", "noSessions": "暂无已保存的会话", "tableCount": "{{count}} 个表格", "chartCount": "{{count}} 个图表", @@ -921,6 +922,15 @@ "sortRecentlyModifiedFirst": "最近修改优先", "sortNameAsc": "名称 (a–z)", "sortSessions": "排序会话", + "organizeSessions": "分组和排序会话", + "groupSessions": "分组", + "groupBySource": "数据源", + "groupSourceShort": "数据源", + "noGrouping": "不分组", + "sourceUpload": "上传", + "sourceExampleDatasets": "示例数据集", + "sourceNoData": "无数据", + "sourceOther": "其他", "runCatalogSearch": "搜索", "clearCatalogSearch": "清除搜索", "timeJustNow": "刚刚", diff --git a/src/i18n/locales/zh/messages.json b/src/i18n/locales/zh/messages.json index 59185a446..fafcd7a73 100644 --- a/src/i18n/locales/zh/messages.json +++ b/src/i18n/locales/zh/messages.json @@ -24,8 +24,9 @@ "formulateAndOverride": "生成并覆盖", "viewSystemMessages": "查看系统消息", "systemMessagesWithCount": "系统消息（{{count}}）", + "showingLatest": "显示最近 {{count}} 条", "clearAllMessages": "清空全部消息", - "details": "[详情]", + "details": "详情", "generatedCode": "[生成代码]", "chatWithAgents": "与 Agent 对话", "you": "你", diff --git a/src/i18n/locales/zh/upload.json b/src/i18n/locales/zh/upload.json index 124ca5874..86140fdc4 100644 --- a/src/i18n/locales/zh/upload.json +++ b/src/i18n/locales/zh/upload.json @@ -43,10 +43,10 @@ "agentChatSuggestionsLabel": "试试这样问", "agentChatSendTooltip": "开始与助手对话", "dataSourcesLabel": "已连接：", - "addSourceLabel": "或直接添加数据：", + "addSourceLabel": "添加数据：", "agentChatQuickAction": { "connect": "帮我连接数据源", - "askConnected": "已连接的数据源里有哪些数据？" + "askConnected": "我的数据源中有哪些可用数据？" }, "agentChatSuggestion": { "askConnected": "已连接的数据源里有哪些数据集？", @@ -69,6 +69,7 @@ "addConnectionDesc": "连接到实时数据库", "connectorConnected": "已连接", "connectorDisconnected": "点击连接", + "connectorNotConnected": "未连接", "pickDataSourceType": "选择数据源类型以创建新连接。", "nameYourConnection": "为您的 {{type}} 连接命名。", "connectionName": "连接名称", diff --git a/src/views/DataFormulator.tsx b/src/views/DataFormulator.tsx index cc3cb9d0e..fcb8415bb 100644 --- a/src/views/DataFormulator.tsx +++ b/src/views/DataFormulator.tsx @@ -797,28 +797,39 @@ export const DataFormulatorFC = ({ }) => { {/* Hero — fills the viewport so title + input own the first screen; Demos/Sessions live below the fold and just peek up. */} - - + + + + {toolName} + + + - {toolName} + {t('landing.tagline')} - - {t('landing.tagline')} - {/* Hosted-demo notice — borderless strip (it's prose, not a button) placed before the Import Data section. The rocket @@ -906,7 +917,7 @@ export const DataFormulatorFC = ({ }) => { )} - + openUploadDialog(tab)} onSelectConnector={(conn) => { @@ -932,7 +943,7 @@ export const DataFormulatorFC = ({ }) => { demo, since first-time visitors won't have any sessions yet and demos are the most engaging entry point. */} - + {t('landing.demos')} { {/* ── Saved workspaces section ──────────────────────────── */} - {/* Section header — left-aligned label with the sort control - on the right, aligned to the card grid. */} - + {t('workspace.yourSessions')} - )[sessionSort]}`} placement="bottom"> - + + + + + + + + {t('sidebar.groupSessions', { defaultValue: 'Group' })} + + {([ + ['source', t('sidebar.groupBySource', { defaultValue: 'Data source' })], + ['none', t('sidebar.noGrouping', { defaultValue: 'No grouping' })], + ] as [SessionGroupKey, string][]).map(([key, label]) => ( + { + setSessionGroup(key); + setSessionSortAnchor(null); + }} + sx={{ fontSize: textVar.sm, py: 0.75 }} + > + + {sessionGroup === key && } + + + + ))} + + + {t('sidebar.sortSessions', { defaultValue: 'Sort' })} + {([ - ['updated_desc', t('sidebar.sortRecentlyModifiedFirst')], ['created_desc', t('sidebar.sortNewestFirst')], ['created_asc', t('sidebar.sortOldestFirst')], + ['updated_desc', t('sidebar.sortRecentlyModifiedFirst')], ['name_asc', t('sidebar.sortNameAsc')], ] as [SessionSortKey, string][]).map(([key, label]) => ( ))} - {pinAction} - - - - - {sessions.length === 0 ? ( @@ -2273,7 +2355,24 @@ const DataSourceSidebarPanel: React.FC<{ ) : ( - sortedSessions.map((s) => { + sessionSections.map((section, sectionIndex) => ( + + {sessionGroup === 'source' && ( + + + {section.label} + + + + )} + {section.sessions.map((s) => { const isRenaming = renamingSession === s.id; return ( ); - }) + })} + + )) )} diff --git a/src/views/MessageSnackbar.tsx b/src/views/MessageSnackbar.tsx index 48d6bb9e4..43b4d0797 100644 --- a/src/views/MessageSnackbar.tsx +++ b/src/views/MessageSnackbar.tsx @@ -7,14 +7,20 @@ import IconButton from '@mui/material/IconButton'; import CloseIcon from '@mui/icons-material/Close'; import { DataFormulatorState, dfActions } from '../app/dfSlice'; import { useDispatch, useSelector } from 'react-redux'; -import { Alert, Box, Paper, Tooltip, Typography } from '@mui/material'; +import { Alert, Box, Button, Paper, Tooltip, Typography, alpha, useTheme } from '@mui/material'; import InfoIcon from '@mui/icons-material/Info'; -import DeleteIcon from '@mui/icons-material/Delete'; +import InfoOutlinedIcon from '@mui/icons-material/InfoOutlined'; +import DeleteOutlineIcon from '@mui/icons-material/DeleteOutline'; import CheckCircleIcon from '@mui/icons-material/CheckCircle'; +import CheckCircleOutlineIcon from '@mui/icons-material/CheckCircleOutline'; import ErrorOutlineIcon from '@mui/icons-material/ErrorOutline'; +import WarningAmberOutlinedIcon from '@mui/icons-material/WarningAmberOutlined'; import ContentCopyIcon from '@mui/icons-material/ContentCopy'; +import ChevronRightIcon from '@mui/icons-material/ChevronRight'; +import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; import { useTranslation } from 'react-i18next'; import { iconVar, textVar } from '../app/layout'; +import { borderColor, radius, shadow } from '../app/tokens'; export interface Message { type: "success" | "info" | "error" | "warning", @@ -26,18 +32,11 @@ export interface Message { diagnostics?: any, // full diagnostic payload from the backend agent pipeline } -const TYPE_SYMBOLS: Record = { - error: '✗', - warning: '⚠', - info: 'ℹ', - success: '✓', -}; - -const TYPE_COLORS: Record = { - error: '#d32f2f', - warning: '#ed6c02', - info: '#0288d1', - success: '#2e7d32', +const SeverityIcon: React.FC<{ type: Message['type'] }> = ({ type }) => { + if (type === 'error') return ; + if (type === 'warning') return ; + if (type === 'success') return ; + return ; }; // Helper function to format timestamp @@ -51,6 +50,7 @@ const formatTimestamp = (timestamp: number) => { }; const DiagnosticsViewer: React.FC<{ diagnostics: any }> = React.memo(({ diagnostics }) => { + const theme = useTheme(); const [expanded, setExpanded] = React.useState(false); const [copied, setCopied] = React.useState(false); const jsonStr = React.useMemo(() => JSON.stringify(diagnostics, null, 2), [diagnostics]); @@ -63,40 +63,48 @@ const DiagnosticsViewer: React.FC<{ diagnostics: any }> = React.memo(({ diagnost }, [jsonStr]); return ( -
- - + + {expanded && ( - - + + )} - + {expanded && ( - 
                     {jsonStr}
-
+ )} -
+ ); }); @@ -107,6 +115,7 @@ export const MessageSnackbar = React.memo(function MessageSnackbar() { const dispatch = useDispatch(); const { t } = useTranslation(); + const theme = useTheme(); const [openLastMessage, setOpenLastMessage] = React.useState(false); const [latestMessage, setLatestMessage] = React.useState(); @@ -184,170 +193,284 @@ export const MessageSnackbar = React.memo(function MessageSnackbar() { return ( - setOpenMessages(true)} + aria-label={t('messages.viewSystemMessages')} + onClick={() => { + setOpenLastMessage(false); + setOpenMessages(open => !open); + }} > - {buttonSeverity === "error" ? : - buttonSeverity === "warning" ? : - buttonSeverity === "success" ? : - } + {buttonSeverity === 'error' ? : + buttonSeverity === 'warning' ? : + buttonSeverity === 'success' ? : + } - {/* Header */} - - - {t('messages.systemMessagesWithCount', { count: messages.length })}{messages.length > MAX_DISPLAY_MESSAGES ? ` — showing latest ${MAX_DISPLAY_MESSAGES}` : ''} - + + + + {t('messages.systemMessagesWithCount', { count: messages.length })} + + {messages.length > MAX_DISPLAY_MESSAGES && ( + + {t('messages.showingLatest', { + count: MAX_DISPLAY_MESSAGES, + defaultValue: 'Showing the latest {{count}}', + })} + + )} + { dispatch(dfActions.clearMessages()); dispatch(dfActions.setDisplayedMessageIndex(0)); setOpenMessages(false); }} + sx={{ color: 'text.secondary', '&:hover': { color: 'error.main' } }} > - + setOpenMessages(false)} + sx={{ color: 'text.secondary' }} > - + - {/* Message list — plain text, no MUI Alert per row */} -
{messages.length === 0 && ( - {t('messages.noMessages')} + + + + {t('messages.noMessages')} + + )} {groupedMessages.map((msg, index) => { - const color = TYPE_COLORS[msg.type] || '#333'; - const symbol = TYPE_SYMBOLS[msg.type] || '•'; + const color = theme.palette[msg.type].main; const hasDetails = !!(msg.detail || msg.code || msg.diagnostics); const isExpanded = expandedMessages.has(index); return ( -
- - {symbol} - [{formatTimestamp(msg.timestamp)}] - ({msg.component}) {msg.value} - {msg.count > 1 && ( - ×{msg.count} - )} - {hasDetails && ( - toggleExpand(index)} - > - {isExpanded ? `▾ ${t('messages.details')}` : `▸ ${t('messages.details')}`} - - )} - - {hasDetails && isExpanded && ( -
- {msg.detail && ( -
- — details — - {msg.detail} -
+ + + + + + + {msg.value} + + + + {msg.component} + + + {formatTimestamp(msg.timestamp)} + + {msg.count > 1 && ( + + ×{msg.count} + )} - {msg.code && ( -
- — code — - 
 : }
+                                                    onClick={() => toggleExpand(index)}
+                                                    sx={{
+                                                        minWidth: 0, p: 0,
+                                                        textTransform: 'none', fontSize: textVar.xxs,
+                                                        color: 'text.secondary',
+                                                        '& .MuiButton-startIcon': { mr: 0.125 },
+                                                        '&:hover': { color: 'primary.main', backgroundColor: 'transparent' },
+                                                    }}
+                                                >
+                                                    {t('messages.details')}
+                                                
+                                            )}
+                                        
+                                        {hasDetails && isExpanded && (
+                                            
+                                                {msg.detail && (
+                                                    
+                                                        {msg.detail}
+                                                    
+                                                )}
+                                                {msg.code && (
+                                                    
                                                         {msg.code.split('\n').filter(line => line.trim() !== '').join('\n')}
-
-
- )} - {msg.diagnostics && ( - - )} -
- )} -
+ + )} + {msg.diagnostics && } + + )} + + ); })} -
+ - {/* Last message toast — keep the single Alert for latest message popup */} - {latestMessage != undefined ? - - - [{formatTimestamp(latestMessage.timestamp)}] ({latestMessage.component}) {latestMessage?.value} - - {latestMessage?.detail && <> -
{latestMessage.detail}
- } - {latestMessage?.code && - 
+                
+                    
+                        {latestMessage.component} · {formatTimestamp(latestMessage.timestamp)}
+                    
+                    
+                        {latestMessage.value}
+                    
+                    {latestMessage.detail && (
+                        
+                            {latestMessage.detail}
+                        
+                    )}
+                    {latestMessage.code && (
+                        
                             {latestMessage.code.split('\n').filter(line => line.trim() !== '').join('\n')}
-
- } + + )} - : ""} + + ) : null} ); }); \ No newline at end of file diff --git a/src/views/UnifiedDataUploadDialog.tsx b/src/views/UnifiedDataUploadDialog.tsx index 16c0469d3..9d8144f50 100644 --- a/src/views/UnifiedDataUploadDialog.tsx +++ b/src/views/UnifiedDataUploadDialog.tsx @@ -610,10 +610,16 @@ export const DataLoadMenu: React.FC = ({ const isConnected = !!conn.connected || !!conn.sso_auto_connect; const statusLabel = isConnected ? t('upload.connectorConnected') - : t('upload.connectorDisconnected'); - const detail = isLocalFolder - ? (folderDisplay || t('upload.localFolderConnected', { defaultValue: 'Local folder' })) - : getConnectorTypeDescription(conn.source_type, conn.connected, t); + : t('upload.connectorNotConnected', { defaultValue: 'Not connected' }); + // A connector is its type plus which instance it points at; fall back to + // the generic type blurb for loaders that have no identifying params. + const identity = conn.connection_identity || (isLocalFolder ? folderDisplay : ''); + const detail = identity + || getConnectorTypeDescription(conn.source_type, conn.connected, t); + const tooltipIdentity = isLocalFolder ? folderTooltip : identity; + const tooltipLines = Array.from(new Set( + [conn.type_name, tooltipIdentity || (isConnected ? detail : '')].filter(Boolean) + )); return { value: `connector:${conn.id}` as UploadTabType, title: conn.display_name, @@ -627,7 +633,26 @@ export const DataLoadMenu: React.FC = ({ }} /> ), disabled: false, - tooltip: `${statusLabel}${detail ? ` · ${detail}` : ''}${isLocalFolder && folderTooltip ? ` · ${folderTooltip}` : ''}`, + // A disconnected connector's description is its status, and a short + // folder path is already shown in full — so drop repeated segments. + tooltip: ( + + {statusLabel} + {tooltipLines.length > 0 && ( + + {tooltipLines.map((line) => ( + + {line} + + ))} + + )} + + ), }; }), ]; @@ -763,13 +788,12 @@ export const DataLoadMenu: React.FC = ({ size="small" sx={{ fontSize: textVar.md, height: 30, borderRadius: 2, - color: alpha(theme.palette.text.primary, 0.78), - borderColor: alpha(theme.palette.text.primary, 0.22), - backgroundColor: alpha(theme.palette.background.paper, 0.72), - '& .MuiChip-icon': { fontSize: textVar.xl, ml: 0.5, color: alpha(theme.palette.text.primary, 0.55) }, + color: 'text.secondary', + borderColor: alpha(theme.palette.text.primary, 0.12), + '& .MuiChip-icon': { fontSize: textVar.lg, ml: 0.5, color: 'text.disabled' }, '&:hover': { - bgcolor: alpha(theme.palette.primary.main, 0.06), - borderColor: alpha(theme.palette.primary.main, 0.4), + bgcolor: 'action.hover', + borderColor: alpha(theme.palette.text.primary, 0.2), }, }} /> @@ -822,7 +846,7 @@ export const DataLoadMenu: React.FC = ({ }} attachments={agentAttachments} onAttachmentsChange={setAgentAttachments} - minRows={3} + minRows={4} tabSuggestion={t('upload.agentChatTabSuggestion', { defaultValue: 'What dataset do we have here?', })} @@ -880,66 +904,30 @@ export const DataLoadMenu: React.FC = ({ ))} - {/* Row 2 — add-a-source actions: same muted link family as the - connected sources, differentiated only by a subtle shaded - background chip (no primary color). */} + {/* Row 2 — add-a-source actions use the same lightweight link + style as connected sources; the row label provides hierarchy. */} - {t('upload.addSourceLabel', { defaultValue: 'Or add data directly:' })} + {t('upload.addSourceLabel', { defaultValue: 'Add data:' })} {connectorActionSources.map((source) => ( - handleConnectionClick(source.value)} + icon={source.icon} + title={source.title} + description={source.description} + onClick={() => handleConnectionClick(source.value)} disabled={source.disabled} - title={source.description} - sx={{ - display: 'inline-flex', - alignItems: 'center', - gap: 0.5, - px: 1, - py: 0.375, - border: `1px solid ${alpha(theme.palette.text.primary, 0.12)}`, - borderRadius: 1, - font: 'inherit', - whiteSpace: 'nowrap', - cursor: source.disabled ? 'not-allowed' : 'pointer', - opacity: source.disabled ? 0.5 : 1, - color: alpha(theme.palette.text.primary, 0.8), - bgcolor: alpha(theme.palette.text.primary, 0.07), - transition: 'background-color 120ms ease, color 120ms ease', - '&:hover': source.disabled ? {} : { - bgcolor: alpha(theme.palette.primary.main, 0.08), - borderColor: alpha(theme.palette.primary.main, 0.3), - color: 'text.primary', - }, - '& .MuiSvgIcon-root': { fontSize: iconVar.md }, - }} - > - {source.icon} - - {source.title} - - + /> ))} diff --git a/src/views/dataLoadingSuggestions.ts b/src/views/dataLoadingSuggestions.ts index 67f833eda..3f8b33f10 100644 --- a/src/views/dataLoadingSuggestions.ts +++ b/src/views/dataLoadingSuggestions.ts @@ -181,10 +181,10 @@ export function buildDataLoadingQuickActions( { t, setInput, setImages, setAttachments, requestAutoSend }: BuildSuggestionsArgs, ): DataLoadingQuickAction[] { const connectLabel = t('upload.agentChatQuickAction.connect', { - defaultValue: 'Help me connect to my data source', + defaultValue: 'Help me connect my data source', }); const askLabel = t('upload.agentChatQuickAction.askConnected', { - defaultValue: 'What data do we have from connected sources?', + defaultValue: 'What data are available from my sources?', }); const fillAndSend = (text: string) => { diff --git a/tests/backend/data/test_workspace_manager.py b/tests/backend/data/test_workspace_manager.py index ec88d9f5b..6fe53a2ed 100644 --- a/tests/backend/data/test_workspace_manager.py +++ b/tests/backend/data/test_workspace_manager.py @@ -140,6 +140,10 @@ def test_update_display_name_patches_session_state(self, manager): manager.create_workspace("ws") manager.save_session_state("ws", { "tables": [], + "inputTables": [{ + "id": "sales", + "source": {"kind": "connector", "connectorId": "warehouse"}, + }], "activeWorkspace": {"id": "ws", "displayName": "Old Name"}, }) @@ -150,6 +154,7 @@ def test_update_display_name_patches_session_state(self, manager): .read_text(encoding="utf-8") ) assert meta["displayName"] == "New Name" + assert meta["sourceIds"] == ["warehouse"] state = manager.load_session_state("ws") assert state["activeWorkspace"]["displayName"] == "New Name" @@ -180,6 +185,47 @@ def test_overwrite_session_state(self, manager): loaded = manager.load_session_state("test") assert loaded["version"] == 2 + def test_session_list_summarizes_data_sources(self, manager): + manager.create_workspace("sources") + manager.save_session_state("sources", { + "inputTables": [ + { + "id": "sales", + "source": { + "kind": "connector", + "connectorId": "warehouse", + }, + }, + { + "id": "customers", + "sourceConfig": { + "type": "database", + "connectorId": "warehouse", + }, + }, + { + "id": "forecast", + "sourceConfig": {"type": "file"}, + }, + { + "id": "legacy-kusto", + "sourceConfig": { + "type": "database", + "connector_id": "kusto-prod", + }, + }, + { + "id": "unidentified-database", + "sourceConfig": {"type": "database"}, + }, + ], + }) + + summary = manager.list_workspaces()[0] + + assert summary["table_count"] == 5 + assert summary["source_ids"] == ["kusto-prod", "upload", "warehouse"] + class TestOpenWorkspace: """Integration: WorkspaceManager creates workspace, opens as Workspace, writes data.""" diff --git a/tests/backend/routes/test_session_routes_migration.py b/tests/backend/routes/test_session_routes_migration.py index 80a43aec7..855496884 100644 --- a/tests/backend/routes/test_session_routes_migration.py +++ b/tests/backend/routes/test_session_routes_migration.py @@ -53,6 +53,27 @@ def test_save_session_reports_storage_full(self, client): assert "No space left on device" not in body["error"]["message"] +class TestListSessionRoute: + def test_list_includes_source_summary(self, client): + manager = MagicMock() + manager.list_workspaces.return_value = [{ + "id": "sales", + "display_name": "Sales", + "created_at": "2026-08-17T10:00:00Z", + "updated_at": "2026-08-17T11:00:00Z", + "source_ids": ["sample_datasets", "warehouse"], + }] + + with ( + patch("data_formulator.routes.sessions.get_identity_id", return_value="browser:abc"), + patch("data_formulator.routes.sessions.get_workspace_manager", return_value=manager), + ): + resp = client.get("/api/sessions/list") + + assert resp.status_code == 200 + session = resp.get_json()["data"]["sessions"][0] + assert session["source_ids"] == ["sample_datasets", "warehouse"] + class TestMigrateRoute: def test_migrate_moves_and_cleans_source(self, client): source_mgr = MagicMock() diff --git a/tests/frontend/unit/app/useAutoSave.test.tsx b/tests/frontend/unit/app/useAutoSave.test.tsx index acf4e3e11..34a1faa3e 100644 --- a/tests/frontend/unit/app/useAutoSave.test.tsx +++ b/tests/frontend/unit/app/useAutoSave.test.tsx @@ -8,7 +8,7 @@ const mocks = vi.hoisted(() => ({ activeWorkspace: { id: 'ws-1', displayName: 'Workspace 1' }, inputTables: [{ id: 'table-1' }], derivedTables: [], - }, + } as any, saveWorkspaceState: vi.fn(), handleApiError: vi.fn(), })); @@ -35,6 +35,12 @@ function AutoSaveHarness() { describe('useAutoSave', () => { beforeEach(() => { vi.useFakeTimers(); + mocks.state = { + sessionLoading: false, + activeWorkspace: { id: 'ws-1', displayName: 'Workspace 1' }, + inputTables: [{ id: 'table-1' }], + derivedTables: [], + }; mocks.saveWorkspaceState.mockReset(); mocks.handleApiError.mockReset(); }); @@ -57,6 +63,47 @@ describe('useAutoSave', () => { expect(mocks.handleApiError).toHaveBeenCalledWith(err, 'Auto-save'); }); + it('saves the latest state when a change arrives during an in-flight save', async () => { + let finishFirstSave!: () => void; + mocks.saveWorkspaceState.mockImplementationOnce(() => new Promise(resolve => { + finishFirstSave = resolve; + })); + + const { rerender } = render(); + + act(() => { + vi.advanceTimersByTime(3000); + }); + expect(mocks.saveWorkspaceState).toHaveBeenCalledTimes(1); + + mocks.state = { + ...mocks.state, + inputTables: [{ + id: 'table-1', + source: { kind: 'connector', connectorId: 'kusto-prod' }, + }], + }; + rerender(); + + act(() => { + vi.advanceTimersByTime(3000); + }); + expect(mocks.saveWorkspaceState).toHaveBeenCalledTimes(1); + + await act(async () => { + finishFirstSave(); + await Promise.resolve(); + }); + + expect(mocks.saveWorkspaceState).toHaveBeenCalledTimes(2); + expect(mocks.saveWorkspaceState.mock.calls[1][0]).toMatchObject({ + inputTables: [{ + id: 'table-1', + source: { kind: 'connector', connectorId: 'kusto-prod' }, + }], + }); + }); + it('strips connector form prefills from workspace snapshots', () => { const state = { ...mocks.state, diff --git a/tests/frontend/unit/views/DataSourceSidebar.test.tsx b/tests/frontend/unit/views/DataSourceSidebar.test.tsx index 095c28cbd..17f0ac6c7 100644 --- a/tests/frontend/unit/views/DataSourceSidebar.test.tsx +++ b/tests/frontend/unit/views/DataSourceSidebar.test.tsx @@ -166,7 +166,7 @@ describe('DataSourceSidebar', () => { })); }); - it('shows recently modified sessions first and can switch to creation order', async () => { + it('shows newest-created sessions first and can switch to recently modified order', async () => { mockState.dataSourceSidebarTab = 'sessions'; vi.mocked(listWorkspaces).mockResolvedValue([ { @@ -187,11 +187,52 @@ describe('DataSourceSidebar', () => { const recentlyEdited = await screen.findByText('Recently edited'); const newerCreation = screen.getByText('Newer creation'); + expect(newerCreation.compareDocumentPosition(recentlyEdited) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + + fireEvent.click(screen.getByRole('button', { name: 'Group and sort sessions' })); + fireEvent.click(await screen.findByText('sidebar.sortRecentlyModifiedFirst')); + expect(recentlyEdited.compareDocumentPosition(newerCreation) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + }); + + it('groups sessions by their summarized data sources by default', async () => { + mockState.dataSourceSidebarTab = 'sessions'; + vi.mocked(apiRequest).mockResolvedValue({ + data: { + connectors: [ + { id: 'kusto-prod', display_name: 'Kusto' }, + { id: 'mysql-main', display_name: 'MyMysqlDB' }, + { id: 'local-datasets', display_name: '~/datasets' }, + ], + }, + }); + vi.mocked(listWorkspaces).mockResolvedValue([ + { + id: 'mixed', + display_name: 'Mixed sources', + created_at: '2026-08-15T10:00:00Z', + saved_at: '2026-08-15T10:00:00Z', + source_ids: ['kusto-prod', 'mysql-main'], + }, + { + id: 'local', + display_name: 'Local data', + created_at: '2026-08-14T10:00:00Z', + saved_at: '2026-08-14T10:00:00Z', + source_ids: ['local-datasets'], + }, + ]); - fireEvent.click(screen.getByRole('button', { name: 'sidebar.sortSessions' })); - fireEvent.click(await screen.findByText('sidebar.sortNewestFirst')); + render(); - expect(newerCreation.compareDocumentPosition(recentlyEdited) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + expect(await screen.findByText('Kusto / MyMysqlDB')).toBeInTheDocument(); + expect(screen.getByText('~/datasets')).toBeInTheDocument(); + + fireEvent.click(screen.getByRole('button', { name: 'Group and sort sessions' })); + fireEvent.click(await screen.findByText('No grouping')); + + expect(screen.queryByText('Kusto / MyMysqlDB')).not.toBeInTheDocument(); + expect(screen.getByText('Mixed sources')).toBeInTheDocument(); + expect(screen.getByText('Local data')).toBeInTheDocument(); }); }); From fbd081b660400029ceed74f26bd2cd8062c3ed7f Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Mon, 17 Aug 2026 23:15:25 -0700 Subject: [PATCH 2/5] hmm? --- .gitignore | 1 + loops/model-evaluation/plan.md | 66 ---------------------------------- 2 files changed, 1 insertion(+), 66 deletions(-) delete mode 100644 loops/model-evaluation/plan.md diff --git a/.gitignore b/.gitignore index 6f637365c..2277e43c1 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ design-docs/ deploy-scripts/ test-data-loader/ scripts/ +docs/esrp/* ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. diff --git a/loops/model-evaluation/plan.md b/loops/model-evaluation/plan.md deleted file mode 100644 index 1fd19bf34..000000000 --- a/loops/model-evaluation/plan.md +++ /dev/null @@ -1,66 +0,0 @@ -# Loop — Open-Source (Ollama) Model Evaluation - -**High-level plan.** Execute end-to-end, making reasonable decisions when details are -ambiguous, and record them in the final report (`report.md`; all working artifacts go -under `work/`). - -## Goal - -Benchmark open-source (Ollama) models that drive Data Formulator's analyst agents — -inspect tabular data, write transformation code, and commit a visualization — and report -**two independent axes**: - -1. **Success rate** — does the agent actually produce a rendered chart? (reliability) -2. **Quality when produced** — how good is the chart when it finishes, scored 0-100 by a - code + vision grader? (competence) - -Keep them separate: a model can write good code yet fail to deliver it through the -protocol. The dominant open-model failure mode is **driving the tool/transport, not -analyzing the data**, so each model runs through more than one agent transport: - -- `analyst` — native function/tool calls (with a content-JSON salvage fallback). -- `mini` — single-decision, pure-prompt JSON contract; the production low-cost agent. - -Always include the Azure references `gpt-5.5`, `gpt-5-mini` as the baseline. - -## Data - -A frozen **45-question** set across **15 datasets** from the `../visbench` benchmark, fed -as the **raw / grouped source tables** (not VisBench's derived single-table `data.csv`) so -the agent must do its own joins: - -- **vega_datasets** single tables — 9 single-table questions. -- **TidyTuesday** multi-CSV weeks — 18 multi-table questions. -- **Spider** databases grouped by DB — 18 multi-table questions. - -Reuse VisBench's quality-filtered question and reference chart for each item. The single- -vs multi-table split (9 / 36) is the axis along which models diverge most. - -## Steps - -1. **Select & pull models** — the open roster across size tiers (1B → 120B) plus the three - Azure references. -2. **Prepare the benchmark** — materialize the 45 questions as raw/grouped tables and - freeze the VisBench questions + reference charts, reused identically across every model - and agent. -3. **Run agents** — every `(agent, model, question)` cell with `--agent` in `analyst` - and `mini`; capture the event stream and render each chart to PNG. Frozen controls: - `max_iterations = 5`, 240 s timeout, resumable. -4. **Score (two phases, GPT-5.5 grader):** - - **Phase 1 — reliability:** five sequential gates (responded → emitted action → code - ran → output → **produced chart**). The chart gate is decisive and defines the - success rate; only those runs proceed. - - **Phase 2 — quality (0-100, produced charts only):** code review vs the question - (0-50) + vision review of the rendered PNG vs the reference chart (0-50). -5. **Aggregate & report** — report the two axes separately (never collapse them); for - ranking only, derive success-weighted quality (Phase 2 over all 45, no-chart = 0) and - combined = `0.3 × (success_rate × 100) + 0.7 × success-weighted quality`. Always show - the single- vs multi-table split, the per-gate drop-off, comparison to the references, - and recommendations per size tier (with which `--agent`). - -## Principles - -- **Two axes stay separate** — `combined` is for ranking only. -- **Freeze controls** — same questions, grader, `max_iterations`, and timeout across every cell. -- **`mini` is the production low-cost agent** — `simple` was removed; don't run `--agent simple`. -- **`uv` only**, no secrets (Azure auth via Entra ID), resumable, all artifacts under `work/`. From 3de3293d742e14cbeccd37950e304f887e67c97f Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Tue, 18 Aug 2026 17:45:35 -0700 Subject: [PATCH 3/5] fixes --- .../agents/agent_data_loading_chat.py | 33 +- py-src/data_formulator/agents/agent_utils.py | 9 +- py-src/data_formulator/agents/context.py | 13 +- py-src/data_formulator/analyst/agent.py | 78 +++- .../analyst/skills/core/SKILL.md | 5 +- .../analyst/skills/core/tools.json | 6 +- .../analyst/skills/data-loading/SKILL.md | 35 +- .../analyst/skills/data-loading/skill.py | 3 +- .../analyst/skills/data-loading/tools.json | 3 +- .../analyst/skills/data_loading/SKILL.md | 109 ------ .../analyst/skills/data_loading/skill.py | 362 ------------------ .../analyst/skills/data_loading/tools.json | 233 ----------- .../analyst/skills/report/SKILL.md | 2 +- py-src/data_formulator/analyst/tools.py | 8 +- py-src/data_formulator/data_connector.py | 172 ++++++--- .../data_loader/sample_datasets_loader.py | 7 +- .../data_operations/discovery.py | 95 ++++- .../data_formulator/datalake/catalog_cache.py | 87 ++++- .../datalake/connector_preferences.py | 60 +++ py-src/data_formulator/routes/agents.py | 2 + src/app/dfSlice.tsx | 13 +- src/app/stateMigrations.ts | 39 +- src/components/ComponentType.tsx | 1 - src/i18n/locales/en/common.json | 12 +- src/i18n/locales/en/dataLoading.json | 8 +- src/i18n/locales/zh/common.json | 12 +- src/i18n/locales/zh/dataLoading.json | 8 +- src/views/DataSourceSidebar.tsx | 127 +++--- src/views/DataThread.tsx | 43 ++- src/views/DataThreadCards.tsx | 13 +- src/views/DataView.tsx | 3 +- src/views/InteractionEntryCard.tsx | 6 +- src/views/LogViewerDialog.tsx | 307 ++++++++++++++- src/views/ReportView.tsx | 17 +- src/views/SimpleChartRecBox.tsx | 25 +- src/views/SourceTableShelf.tsx | 4 +- src/views/VisualizationView.tsx | 24 +- src/views/analystToolProgress.ts | 84 ++++ .../agents/test_analyst_scratch_files.py | 2 + tests/backend/agents/test_context.py | 31 +- .../test_data_loading_discovery_tools.py | 138 ++++++- .../backend/agents/test_data_loading_skill.py | 68 ++++ tests/backend/data/test_catalog_cache.py | 24 ++ .../data/test_data_connector_framework.py | 53 +++ .../test_analyst_data_operation_flow.py | 45 +++ .../unit/app/dfSliceTableCollections.test.ts | 24 +- .../frontend/unit/app/stateMigrations.test.ts | 35 +- tests/frontend/unit/app/useAutoSave.test.tsx | 15 + .../unit/views/DataSourceSidebar.test.tsx | 103 +++++ .../unit/views/LogViewerDialog.test.ts | 94 +++++ .../unit/views/VisualizationView.test.ts | 19 + .../unit/views/analystToolProgress.test.ts | 53 +++ 52 files changed, 1798 insertions(+), 974 deletions(-) delete mode 100644 py-src/data_formulator/analyst/skills/data_loading/SKILL.md delete mode 100644 py-src/data_formulator/analyst/skills/data_loading/skill.py delete mode 100644 py-src/data_formulator/analyst/skills/data_loading/tools.json create mode 100644 py-src/data_formulator/datalake/connector_preferences.py create mode 100644 src/views/analystToolProgress.ts create mode 100644 tests/frontend/unit/views/LogViewerDialog.test.ts create mode 100644 tests/frontend/unit/views/VisualizationView.test.ts create mode 100644 tests/frontend/unit/views/analystToolProgress.test.ts diff --git a/py-src/data_formulator/agents/agent_data_loading_chat.py b/py-src/data_formulator/agents/agent_data_loading_chat.py index 01419374a..36062dccc 100644 --- a/py-src/data_formulator/agents/agent_data_loading_chat.py +++ b/py-src/data_formulator/agents/agent_data_loading_chat.py @@ -789,11 +789,11 @@ def _build_connector_summary_block( *, max_total_chars: int = 1200, ) -> str: - """Render a compact directory of cached connector catalogs. + """Render a compact directory of currently loadable connectors. - Only shows source IDs with table counts (and folder counts when the - catalog is hierarchical). The agent is expected to call ``list_data`` - for full inventory. + Shows connected sources even before their catalog has been cached. Retained + catalogs for disconnected sources stay on disk but are not agent-visible. + The agent is expected to call ``list_data`` for full inventory. Strictly hard-capped at ``max_total_chars``. """ if not user_home: @@ -807,9 +807,19 @@ def _build_connector_summary_block( return " none" try: - source_ids = list_cached_sources(user_home) + from data_formulator.data_connector import ( + connector_is_available, + list_available_connector_ids, + ) + cached_source_ids = set(list_cached_sources(user_home)) + available_source_ids = set(list_available_connector_ids()) + source_ids = sorted( + source_id + for source_id in cached_source_ids | available_source_ids + if connector_is_available(source_id) is not False + ) except Exception: - logger.debug("connector summary: list_cached_sources failed", exc_info=True) + logger.debug("connector summary: source inventory failed", exc_info=True) return " none" if not source_ids: @@ -817,7 +827,7 @@ def _build_connector_summary_block( user_home_path = Path(user_home) lines: list[str] = [] - for sid in sorted(source_ids): + for sid in source_ids: try: tables = load_catalog(user_home_path, sid) or [] except Exception: @@ -825,14 +835,17 @@ def _build_connector_summary_block( tables = [] n, k = _summarize_catalog_shape(tables) if n == 0: - lines.append(f"- {sid}: 0 tables cached") + status = "connected, catalog not cached" if sid in available_source_ids else "0 tables cached" + lines.append(f"- {sid}: {status}") elif k > 0: + availability = "connected" if sid in available_source_ids else "catalog available" lines.append( - f"- {sid}: {n} table{'s' if n != 1 else ''} " + f"- {sid}: {availability}; {n} table{'s' if n != 1 else ''} " f"across {k} folder{'s' if k != 1 else ''}" ) else: - lines.append(f"- {sid}: {n} table{'s' if n != 1 else ''}") + availability = "connected" if sid in available_source_ids else "catalog available" + lines.append(f"- {sid}: {availability}; {n} table{'s' if n != 1 else ''}") lines.append( " (call list_data() for sources, list_data(source_id, ...) to drill, " diff --git a/py-src/data_formulator/agents/agent_utils.py b/py-src/data_formulator/agents/agent_utils.py index f8e24ada9..1754b004f 100644 --- a/py-src/data_formulator/agents/agent_utils.py +++ b/py-src/data_formulator/agents/agent_utils.py @@ -583,8 +583,8 @@ def generate_data_summary( Use WorkspaceWithTempData context manager to mount temp tables to workspace. When ``primary_tables`` is provided, the output is structured into tiered sections: - - **[PRIMARY TABLE]** / **[PRIMARY TABLES]**: Full detail for the tables the user is focused on. - - **[OTHER AVAILABLE TABLES]**: Full detail for the remaining tables. + - **[PRIMARY ANALYSIS INPUTS]**: Full detail for the input tables the user is focused on. + - **[OTHER ANALYSIS INPUTS]**: Full detail for the remaining input tables. Sections are omitted when empty. Args: @@ -737,10 +737,9 @@ def assemble_table_summary(table, idx): sections = [] if primary_parts: - header = "[PRIMARY TABLE]" if len(primary_parts) == 1 else "[PRIMARY TABLES]" - sections.append(header + "\n\n" + separator.join(primary_parts)) + sections.append("[PRIMARY ANALYSIS INPUTS]\n\n" + separator.join(primary_parts)) if other_parts: - sections.append("[OTHER AVAILABLE TABLES]\n\n" + separator.join(other_parts)) + sections.append("[OTHER ANALYSIS INPUTS]\n\n" + separator.join(other_parts)) return "\n\n".join(sections) # Join with visual separators (no tiering) diff --git a/py-src/data_formulator/agents/context.py b/py-src/data_formulator/agents/context.py index 8dc743c93..e24c2ec75 100644 --- a/py-src/data_formulator/agents/context.py +++ b/py-src/data_formulator/agents/context.py @@ -159,7 +159,7 @@ def build_lightweight_table_context( """Build compact table context with schema, metadata, value samples, and rows. When ``primary_tables`` is provided, tables are grouped into - [PRIMARY TABLE(S)] and [OTHER AVAILABLE TABLES] sections. + [PRIMARY ANALYSIS INPUTS] and [OTHER ANALYSIS INPUTS] sections. """ table_desc_cache, col_desc_cache, import_opts_cache = _get_workspace_metadata_lookups(workspace) table_extra_cache: dict[str, list[str]] = {} @@ -263,7 +263,7 @@ def _table_section(table: dict[str, Any]) -> str: return _client_schema_section(table, label) load_hint = ( - "\nThe tables above are the data already loaded into this workspace, and the " + "\nThe analysis input tables above are already materialized and are the " "only data you can read directly. Anything not listed here has not been loaded " "yet: find it in a connected source and propose loading it before relying on it.\n" "To load a table in code: pd.read_parquet('file.parquet') or " @@ -278,12 +278,11 @@ def _table_section(table: dict[str, Any]) -> str: sections = [] if primary_tables_list: - header = "[PRIMARY TABLE]" if len(primary_tables_list) == 1 else "[PRIMARY TABLES]" primary_parts = [_table_section(t) for t in primary_tables_list] - sections.append(header + "\n\n" + "\n\n".join(primary_parts)) + sections.append("[PRIMARY ANALYSIS INPUTS]\n\n" + "\n\n".join(primary_parts)) if other_tables_list: other_parts = [_table_section(t) for t in other_tables_list] - sections.append("[OTHER AVAILABLE TABLES]\n\n" + "\n\n".join(other_parts)) + sections.append("[OTHER ANALYSIS INPUTS]\n\n" + "\n\n".join(other_parts)) return "\n\n".join(sections) + "\n" + load_hint sections = [_table_section(table) for table in input_tables] @@ -375,6 +374,10 @@ def handle_read_catalog_metadata( if not user_home: return "Cannot read catalog metadata: user home not available." + from data_formulator.datalake.connector_preferences import connector_is_enabled + if not connector_is_enabled(user_home, source_id): + return f"Source '{source_id}' is disconnected." + # Surface zero-config admin connectors (e.g. sample_datasets) on first use. ensure_no_auth_catalogs_cached(user_home) diff --git a/py-src/data_formulator/analyst/agent.py b/py-src/data_formulator/analyst/agent.py index a9359d4f5..62ea523ce 100644 --- a/py-src/data_formulator/analyst/agent.py +++ b/py-src/data_formulator/analyst/agent.py @@ -81,6 +81,40 @@ # emitted match — never the same text pasted by a user or echoed by the model. _SKILL_LOADED_BANNER = "[SKILL LOADED: {name}]" _SKILL_LOADED_RE = re.compile(r"^\[SKILL LOADED: ([^\]]+)\]") +_SKILL_PRELOADED_RE = re.compile( + r"\[SKILL: ([^\]]+)\] Preloaded for this run" +) + +_TOOL_PROGRESS_ARG_KEYS: dict[str, tuple[str, ...]] = { + "list_data": ("source_id", "path", "filter"), + "find_data": ("query", "scope"), + "describe_data": ("source_id", "table_key"), + "probe_data": ("source_id", "table_key", "query"), + "describe_connector": ("source_type",), + "inspect_chart": ("chart_id",), + "search_data_tables": ("query",), + "search_knowledge": ("query",), +} + + +def _tool_progress_args(tool_name: str, args: dict[str, Any]) -> dict[str, Any]: + """Return model arguments safe and useful for user-facing progress.""" + progress_args = { + key: args[key] + for key in _TOOL_PROGRESS_ARG_KEYS.get(tool_name, ()) + if key in args + } + if tool_name == "probe_data" and isinstance(progress_args.get("query"), dict): + query = progress_args["query"] + progress_args["query"] = { + key: query[key] + for key in ("aggregates", "group_by", "limit") + if key in query + } + filters = query.get("filters") + if isinstance(filters, list) and filters: + progress_args["query"]["filter_count"] = len(filters) + return progress_args # ── Action-argument coercion ────────────────────────────────────────────── # Weaker models sometimes JSON-encode a nested action argument as a string @@ -350,6 +384,14 @@ def _legal_actions(self) -> frozenset[str]: legal.update(meta.action_names) return frozenset(legal) + @staticmethod + def _initial_loaded_skills(input_tables: list[dict[str, Any]]) -> set[str]: + """Return the skill gates that must be open before the first LLM call.""" + loaded = {_CORE_SKILL} + if not input_tables: + loaded.add("data-loading") + return loaded + # ------------------------------------------------------------------ # Public API # ------------------------------------------------------------------ @@ -388,14 +430,15 @@ def run( iteration = completed_step_count final_status = "max_iterations" - # Reset per-run skill + payload state. ``core`` is auto-loaded: its - # baseline tools + actions are always available and its SKILL.md body is - # appended to the system frame (see _build_system_prompt). Gated skills - # are added to this set as the model loads them. The payload carries + # Reset per-run skill + payload state. ``core`` is always loaded. With + # no analysis input tables, data loading is the immediate workflow, so expose + # its tools, actions, and guidance before the first model call instead + # of spending a round on load_skill. Other gated skills are added as the + # model loads them. The payload carries # everything a dispatched skill handler needs to build its own context # (e.g. the report skill rebuilds [AVAILABLE CHARTS] + thread # context). - self._loaded_skills = {_CORE_SKILL} + self._loaded_skills = self._initial_loaded_skills(input_tables) self._run_payload = { "input_tables": input_tables, "charts": charts or [], @@ -663,6 +706,10 @@ def _rehydrate_loaded_skills(self, trajectory: list[dict]) -> None: name = self.registry.canonical_name(m.group(1).strip()) if self.registry.has(name): self._loaded_skills.add(name) + for preloaded in _SKILL_PRELOADED_RE.finditer(content): + name = self.registry.canonical_name(preloaded.group(1).strip()) + if self.registry.has(name): + self._loaded_skills.add(name) def _load_skill_into_context( self, name: str, trajectory: list[dict], @@ -1165,15 +1212,18 @@ def _build_system_prompt( context_lines = [] if has_primary_tables: context_lines.append( - "- **[PRIMARY TABLE(S)]**: The table(s) the user is focused on. " - "Prioritize these, but freely use other available tables if needed." + "- **[PRIMARY ANALYSIS INPUTS]**: The analysis input table(s) the " + "user is focused on. Prioritize these, but freely use other " + "analysis inputs if needed." ) context_lines.append( - "- **[OTHER AVAILABLE TABLES]**: Additional tables in the workspace." + "- **[OTHER ANALYSIS INPUTS]**: Additional materialized input " + "tables the analyst can read directly." ) else: context_lines.append( - "- **[AVAILABLE TABLES]**: All tables in the workspace." + "- **[ANALYSIS INPUT TABLES]**: All materialized root data inputs " + "the analyst can read directly." ) context_lines.append( " Use `inspect_source_data` to get detailed stats and sample rows. " @@ -1234,6 +1284,12 @@ def _build_system_prompt( f"\n\n[SKILL: {_CORE_SKILL}] Always-on baseline — these tools and " f"actions are active for the whole run.\n\n{core_body}" ) + for name in sorted(self._loaded_skills - {_CORE_SKILL}): + body = self.registry.load_body(name) + prompt += ( + f"\n\n[SKILL: {name}] Preloaded for this run — its tools and " + f"actions are active now.\n\n{body}" + ) if self._knowledge_store: knowledge_rules = self._knowledge_store.load_always_apply_rules() @@ -1277,7 +1333,7 @@ def _build_initial_messages( if primary_tables: user_content = f"{table_summaries}\n\n" else: - user_content = f"[AVAILABLE TABLES]\n\n{table_summaries}\n\n" + user_content = f"[ANALYSIS INPUT TABLES]\n\n{table_summaries}\n\n" if focused_block: user_content += f"{focused_block}\n\n" if peripheral_block: @@ -1597,10 +1653,12 @@ def _tool_loop( yield { "type": "tool_start", "tool": tool_name, + "args": _tool_progress_args(tool_name, tool_args), "purpose": tool_args.get("purpose") if tool_name == "execute_python_script" else None, "code": tool_args.get("code") if tool_name == "execute_python_script" else None, "table_names": tool_args.get("table_names") if tool_name == "inspect_source_data" else None, "skill": tool_args.get("name") if tool_name == "load_skill" else None, + "query": tool_args.get("query") if tool_name in ("search_data_tables", "search_knowledge") else None, } tool_t0 = time.time() diff --git a/py-src/data_formulator/analyst/skills/core/SKILL.md b/py-src/data_formulator/analyst/skills/core/SKILL.md index 15e4a6294..6775e0c1a 100644 --- a/py-src/data_formulator/analyst/skills/core/SKILL.md +++ b/py-src/data_formulator/analyst/skills/core/SKILL.md @@ -31,7 +31,7 @@ to use it well. variables do NOT persist between calls, so combine related steps into a single script. - **inspect_source_data(table_names)** — get schema, stats, and sample rows for - source tables (cheaper than `execute_python_script` for basic inspection). + analysis input tables (cheaper than `execute_python_script` for basic inspection). - **load_skill(name)** — load a skill's instructions into context so you can use the action it unlocks (see the Skills section of your system instructions). @@ -44,6 +44,9 @@ requires connected data that isn't present, call `load_skill("data-loading")` and follow that skill's discovery and immutable proposal workflow in this same conversation. Do not hand off to the standalone Data Loading agent. +When `[ANALYSIS INPUT TABLES]` is empty, the `data-loading` skill is already +loaded — use it to find out what is available and tell the user what you found. + The initial context already includes sample rows and statistics for each table. If the data is straightforward, go straight to the action without calling tools. Tool results are returned to you before you act. diff --git a/py-src/data_formulator/analyst/skills/core/tools.json b/py-src/data_formulator/analyst/skills/core/tools.json index 599293a22..16bf9e1ab 100644 --- a/py-src/data_formulator/analyst/skills/core/tools.json +++ b/py-src/data_formulator/analyst/skills/core/tools.json @@ -24,14 +24,14 @@ "type": "function", "function": { "name": "inspect_source_data", - "description": "Get a detailed summary of one or more source tables — schema, field-level statistics, and sample rows. Cheaper than execute_python_script for basic data inspection.", + "description": "Get a detailed summary of one or more analysis input tables — schema, field-level statistics, and sample rows. Cheaper than execute_python_script for basic data inspection.", "parameters": { "type": "object", "properties": { "table_names": { "type": "array", "items": { "type": "string" }, - "description": "List of workspace table names, as listed in the available-tables context, to inspect." + "description": "Names listed in the analysis-input-tables context to inspect." } }, "required": ["table_names"] @@ -61,7 +61,7 @@ "input_tables": { "type": "array", "items": { "type": "string" }, - "description": "Workspace table names, as listed in the available-tables context, that the code reads." + "description": "Names listed in the analysis-input-tables context that the code reads." }, "code": { "type": "string", diff --git a/py-src/data_formulator/analyst/skills/data-loading/SKILL.md b/py-src/data_formulator/analyst/skills/data-loading/SKILL.md index a43f70906..0531df7eb 100644 --- a/py-src/data_formulator/analyst/skills/data-loading/SKILL.md +++ b/py-src/data_formulator/analyst/skills/data-loading/SKILL.md @@ -24,15 +24,34 @@ actions: # Skill: Data discovery -The workspace tables listed in your context are the data already loaded into the -system, and the only data that can be read directly. Everything these tools -return is *not* loaded yet — it lives in a connected source and only becomes -usable after the user selects a loading option and the server materializes it. +The analysis input tables listed in your context are already materialized and +are the only data that can be read directly. Everything these tools return is +*not* loaded yet — it lives in a connected source and only becomes usable after +the user selects a loading option and the server materializes it. Use these tools to determine whether connected sources contain data needed for the user's goal. They are read-only: discovering, describing, or probing a source does not add anything to the workspace analysis inputs. +## When nothing is loaded yet + +Discovery is cheap: call `list_data({})` and browse before you say anything +about what is or isn't available. The inventory lists each source's top-level +contents, and opening a source returns its whole subtree — so a couple of calls +show you the shape. Then tell the user what you actually found — which sources +are connected, what they hold, and which tables look relevant. Name real tables. + +Pick the path that fits: + +- The user named a subject → `find_data`, then propose the tables that match. +- The user asked what data exists, or asked nothing specific → summarize the + inventory and propose the most useful starting tables. +- Nothing is connected → `list_connectors`, then `propose_connection`, or say + they can upload a file. + +Use `ask_user` only for a choice you genuinely cannot make yourself, and never +before you have looked. Asking which source to inspect first is not an answer. + ## Adding a connector When the user wants to connect a new source, do not merely ask them to navigate @@ -66,9 +85,9 @@ seeds and are removed from persisted UI state. values. Pass the exact `source_id` and `table_key` returned by discovery. 3. Use `probe_data` only when metadata is insufficient to choose a useful bounded result. Probes are limited, read-only, and may be approximate. -4. First reconcile discoveries with every table in `[PRIMARY TABLE(S)]`, - `[OTHER AVAILABLE TABLES]`, or `[AVAILABLE TABLES]`. If the needed data is - already loaded, use or explain that workspace table instead of proposing it. +4. First reconcile discoveries with every table in `[PRIMARY ANALYSIS INPUTS]`, + `[OTHER ANALYSIS INPUTS]`, or `[ANALYSIS INPUT TABLES]`. If the needed data is + already loaded, use or explain that analysis input instead of proposing it. 5. When there are genuinely missing useful alternatives, call `propose_data_operation` with one to three complete immutable plans. This pauses for the user's choice; it @@ -90,6 +109,8 @@ yourself. Write it as you'd say it to a colleague, not as a schema summary. and one or more tables. The labels are buttons, not sentences — the reasoning belongs in your message text. The application displays table previews separately, so don't list columns as a substitute for explaining. +- An option is one coherent choice: one or a group of tables that serve the same + analysis, and leave out the ones that don't. - Use only source IDs, table keys, columns, and values grounded by discovery. - For a whole table, omit `query`. Use the optional raw-row query only when the request needs filters, projection, ordering, or an intentional limit. It uses diff --git a/py-src/data_formulator/analyst/skills/data-loading/skill.py b/py-src/data_formulator/analyst/skills/data-loading/skill.py index 8ee921710..b4d436a7b 100644 --- a/py-src/data_formulator/analyst/skills/data-loading/skill.py +++ b/py-src/data_formulator/analyst/skills/data-loading/skill.py @@ -305,7 +305,8 @@ def _propose_data_operation( names = ", ".join(dict.fromkeys(loaded_tables)) raise ValueError( f"This proposal duplicates data already loaded in the workspace: {names}. " - "Use those workspace tables directly, explain their relevance, or propose only missing data." + "Use those analysis input tables directly, explain their relevance, " + "or propose only missing data." ) DataOperationRepository.for_workspace(ctx.workspace).create( operation, diff --git a/py-src/data_formulator/analyst/skills/data-loading/tools.json b/py-src/data_formulator/analyst/skills/data-loading/tools.json index 30db03417..41af84a72 100644 --- a/py-src/data_formulator/analyst/skills/data-loading/tools.json +++ b/py-src/data_formulator/analyst/skills/data-loading/tools.json @@ -3,7 +3,7 @@ "type": "function", "function": { "name": "list_data", - "description": "Browse cached connected-source catalogs. With no arguments, list source summaries. With source_id, list its top-level entries. Add path to browse direct children and filter for a case-insensitive substring match.", + "description": "Browse cached connected-source catalogs. With no arguments, list each source with its top-level contents — usually enough to say what is available. With source_id, list that level plus `tree`, a nested view of everything below it (objects are folders, null marks a table), so you rarely need to walk level by level. Add path to move down and filter for a case-insensitive substring match.", "parameters": { "type": "object", "properties": { @@ -179,6 +179,7 @@ }, "tables": { "type": "array", + "description": "The tables that serve the same analysis.", "minItems": 1, "items": { "type": "object", diff --git a/py-src/data_formulator/analyst/skills/data_loading/SKILL.md b/py-src/data_formulator/analyst/skills/data_loading/SKILL.md deleted file mode 100644 index a43f70906..000000000 --- a/py-src/data_formulator/analyst/skills/data_loading/SKILL.md +++ /dev/null @@ -1,109 +0,0 @@ ---- -name: data-loading -description: >- - Discover connected data sources, add new data connectors through a - user-confirmed form, inspect table metadata, and run bounded read-only probes - when the current workspace data is insufficient. -when_to_use: >- - The user's question needs data that is not already available as a workspace - input, the user asks what connected data is available, or the user wants to - connect a database, warehouse, or cloud source. Not for analyzing tables - already listed in the workspace context. -always_on: false -tools: - - list_data - - find_data - - describe_data - - probe_data - - list_connectors - - describe_connector -actions: - - propose_data_operation - - propose_connection ---- - -# Skill: Data discovery - -The workspace tables listed in your context are the data already loaded into the -system, and the only data that can be read directly. Everything these tools -return is *not* loaded yet — it lives in a connected source and only becomes -usable after the user selects a loading option and the server materializes it. - -Use these tools to determine whether connected sources contain data needed for -the user's goal. They are read-only: discovering, describing, or probing a -source does not add anything to the workspace analysis inputs. - -## Adding a connector - -When the user wants to connect a new source, do not merely ask them to navigate -to settings and do not attempt to connect on their behalf. - -1. Call `list_connectors` first because available built-ins and plugins vary by - deployment. For a broad request such as "help me connect", summarize the - concrete available types and ask which one they use. -2. Once the source type is known, call `describe_connector` when field or auth - details are useful. -3. **When the requested source type is known and available, you MUST call - `propose_connection` in this same turn.** Do not stop with text such as - "I'll open the form", "you'll need to provide", or a list of required - fields. Only the action opens the form. Include one or two helpful sentences - alongside the action call explaining what the user should review or supply; - this text appears above the chat while the form opens on the canvas. Pass - `prefilled` values the user already supplied, including values parsed from a - connection string or config snippet. Never invent missing values. -4. The form is only a proposal. The user reviews it and clicks Connect; the - action must never connect automatically. - -Prefilled values may include credentials the user deliberately supplied. Do not -repeat those values in prose or subsequent tool output. They are transient form -seeds and are removed from persisted UI state. - -## Discovery sequence - -1. Use `find_data` when the user names a business concept or table. Use - `list_data` when you need to browse available sources or hierarchy. -2. Use `describe_data` before relying on columns, types, row counts, or filter - values. Pass the exact `source_id` and `table_key` returned by discovery. -3. Use `probe_data` only when metadata is insufficient to choose a useful - bounded result. Probes are limited, read-only, and may be approximate. -4. First reconcile discoveries with every table in `[PRIMARY TABLE(S)]`, - `[OTHER AVAILABLE TABLES]`, or `[AVAILABLE TABLES]`. If the needed data is - already loaded, use or explain that workspace table instead of proposing it. -5. When there are genuinely missing useful alternatives, call - `propose_data_operation` with one - to three complete immutable plans. This pauses for the user's choice; it - does not load data yet. - -## Proposing loading options - -Write your answer as **message text alongside the call** — that prose is what -the user reads, so it carries the whole answer. Do not put it in an action -field, and do not leave the call bare. Say what you went looking for, what you -actually found, and what each option would give them — enough that they can -choose without opening a single preview. Two to four sentences; more when the -options differ in ways that matter (grain, coverage, freshness, joins needed), -fewer when the choice is obvious. Name real tables and columns you saw during -discovery, and say plainly when an option is a compromise or when you'd pick one -yourself. Write it as you'd say it to a colleague, not as a schema summary. - -- Each `option` is a complete alternative: a concise action label (2–6 words) - and one or more tables. The labels are buttons, not sentences — the - reasoning belongs in your message text. The application displays table - previews separately, so don't list columns as a substitute for explaining. -- Use only source IDs, table keys, columns, and values grounded by discovery. -- For a whole table, omit `query`. Use the optional raw-row query only when the - request needs filters, projection, ordering, or an intentional limit. It uses - the same `filters` / `columns` / `order_by` / `limit` vocabulary as - `probe_data`, without aggregation. -- Do not invent operation IDs, plan IDs, or hashes. The server creates them. -- Never propose an exact connector query already represented by a workspace - table. The server also enforces this using persisted load provenance. - -## Grounding rules - -- Never invent source IDs, table keys, columns, or category values. -- Prefer cached catalog discovery before a live probe. -- Treat probe rows as evidence for planning, not as analysis input data. -- Keep queries structured and bounded. Do not generate source-specific SQL. -- If a source is unavailable or permissions changed, report the tool result and - ask the user for the needed connection or choose another source. \ No newline at end of file diff --git a/py-src/data_formulator/analyst/skills/data_loading/skill.py b/py-src/data_formulator/analyst/skills/data_loading/skill.py deleted file mode 100644 index 8ee921710..000000000 --- a/py-src/data_formulator/analyst/skills/data_loading/skill.py +++ /dev/null @@ -1,362 +0,0 @@ -from __future__ import annotations - -import json -from typing import Any, Generator - -from data_formulator.analyst.skills.base import Event, SkillContext, ToolResult -from data_formulator.data_operations import ( - ConnectorQueryStep, - DataDiscoveryService, - DataOperation, - DataOperationExecutor, - DataOperationPlan, - DataOperationRepository, - LoadQuery, - ProbeBudget, -) - -_PROBE_BUDGET_KEY = "data_loading.probe_budget" -_CONNECTORS_LISTED_KEY = "data_loading.connectors_listed" -_CONNECTORS_DISABLED_NOTE = ( - "External data connectors are disabled in this deployment. Use file upload " - "or built-in sample datasets instead." -) - - -class DataLoadingSkill: - """Read-only connected-source discovery for the unified analyst.""" - - def handle_tool( - self, - name: str, - args: dict[str, Any], - ctx: SkillContext, - ) -> ToolResult: - service = DataDiscoveryService(ctx.workspace) - if name == "list_data": - result = service.list_data(args) - elif name == "find_data": - result = service.find_data(args) - elif name == "describe_data": - result = service.describe_data(args) - elif name == "probe_data": - result = service.probe_data(args, self._probe_budget(ctx)) - elif name == "list_connectors": - result = self._list_connectors(ctx) - elif name == "describe_connector": - result = self._describe_connector(args) - else: - result = {"error": f"data-loading has no tool '{name}'."} - return ToolResult(text=json.dumps(result, ensure_ascii=False, default=str)) - - def handle_action( - self, - action: str, - spec: dict[str, Any], - ctx: SkillContext, - ) -> Generator[Event, None, str | None]: - if action == "propose_data_operation": - return (yield from self._propose_data_operation(spec, ctx)) - if action == "propose_connection": - return (yield from self._propose_connection(spec, ctx)) - message = f"data-loading has no committing action '{action}' in this phase." - yield { - "type": "error", - "message": message, - "message_code": "agent.unknownAction", - } - return message - - @staticmethod - def _connectors_disabled() -> bool: - try: - from flask import current_app - return bool(current_app.config.get("CLI_ARGS", {}).get("disable_data_connectors")) - except Exception: - return False - - @staticmethod - def _skill_state(ctx: SkillContext) -> dict[str, Any]: - state = ctx.payload.get("skill_state") - if not isinstance(state, dict): - state = {} - ctx.payload["skill_state"] = state - return state - - def _list_connectors(self, ctx: SkillContext) -> dict[str, Any]: - self._skill_state(ctx)[_CONNECTORS_LISTED_KEY] = True - if self._connectors_disabled(): - return {"connectors": [], "unavailable": [], "note": _CONNECTORS_DISABLED_NOTE} - - from data_formulator.data_loader import DATA_LOADERS, DISABLED_LOADERS - - connectors = [] - for key, loader_class in DATA_LOADERS.items(): - if key == "sample_datasets": - continue - try: - auth_mode = loader_class.auth_mode() - except Exception: - auth_mode = None - connectors.append({ - "type": key, - "name": loader_class.DISPLAY_NAME or key.replace("_", " ").title(), - "summary": loader_class.DESCRIPTION or "", - "auth_mode": auth_mode, - "available": True, - }) - return { - "connectors": connectors, - "unavailable": [ - { - "type": key, - "name": key.replace("_", " ").title(), - "install_hint": hint, - } - for key, hint in DISABLED_LOADERS.items() - if key != "sample_datasets" - ], - "next_action": ( - "If the user requested one of these connector types, call " - "propose_connection now. Do not end the turn by saying you will open a form." - ), - } - - def _describe_connector(self, args: dict[str, Any]) -> dict[str, Any]: - if self._connectors_disabled(): - return {"error": _CONNECTORS_DISABLED_NOTE} - - from data_formulator.data_loader import DATA_LOADERS, DISABLED_LOADERS - - source_type = str(args.get("source_type") or "").strip() - loader_class = DATA_LOADERS.get(source_type) - if loader_class is None: - hint = DISABLED_LOADERS.get(source_type) - detail = f" (needs: {hint})" if hint else "" - return {"error": f"Connector {source_type!r} is unavailable{detail}. Call list_connectors."} - - def safe(callable_): - try: - return callable_() - except Exception: - return None - - return { - "type": source_type, - "name": loader_class.DISPLAY_NAME or source_type.replace("_", " ").title(), - "summary": loader_class.DESCRIPTION or "", - "auth_mode": safe(loader_class.auth_mode), - "auth_paths": safe(loader_class.auth_paths), - "auth_instructions": safe(loader_class.auth_instructions), - "params": [ - { - "name": param.get("name"), - "required": bool(param.get("required")), - "tier": param.get("tier"), - "sensitive": bool(param.get("sensitive") or param.get("type") == "password"), - "description": param.get("description"), - } - for param in (safe(loader_class.list_params) or []) - if isinstance(param, dict) - ], - "next_action": ( - "Call propose_connection now to open this form. Describing the " - "requirements in text does not open it." - ), - } - - def _propose_connection( - self, - spec: dict[str, Any], - ctx: SkillContext, - ) -> Generator[Event, None, str | None]: - if self._connectors_disabled(): - yield {"type": "error", "message": _CONNECTORS_DISABLED_NOTE, "message_code": "agent.connectorsDisabled"} - return _CONNECTORS_DISABLED_NOTE - if not self._skill_state(ctx).get(_CONNECTORS_LISTED_KEY): - message = "Call list_connectors before propose_connection." - yield {"type": "error", "message": message, "message_code": "agent.invalidConnector"} - return message - - from data_formulator.data_loader import DATA_LOADERS, DISABLED_LOADERS - - source_type = str(spec.get("source_type") or "").strip() - if source_type not in DATA_LOADERS or source_type == "sample_datasets": - hint = DISABLED_LOADERS.get(source_type) - message = f"Connector {source_type!r} is unavailable" + (f" (needs: {hint})." if hint else ".") - yield {"type": "error", "message": message, "message_code": "agent.invalidConnector"} - return message - - prefilled_raw = spec.get("prefilled") or {} - prefilled = {} - if isinstance(prefilled_raw, dict): - prefilled = { - str(key): str(value) - for key, value in prefilled_raw.items() - if value not in (None, "") - } - display_name = DATA_LOADERS[source_type].DISPLAY_NAME or source_type - response = str(ctx.payload.get("action_narration") or "").strip() - yield { - "type": "interact", - "thought": spec.get("thought", ""), - "form": { - "kind": "connector", - "title": f"Connect to {display_name}", - "response": response or f"Complete the {display_name} connection form to add this data source.", - "connector": { - "source_type": source_type, - "prefilled": prefilled, - }, - }, - } - return None - - @staticmethod - def _already_loaded_tables(steps: tuple[ConnectorQueryStep, ...], workspace) -> list[str]: - metadata = workspace.get_metadata() - if metadata is None: - return [] - loaded: list[str] = [] - for step in steps: - expected_options = DataOperationExecutor._build_import_options(step) - for table_name, table_metadata in metadata.tables.items(): - if table_metadata.source_table != step.source_table: - continue - import_options = dict(table_metadata.import_options or {}) - provenance = import_options.pop("data_operation", {}) - same_source = not provenance or ( - provenance.get("source_id") in (None, step.source_id) - and provenance.get("table_key") in (None, step.table_key) - ) - if same_source and import_options == expected_options: - loaded.append(table_name) - break - return loaded - - @staticmethod - def _propose_data_operation( - spec: dict[str, Any], - ctx: SkillContext, - ) -> Generator[Event, None, str | None]: - try: - raw_plans = spec.get("options") - if not isinstance(raw_plans, list) or not 1 <= len(raw_plans) <= 3: - raise ValueError("propose_data_operation requires one to three options") - discovery = DataDiscoveryService(ctx.workspace) - resolved_plans: list[DataOperationPlan] = [] - for raw_plan in raw_plans: - raw_steps = raw_plan.get("tables") - if not isinstance(raw_steps, list) or not raw_steps: - raise ValueError("Each loading option requires at least one table") - steps: list[ConnectorQueryStep] = [] - for raw_step in raw_steps: - source_id = str(raw_step["source_id"]) - table_key = str(raw_step["table_key"]) - if not _source_is_available(source_id): - raise ValueError( - f"source {source_id!r} is not connected, so it cannot be loaded from. " - "Propose data from a connected source, or tell the user to reconnect it first." - ) - resolved = discovery.resolve_load_table(source_id, table_key) - if resolved is None: - raise ValueError( - f"table_key {table_key!r} was not found in source {source_id!r}" - ) - steps.append(ConnectorQueryStep( - source_id=source_id, - table_key=table_key, - display_name=str(resolved["display_name"]), - source_table=str(resolved["source_table"]), - source_table_name=( - str(resolved["source_table_name"]) - if resolved.get("source_table_name") is not None - else None - ), - query=LoadQuery.from_dict(raw_step.get("query")), - )) - resolved_plans.append(DataOperationPlan( - label=str(raw_plan["label"]).strip(), - summary="", - steps=tuple(steps), - )) - plans = tuple( - resolved_plans - ) - # The agent's own prose is the answer; `response` is only a fallback - # for models that emit a bare tool call with no accompanying text. - narration = str(ctx.payload.get("action_narration") or "").strip() - response = narration or str(spec.get("response", "")).strip() - operation = DataOperation( - reason="", - plans=plans, - description=response, - ) - if not operation.description or any(not plan.label for plan in plans): - raise ValueError( - "say what you found and why in your reply text, and give each option a label" - ) - conversation_id = str(ctx.payload.get("conversation_id", "")).strip() - loaded_tables = DataLoadingSkill._already_loaded_tables( - tuple(step for plan in plans for step in plan.steps), - ctx.workspace, - ) - if loaded_tables: - names = ", ".join(dict.fromkeys(loaded_tables)) - raise ValueError( - f"This proposal duplicates data already loaded in the workspace: {names}. " - "Use those workspace tables directly, explain their relevance, or propose only missing data." - ) - DataOperationRepository.for_workspace(ctx.workspace).create( - operation, - conversation_id=conversation_id, - ) - except (KeyError, TypeError, ValueError) as exc: - message = str(exc) - yield { - "type": "error", - "message": message, - "message_code": "agent.invalidDataOperation", - } - return message - - yield { - "type": "interact", - "thought": spec.get("thought", ""), - "data_operation": operation.to_public_dict(), - "questions": [{ - "text": operation.description, - "responseType": "single_choice", - "required": True, - "options": [ - {"label": plan.label, "value": plan.id} - for plan in operation.plans - ], - }], - } - return None - - @staticmethod - def _probe_budget(ctx: SkillContext) -> ProbeBudget: - state = ctx.payload.get("skill_state") - if not isinstance(state, dict): - state = {} - ctx.payload["skill_state"] = state - budget = state.get(_PROBE_BUDGET_KEY) - if not isinstance(budget, ProbeBudget): - budget = ProbeBudget() - state[_PROBE_BUDGET_KEY] = budget - return budget - - -def _source_is_available(source_id: str) -> bool: - """Only False when we can positively tell the source is unreachable.""" - try: - from data_formulator.data_connector import connector_is_available - return connector_is_available(source_id) is not False - except Exception: - return True - - -def get_skill() -> DataLoadingSkill: - return DataLoadingSkill() \ No newline at end of file diff --git a/py-src/data_formulator/analyst/skills/data_loading/tools.json b/py-src/data_formulator/analyst/skills/data_loading/tools.json deleted file mode 100644 index 30db03417..000000000 --- a/py-src/data_formulator/analyst/skills/data_loading/tools.json +++ /dev/null @@ -1,233 +0,0 @@ -[ - { - "type": "function", - "function": { - "name": "list_data", - "description": "Browse cached connected-source catalogs. With no arguments, list source summaries. With source_id, list its top-level entries. Add path to browse direct children and filter for a case-insensitive substring match.", - "parameters": { - "type": "object", - "properties": { - "source_id": { "type": "string", "description": "Connected source identifier. Omit for source summaries." }, - "path": { "type": "array", "items": { "type": "string" }, "description": "Hierarchy path segments." }, - "filter": { "type": "string", "description": "Substring filter on direct children." } - }, - "required": [] - } - } - }, - { - "type": "function", - "function": { - "name": "find_data", - "description": "Regex search across cached connected-source catalogs and optionally existing workspace tables. Returns exact source_id and table_key values for follow-up inspection.", - "parameters": { - "type": "object", - "properties": { - "query": { "type": "string", "description": "Case-insensitive regex. Plain keywords work as literals." }, - "scope": { "type": "string", "description": "all, workspace, connected, a source_id, or source_id:path/segments." }, - "exclude": { "type": "string", "description": "Optional table-name exclusion regex." }, - "fields": { - "type": "array", - "items": { "type": "string", "enum": ["name", "description", "columns"] }, - "description": "Fields to search. Omit for all." - }, - "limit": { "type": "integer" } - }, - "required": ["query"] - } - } - }, - { - "type": "function", - "function": { - "name": "describe_data", - "description": "Read cached metadata, columns, types, description, and row count for one discovered table.", - "parameters": { - "type": "object", - "properties": { - "source_id": { "type": "string" }, - "table_key": { "type": "string" } - }, - "required": ["source_id", "table_key"] - } - } - }, - { - "type": "function", - "function": { - "name": "probe_data", - "description": "Run a bounded read-only structured query against one connected table. Use only after describe_data. Results are evidence for planning and do not become workspace inputs.", - "parameters": { - "type": "object", - "properties": { - "source_id": { "type": "string" }, - "table_key": { "type": "string" }, - "query": { - "type": "object", - "properties": { - "filters": { - "type": "array", - "items": { - "type": "object", - "properties": { - "column": { "type": "string" }, - "op": { "type": "string", "enum": ["EQ", "NEQ", "GT", "GTE", "LT", "LTE", "IN", "ILIKE", "BETWEEN", "IS_NULL"] }, - "value": {} - }, - "required": ["column", "op"] - } - }, - "columns": { "type": "array", "items": { "type": "string" } }, - "group_by": { "type": "array", "items": { "type": "string" } }, - "aggregates": { - "type": "array", - "items": { - "type": "object", - "properties": { - "op": { "type": "string", "enum": ["count", "count_distinct", "sum", "avg", "min", "max"] }, - "column": { "type": "string" }, - "as": { "type": "string" } - }, - "required": ["op"] - } - }, - "order_by": { - "type": "array", - "items": { - "type": "object", - "properties": { - "column": { "type": "string" }, - "dir": { "type": "string", "enum": ["asc", "desc"] } - }, - "required": ["column"] - } - }, - "limit": { "type": "integer" } - } - } - }, - "required": ["source_id", "table_key"] - } - } - }, - { - "type": "function", - "function": { - "name": "list_connectors", - "description": "List connector types available in this deployment. Call this before propose_connection because built-ins, plugins, and missing dependencies vary by deployment. If the user's requested type is present, you MUST call propose_connection in the same turn; do not merely say you will open a form.", - "parameters": { - "type": "object", - "properties": {} - } - } - }, - { - "type": "function", - "function": { - "name": "describe_connector", - "description": "Return setup fields and authentication choices for one source_type returned by list_connectors. After this, call propose_connection in the same turn; describing fields does not open the form.", - "parameters": { - "type": "object", - "properties": { - "source_type": { "type": "string", "description": "Connector type key returned by list_connectors." } - }, - "required": ["source_type"] - } - } - }, - { - "type": "function", - "function": { - "name": "propose_connection", - "description": "REQUIRED terminal action when the user wants an available connector and its source_type is known. This is the only operation that opens the user-confirmed add-connector form on the canvas. Call list_connectors first. Prefill only values the user supplied; never invent credentials or connect automatically.", - "parameters": { - "type": "object", - "properties": { - "source_type": { "type": "string", "description": "Connector type key returned by list_connectors." }, - "prefilled": { - "type": "object", - "description": "Optional connector field values already supplied by the user. Values seed the live form and must not be repeated in prose.", - "additionalProperties": {} - } - }, - "required": ["source_type"] - } - } - }, - { - "type": "function", - "function": { - "name": "propose_data_operation", - "description": "Offer one to three complete immutable connected-data loading alternatives and pause for the user's selection. Discovery must ground every source, table, filter, and sort field. This does not execute a load.", - "parameters": { - "type": "object", - "properties": { - "response": { - "type": "string", - "description": "Fallback only. Leave empty when you narrate in your message text, which is what the user reads." - }, - "options": { - "type": "array", - "minItems": 1, - "maxItems": 3, - "items": { - "type": "object", - "properties": { - "label": { - "type": "string", - "description": "Concise action label, ideally 2-6 words." - }, - "tables": { - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "properties": { - "source_id": { "type": "string" }, - "table_key": { "type": "string" }, - "query": { - "type": "object", - "description": "Optional raw-row subset. Omit to load the whole table subject to server limits.", - "properties": { - "filters": { - "type": "array", - "items": { - "type": "object", - "properties": { - "column": { "type": "string" }, - "op": { "type": "string", "enum": ["EQ", "NEQ", "GT", "GTE", "LT", "LTE", "IN", "ILIKE", "BETWEEN", "IS_NULL"] }, - "value": {} - }, - "required": ["column", "op"] - } - }, - "columns": { "type": "array", "items": { "type": "string" } }, - "order_by": { - "type": "array", - "maxItems": 1, - "items": { - "type": "object", - "properties": { - "column": { "type": "string" }, - "dir": { "type": "string", "enum": ["asc", "desc"] } - }, - "required": ["column"] - } - }, - "limit": { "type": "integer", "minimum": 1 } - } - } - }, - "required": ["source_id", "table_key"] - } - } - }, - "required": ["label", "tables"] - } - } - }, - "required": ["options"] - } - } - } -] \ No newline at end of file diff --git a/py-src/data_formulator/analyst/skills/report/SKILL.md b/py-src/data_formulator/analyst/skills/report/SKILL.md index 1397fcd52..245fde154 100644 --- a/py-src/data_formulator/analyst/skills/report/SKILL.md +++ b/py-src/data_formulator/analyst/skills/report/SKILL.md @@ -38,7 +38,7 @@ all chart/data inspection first — once you call `write_report`, the report is delivered as-is and the run ends. ## Context available to you -- **[PRIMARY TABLE(S)]** / **[OTHER AVAILABLE TABLES]**: Lightweight schema of datasets. +- **[PRIMARY ANALYSIS INPUTS]** / **[OTHER ANALYSIS INPUTS]**: Lightweight schema of materialized input datasets. - **[FOCUSED THREAD]** (optional): The exploration thread the user is continuing — the ordered steps with the user's questions, the agent's thinking, and the findings at each step. This is the spine of the story you are telling. diff --git a/py-src/data_formulator/analyst/tools.py b/py-src/data_formulator/analyst/tools.py index cde1f34f0..854c595ac 100644 --- a/py-src/data_formulator/analyst/tools.py +++ b/py-src/data_formulator/analyst/tools.py @@ -9,7 +9,7 @@ - ``execute_python_script`` — run a general-purpose Python script in the sandbox to inspect/compute (stdout returned). - - ``inspect_source_data`` — schema + stats + sample rows for source tables. + - ``inspect_source_data`` — schema + stats + sample rows for analysis inputs. - ``load_skill`` — pull a skill's ``SKILL.md`` body into context, unlocking its gated actions (progressive disclosure; reading a doc is read-only). @@ -53,8 +53,8 @@ "function": { "name": "inspect_source_data", "description": ( - "Get a detailed summary of one or more source tables — schema, " - "field-level statistics, and sample rows. Cheaper than explore() " + "Get a detailed summary of one or more analysis input tables — schema, " + "field-level statistics, and sample rows. Cheaper than explore() " "for basic data inspection." ), "parameters": { @@ -63,7 +63,7 @@ "table_names": { "type": "array", "items": {"type": "string"}, - "description": "List of workspace table names, as listed in the available-tables context, to inspect.", + "description": "Names listed in the analysis-input-tables context to inspect.", }, }, "required": ["table_names"], diff --git a/py-src/data_formulator/data_connector.py b/py-src/data_formulator/data_connector.py index 49fd208d0..0b4510674 100644 --- a/py-src/data_formulator/data_connector.py +++ b/py-src/data_formulator/data_connector.py @@ -762,15 +762,16 @@ def _try_sso_auto_connect(self, identity: str) -> ExternalDataLoader | None: def _require_loader(self) -> ExternalDataLoader: identity = self._get_identity() + from data_formulator.datalake.connector_preferences import connector_is_enabled + from data_formulator.datalake.workspace import get_user_home + if not connector_is_enabled(get_user_home(identity), self._source_id): + raise ValueError("Connector is disconnected. Please connect first.") loader = self._loaders.get(identity) if loader is not None: return loader - # No-auth connectors (e.g. built-in example datasets) are always - # available — there's nothing to connect, so lazily instantiate and - # cache the loader on first use. This mirrors the ``auth_mode == "none"`` - # special-casing in the connect/get-status/preview/import endpoints and - # keeps no-auth sources working for catalog/preview/import even when - # external data connectors are disabled (e.g. ephemeral/demo mode). + # Enabled no-auth connectors need no setup, so lazily instantiate and + # cache the loader on first use. The preference check above keeps a + # user-disconnected built-in unavailable to both UI and agent paths. if _loader_auth_mode(self._loader_class) == "none": loader = self._loader_class() self._loaders[identity] = loader @@ -845,6 +846,45 @@ def resolve_catalog_refresh_target( return loader_class, loader +def _connector_connection_status( + connector: DataConnector, + identity: str | None, + *, + sso_token: Any = None, + token_store: Any = None, +) -> tuple[bool, bool, bool]: + """Return ``(connected, has_stored_credentials, sso_auto_connect)``.""" + enabled = True + if identity: + from data_formulator.datalake.connector_preferences import connector_is_enabled + from data_formulator.datalake.workspace import get_user_home + enabled = connector_is_enabled(get_user_home(identity), connector._source_id) + if not enabled: + return False, False, False + + auth_mode = _loader_auth_mode(connector._loader_class) + if auth_mode == "none": + return True, False, False + if not identity: + return False, False, False + + has_stored = connector.has_stored_credentials(identity) + connected = connector._get_loader(identity) is not None or has_stored + if connected: + return True, has_stored, False + + sso_auto = False + if sso_token is not None and auth_mode in ("token", "sso_exchange", "delegated"): + if token_store is None: + from data_formulator.auth.token_store import TokenStore + token_store = TokenStore() + sso_auto = ( + not token_store.is_sso_reconnect_blocked(connector._source_id) + and bool(connector._default_params.get("url")) + ) + return False, has_stored, sso_auto + + def connector_is_available(source_id: str) -> bool | None: """Whether ``source_id`` could be loaded from right now, without touching it. @@ -859,26 +899,55 @@ def connector_is_available(source_id: str) -> bool | None: except Exception: return None try: - if _loader_auth_mode(connector._loader_class) == "none": - return True identity = connector._get_identity() - if connector._get_loader(identity) is not None: - return True - if connector.has_stored_credentials(identity): - return True from data_formulator.auth.identity import get_sso_token - from data_formulator.auth.token_store import TokenStore - auth_mode = _loader_auth_mode(connector._loader_class) - return ( - auth_mode in ("token", "sso_exchange", "delegated") - and not TokenStore().is_sso_reconnect_blocked(source_id) - and get_sso_token() is not None + connected, _has_stored, sso_auto = _connector_connection_status( + connector, + identity, + sso_token=get_sso_token(), ) + return connected or sso_auto except Exception: logger.debug("availability check failed for %s", source_id, exc_info=True) return None +def list_available_connector_ids() -> list[str]: + """Return connector IDs the current identity can load from.""" + try: + identity = DataConnector._get_identity() + except Exception: + return [] + + sso_token = None + token_store = None + try: + from data_formulator.auth.identity import get_sso_token + sso_token = get_sso_token() + if sso_token is not None: + from data_formulator.auth.token_store import TokenStore + token_store = TokenStore() + except Exception: + logger.debug("SSO status unavailable for connector inventory", exc_info=True) + + available: list[str] = [] + for registry_key, connector, _is_admin in _visible_connector_items(identity): + public_id = _public_connector_id(registry_key, connector) + try: + connected, _has_stored, sso_auto = _connector_connection_status( + connector, + identity, + sso_token=sso_token, + token_store=token_store, + ) + except Exception: + logger.debug("availability check failed for %s", public_id, exc_info=True) + continue + if connected or sso_auto: + available.append(public_id) + return available + + def _parse_source_table(raw: Any) -> tuple[str, str]: """Normalise the ``source_table`` value from a request body. @@ -1274,31 +1343,11 @@ def list_connectors(): result = [] for registry_key, connector, is_admin in _visible_connector_items(identity): - has_stored = False - connected = False - auth_mode = _loader_auth_mode(connector._loader_class) - if auth_mode == "none": - # No-auth connectors (e.g. built-in example datasets) are always - # available — there's no credential to store and no connection - # to establish. - connected = True - elif identity: - has_stored = connector.has_stored_credentials(identity) - connected = ( - connector._get_loader(identity) is not None - or has_stored - ) - sso_blocked = ( - token_store.is_sso_reconnect_blocked(connector._source_id) - if token_store else False - ) - # SSO auto-connect: auth-capable loader + user has SSO token + URL is pinned - sso_auto = ( - not connected - and sso_token is not None - and auth_mode in ("token", "sso_exchange", "delegated") - and not sso_blocked - and bool(connector._default_params.get("url")) + connected, has_stored, sso_auto = _connector_connection_status( + connector, + identity, + sso_token=sso_token, + token_store=token_store, ) cfg = connector.get_frontend_config(include_pinned_in_form=not is_admin) public_id = _public_connector_id(registry_key, connector) @@ -1635,12 +1684,16 @@ def connector_connect(): data = request.get_json() or {} source = _resolve_connector(data) - # No-auth connectors (e.g. built-in example datasets) have nothing to - # connect — they're always available. Return a synthetic success - # response so any (legacy) frontend code that still calls connect is - # a no-op rather than an error. + identity = source._get_identity() + from data_formulator.datalake.connector_preferences import set_connector_enabled + from data_formulator.datalake.workspace import get_user_home + + # No-auth connectors have no form to submit. Connecting simply re-enables + # access to the existing loader and preserved catalog. if _loader_auth_mode(source._loader_class) == "none": + set_connector_enabled(get_user_home(identity), source._source_id, True) loader = source._loader_class() + source._loaders[identity] = loader return json_ok({ "status": "connected", "persisted": False, @@ -1675,6 +1728,8 @@ def connector_connect(): source._loaders.pop(identity, None) raise AppError(ErrorCode.DB_CONNECTION_FAILED, "Connection test failed") + set_connector_enabled(get_user_home(identity), source._source_id, True) + persisted = False if persist: persisted = source._persist_credentials(user_params) @@ -1757,19 +1812,14 @@ def connector_disconnect(): data = request.get_json() or {} source = _resolve_connector(data) - # No-auth connectors (e.g. built-in example datasets) cannot be - # disconnected — they have no credentials to clear and are intentionally - # always available. - if _loader_auth_mode(source._loader_class) == "none": - raise AppError( - ErrorCode.INVALID_REQUEST, - "This connector is always available and cannot be disconnected.", - ) - try: identity = source._get_identity() + from data_formulator.datalake.connector_preferences import set_connector_enabled + from data_formulator.datalake.workspace import get_user_home + set_connector_enabled(get_user_home(identity), source._source_id, False) source._loaders.pop(identity, None) - source._vault_delete(identity) + if _loader_auth_mode(source._loader_class) != "none": + source._vault_delete(identity) try: from data_formulator.auth.token_store import TokenStore TokenStore().clear_service_token(source._source_id) @@ -1792,8 +1842,14 @@ def connector_get_status(): data = request.get_json() or {} source = _resolve_connector(data) - # No-auth connectors are always connected. + identity = source._get_identity() + from data_formulator.datalake.connector_preferences import connector_is_enabled + from data_formulator.datalake.workspace import get_user_home + if _loader_auth_mode(source._loader_class) == "none": + enabled = connector_is_enabled(get_user_home(identity), source._source_id) + if not enabled: + return json_ok({"connected": False, "persisted": False}) loader = source._loader_class() return json_ok({ "connected": True, diff --git a/py-src/data_formulator/data_loader/sample_datasets_loader.py b/py-src/data_formulator/data_loader/sample_datasets_loader.py index 59678f60a..4a767564f 100644 --- a/py-src/data_formulator/data_loader/sample_datasets_loader.py +++ b/py-src/data_formulator/data_loader/sample_datasets_loader.py @@ -61,10 +61,9 @@ def list_params() -> list[dict[str, Any]]: @staticmethod def auth_mode() -> str: - # ``"none"`` declares that this loader needs no authentication and no - # connection setup. The connector framework treats such loaders as - # always-on: they cannot be connected/disconnected, expose no - # credentials UI, and are always reported as ``connected: true``. + # ``"none"`` declares that this loader needs no authentication or + # connection form. Users can still disable its availability through + # the connector preference managed by the framework. return "none" @staticmethod diff --git a/py-src/data_formulator/data_operations/discovery.py b/py-src/data_formulator/data_operations/discovery.py index f0af874f4..d5ba5a391 100644 --- a/py-src/data_formulator/data_operations/discovery.py +++ b/py-src/data_formulator/data_operations/discovery.py @@ -50,11 +50,25 @@ def ensure_catalogs_current(user_home: Any) -> dict[str, Any]: return {} snapshots: dict[str, Any] = {} try: - from data_formulator.data_connector import _ADMIN_CONNECTOR_IDS + from data_formulator.data_connector import ( + _ADMIN_CONNECTOR_IDS, + connector_is_available, + list_available_connector_ids, + ) from data_formulator.datalake.catalog_cache import list_cached_sources + from data_formulator.datalake.connector_preferences import connector_is_enabled from data_formulator.datalake.catalog_refresh import ensure_catalog_freshness - source_ids = set(list_cached_sources(user_home)) | set(_ADMIN_CONNECTOR_IDS) + source_ids = ( + set(list_cached_sources(user_home)) + | set(_ADMIN_CONNECTOR_IDS) + | set(list_available_connector_ids()) + ) + source_ids = { + source_id for source_id in source_ids + if connector_is_enabled(user_home, source_id) + and connector_is_available(source_id) is not False + } for source_id in source_ids: snapshot = ensure_catalog_freshness(Path(user_home), source_id) if snapshot is not None: @@ -79,6 +93,16 @@ def _freshness_payload(snapshot: Any) -> dict[str, Any]: } +def _source_is_discoverable(source_id: str) -> bool: + """Hide sources known to be disconnected; keep unknown status compatible.""" + try: + from data_formulator.data_connector import connector_is_available + return connector_is_available(source_id) is not False + except Exception: + logger.debug("Connector availability unavailable for %s", source_id, exc_info=True) + return True + + class DataDiscoveryService: """Read-only catalog discovery shared by data-loading entry points.""" @@ -103,10 +127,29 @@ def list_data(self, args: dict[str, Any]) -> dict[str, Any]: except Exception: logger.debug("list_data: list_sources_summary failed", exc_info=True) return {"sources": []} - # Mark unreachable sources so the agent steers around them instead - # of proposing a load that can only fail. try: - from data_formulator.data_connector import connector_is_available + from data_formulator.data_connector import list_available_connector_ids + summarized_ids = { + source.get("source_id") or source.get("id") + for source in sources + } + sources.extend({ + "source_id": source_id, + "table_count": 0, + "is_hierarchical": False, + "connected": True, + "catalog_status": "not_cached", + } for source_id in list_available_connector_ids() if source_id not in summarized_ids) + except Exception: + logger.debug("list_data: available connector inventory failed", exc_info=True) + # A retained catalog is storage, not connection state. Do not offer + # sources that are definitively unavailable to the current identity. + sources = [ + source for source in sources + if not (source.get("source_id") or source.get("id")) + or _source_is_discoverable(source.get("source_id") or source.get("id")) + ] + try: for source in sources: sid = source.get("source_id") or source.get("id") if sid in snapshots and ( @@ -115,12 +158,27 @@ def list_data(self, args: dict[str, Any]) -> dict[str, Any]: or snapshots[sid].last_refresh_error ): source["freshness"] = _freshness_payload(snapshots[sid]) - if sid and connector_is_available(sid) is False: - source["connected"] = False except Exception: - logger.debug("list_data: availability check failed", exc_info=True) + logger.debug("list_data: freshness annotation failed", exc_info=True) return {"sources": sources} + from data_formulator.datalake.connector_preferences import connector_is_enabled + if not connector_is_enabled(user_home, source_id) or not _source_is_discoverable(source_id): + return {"error": f"Source '{source_id}' is disconnected."} + + from data_formulator.datalake.catalog_cache import list_cached_sources + if source_id not in set(list_cached_sources(user_home)): + try: + from data_formulator.data_connector import resolve_live_loader + from data_formulator.datalake.catalog_refresh import ensure_catalog_freshness + resolve_live_loader(source_id) + snapshot = ensure_catalog_freshness(user_home, source_id) + if snapshot is not None: + snapshots[source_id] = snapshot + except Exception as exc: + logger.debug("list_data: catalog bootstrap failed", exc_info=True) + return {"error": f"Source '{source_id}' is connected but its catalog could not be loaded: {exc}"} + path = args.get("path") or [] if not isinstance(path, list): return {"error": "path must be an array of strings"} @@ -198,6 +256,16 @@ def find_data(self, args: dict[str, Any]) -> dict[str, Any]: if source_ids != [] and user_home: try: + if source_ids is None: + source_ids = [ + source_id for source_id in list_cached_sources(user_home) + if _source_is_discoverable(source_id) + ] + else: + source_ids = [ + source_id for source_id in source_ids + if _source_is_discoverable(source_id) + ] imported_names = {result["name"] for result in results} cache_hits = search_catalog_cache( user_home, @@ -226,7 +294,11 @@ def find_data(self, args: dict[str, Any]) -> dict[str, Any]: if not results: try: - known = sorted(list_cached_sources(user_home) or []) if user_home else [] + known = sorted( + source_id + for source_id in (list_cached_sources(user_home) or []) + if _source_is_discoverable(source_id) + ) if user_home else [] except Exception: known = [] return { @@ -257,6 +329,11 @@ def describe_data(self, args: dict[str, Any]) -> dict[str, Any]: source_id = args.get("source_id", "") table_key = args.get("table_key", "") + user_home = getattr(self.workspace, "user_home", None) + if user_home: + from data_formulator.datalake.connector_preferences import connector_is_enabled + if not connector_is_enabled(user_home, source_id) or not _source_is_discoverable(source_id): + return {"error": f"Source '{source_id}' is disconnected."} return { "result": handle_read_catalog_metadata( source_id, diff --git a/py-src/data_formulator/datalake/catalog_cache.py b/py-src/data_formulator/datalake/catalog_cache.py index 68ea9810d..add84eebd 100644 --- a/py-src/data_formulator/datalake/catalog_cache.py +++ b/py-src/data_formulator/datalake/catalog_cache.py @@ -378,6 +378,12 @@ def list_cached_sources(workspace_root: Path | str) -> list[str]: sources = [s for s in sources if s in allowed] except Exception: logger.debug("Failed to filter cached sources by admin set", exc_info=True) + try: + from data_formulator.datalake.connector_preferences import disabled_connector_ids + disabled_sources = disabled_connector_ids(workspace_root) + sources = [source for source in sources if source not in disabled_sources] + except Exception: + logger.debug("Failed to filter disabled cached sources", exc_info=True) return sources @@ -518,7 +524,13 @@ def search_catalog_cache( return [] exclude = exclude_tables or set() - all_ids = source_ids or list_cached_sources(workspace_root) + all_ids = source_ids if source_ids is not None else list_cached_sources(workspace_root) + try: + from data_formulator.datalake.connector_preferences import disabled_connector_ids + disabled_sources = disabled_connector_ids(workspace_root) + all_ids = [source_id for source_id in all_ids if source_id not in disabled_sources] + except Exception: + logger.debug("Failed to filter disabled catalog search sources", exc_info=True) # Compile exclude pattern up-front so a bad pattern surfaces clearly. excl_re = None @@ -551,15 +563,62 @@ def search_catalog_cache( # agent toward find_data or a tighter filter rather than pagination. LIST_DATA_LIMIT = 200 +# Enough top-level names to answer "what's in there?" without a drill-down call. +SOURCE_TOP_LEVEL_PREVIEW = 12 + +# Descendant nodes returned alongside one level of children, so a source's shape +# is visible without walking it folder by folder. +SUBTREE_NODE_BUDGET = 150 + + +def _build_subtree( + tables_raw: list[dict[str, Any]], + path: list[str], + budget: int = SUBTREE_NODE_BUDGET, +) -> tuple[dict[str, Any], bool]: + """Nested names below ``path``, one level down and deeper. + + Folders map to objects, tables to ``None``. Only descendants deeper than the + requested level appear — tables *at* the level are already returned in full + (with keys and descriptions) by :func:`list_path_children`. + """ + K = len(path) + tree: dict[str, Any] = {} + nodes = 0 + for t in tables_raw: + tpath = t.get("path") + tpath = [str(s) for s in tpath] if isinstance(tpath, list) else [] + if len(tpath) < K + 2 or tpath[:K] != path: + continue + rest = [seg for seg in tpath[K:] if seg] + if len(rest) < 2: + continue + if nodes >= budget: + return tree, True + node = tree + for seg in rest[:-1]: + child = node.get(seg) + if not isinstance(child, dict): + child = {} + node[seg] = child + nodes += 1 + node = child + if rest[-1] not in node: + node[rest[-1]] = None + nodes += 1 + return tree, False + def list_sources_summary( workspace_root: Path | str, ) -> list[dict[str, Any]]: """Return a per-source summary suitable for ``list_data()`` with no args. - Each entry: ``{source_id, table_count, is_hierarchical}``. Sources whose - cache file is missing or unreadable are skipped silently — the agent - treats the cache as ground truth (see design-docs §8). + Each entry: ``{source_id, table_count, is_hierarchical, top_level}``, where + ``top_level`` previews the source's depth-0 children (folders first, then + loose tables) so the inventory alone usually answers what a source holds. + Sources whose cache file is missing or unreadable are skipped silently — the + agent treats the cache as ground truth (see design-docs §8). """ out: list[dict[str, Any]] = [] for sid in list_cached_sources(workspace_root): @@ -568,15 +627,26 @@ def list_sources_summary( continue tables = raw.get("tables", []) or [] is_hier = False + folders: list[str] = [] + seen_folders: set[str] = set() + leaves: list[str] = [] for t in tables: p = t.get("path") - if isinstance(p, list) and len(p) >= 2: + p = [str(s) for s in p] if isinstance(p, list) else [] + if len(p) >= 2: is_hier = True - break + if p[0] not in seen_folders: + seen_folders.add(p[0]) + folders.append(p[0]) + else: + leaf = p[0] if p else str(t.get("name", "")) + if leaf: + leaves.append(leaf) out.append({ "source_id": raw.get("source_id", sid), "table_count": len(tables), "is_hierarchical": is_hier, + "top_level": (folders + leaves)[:SOURCE_TOP_LEVEL_PREVIEW], }) out.sort(key=lambda r: r["source_id"]) return out @@ -704,6 +774,11 @@ def list_path_children( "total_tables": total_tables, "truncated": truncated, } + subtree, subtree_truncated = _build_subtree(tables_raw, path) + if subtree: + result["tree"] = subtree + if subtree_truncated: + result["tree_truncated"] = True if truncated: remaining = total - len(folders) - len(leaf_tables) result["hint"] = ( diff --git a/py-src/data_formulator/datalake/connector_preferences.py b/py-src/data_formulator/datalake/connector_preferences.py new file mode 100644 index 000000000..db4eca6ba --- /dev/null +++ b/py-src/data_formulator/datalake/connector_preferences.py @@ -0,0 +1,60 @@ +"""Per-user connector availability preferences.""" + +from __future__ import annotations + +import json +import logging +import os +from pathlib import Path +from threading import Lock +from uuid import uuid4 + +from data_formulator.security.path_safety import ConfinedDir + +logger = logging.getLogger(__name__) + +_PREFERENCES_FILE = "connector_preferences.json" +_PREFERENCES_LOCK = Lock() + + +def disabled_connector_ids(user_home: Path | str) -> set[str]: + jail = ConfinedDir(user_home, mkdir=False) + if not jail.exists(_PREFERENCES_FILE): + return set() + try: + raw = json.loads(jail.read_text(_PREFERENCES_FILE)) + values = raw.get("disabled_connector_ids", []) if isinstance(raw, dict) else [] + return {value for value in values if isinstance(value, str) and value} + except Exception: + logger.warning("Failed to read connector preferences", exc_info=True) + return set() + + +def connector_is_enabled(user_home: Path | str, source_id: str) -> bool: + return source_id not in disabled_connector_ids(user_home) + + +def set_connector_enabled( + user_home: Path | str, + source_id: str, + enabled: bool, +) -> None: + jail = ConfinedDir(user_home, mkdir=True) + with _PREFERENCES_LOCK: + disabled = disabled_connector_ids(user_home) + if enabled: + disabled.discard(source_id) + else: + disabled.add(source_id) + + target = jail.resolve(_PREFERENCES_FILE) + temporary = jail.resolve(f".{_PREFERENCES_FILE}.{os.getpid()}.{uuid4().hex}.tmp") + try: + with open(temporary, "w", encoding="utf-8") as file: + json.dump({"disabled_connector_ids": sorted(disabled)}, file) + file.flush() + os.fsync(file.fileno()) + os.replace(temporary, target) + finally: + if temporary.exists(): + temporary.unlink() \ No newline at end of file diff --git a/py-src/data_formulator/routes/agents.py b/py-src/data_formulator/routes/agents.py index 23b49aafd..eec77fc0e 100644 --- a/py-src/data_formulator/routes/agents.py +++ b/py-src/data_formulator/routes/agents.py @@ -135,6 +135,8 @@ def preview_data_operation(): "source_id": step.source_id, **({"table_description": str(table_description).strip()} if table_description else {}), "error": str(exc), + "columns": [], + "rows": [], }) continue previews.append({ diff --git a/src/app/dfSlice.tsx b/src/app/dfSlice.tsx index 9837857b1..c77f380f5 100644 --- a/src/app/dfSlice.tsx +++ b/src/app/dfSlice.tsx @@ -2346,8 +2346,17 @@ export const dataFormulatorSlice = createSlice({ }; } - const displayName = data["result"][0]["suggested_table_name"] as string | undefined; - const info = { tableId, ...(displayName ? { displayName } : {}), fields }; + const suggestedName = data["result"][0]["suggested_table_name"] as string | undefined; + const normalizeName = (name: string) => name.toLowerCase().replace(/[\s_-]+/g, ''); + if (suggestedName && normalizeName(table.displayId || table.id) === normalizeName(table.id)) { + state.inputTables = state.inputTables.map(item => + item.id === tableId ? { ...item, displayId: suggestedName } : item + ); + state.derivedTables = state.derivedTables.map(item => + item.id === tableId ? { ...item, displayId: suggestedName } : item + ); + } + const info = { tableId, fields }; const existingIndex = state.tableSemantics.findIndex(item => item.tableId === tableId); if (existingIndex >= 0) state.tableSemantics[existingIndex] = info; else state.tableSemantics.push(info); diff --git a/src/app/stateMigrations.ts b/src/app/stateMigrations.ts index 7c4134340..618226b6b 100644 --- a/src/app/stateMigrations.ts +++ b/src/app/stateMigrations.ts @@ -26,7 +26,7 @@ */ /** Current persisted-state schema version. Bump when adding a migration. */ -export const DF_STATE_VERSION = 4; +export const DF_STATE_VERSION = 6; type SavedState = Record; @@ -312,6 +312,43 @@ const MIGRATIONS: Migration[] = [ }; }, }, + { + // Table labels have one owner: `displayId`. Older states also stored an + // inferred table label on `tableSemantics`; preserve that suggestion + // only when the table still has its default label, then remove it from + // the field-semantics collection. + to: 6, + migrate: (s) => { + const semantics = Array.isArray(s.tableSemantics) ? s.tableSemantics : []; + const suggestedNames = new Map(); + const tableSemantics = semantics.map(({ displayName, ...info }: any) => { + if (info?.tableId && typeof displayName === 'string' && displayName.trim()) { + suggestedNames.set(info.tableId, displayName.trim()); + } + return info; + }); + const normalizeName = (name: string) => name.toLowerCase().replace(/[\s_-]+/g, ''); + const migrateTableName = (table: any) => { + if (!table?.id) return table; + const suggestion = suggestedNames.get(table.id); + const currentName = table.displayId || table.id; + return suggestion && normalizeName(currentName) === normalizeName(table.id) + ? { ...table, displayId: suggestion } + : table; + }; + return { + ...s, + inputTables: Array.isArray(s.inputTables) + ? s.inputTables.map(migrateTableName) + : s.inputTables, + derivedTables: Array.isArray(s.derivedTables) + ? s.derivedTables.map(migrateTableName) + : s.derivedTables, + tableSemantics, + __stateVersion: 6, + }; + }, + }, ]; /** diff --git a/src/components/ComponentType.tsx b/src/components/ComponentType.tsx index d6528f950..c520fb9dc 100644 --- a/src/components/ComponentType.tsx +++ b/src/components/ComponentType.tsx @@ -370,7 +370,6 @@ export interface FieldSemanticsInfo { export interface TableSemanticsInfo { tableId: string; - displayName?: string; fields: Record; } diff --git a/src/i18n/locales/en/common.json b/src/i18n/locales/en/common.json index 0761e6443..24ad9f50b 100644 --- a/src/i18n/locales/en/common.json +++ b/src/i18n/locales/en/common.json @@ -52,6 +52,7 @@ "title": "Backend Log", "viewLogs": "View backend log", "refresh": "Refresh", + "searchSavedState": "Search saved state (Cmd/Ctrl+F)", "download": "Download full log", "empty": "Log file is empty." }, @@ -462,6 +463,9 @@ "rulesLoaded": "Reading rules: {{rules}}", "knowledgeLoaded": "Reading knowledge: {{knowledge}}", "searching": "searching...", + "listingConnectors": "Checking available connectors", + "readingConnector": "Reading connector setup", + "usingTool": "Using {{tool}}", "producingAction": "outputting {{action}}...", "jumpToThreadRange": "Jump to thread(s) {{label}}", "collapse": "collapse", @@ -562,6 +566,7 @@ "agentWorking": "Agent is working...", "attachUploadFailed": "Failed to attach {{name}}", "replyPlaceholder": "Reply to agent's question...", + "emptyAnalysisInputsPlaceholder": "Press Tab to ask what data are available to load", "explorePlaceholder": "Ask questions or describe what to explore (add context with @)", "explorePlaceholderSingleTable": "Ask questions or describe what to explore", "addMoreData": "Add more data to the workspace", @@ -617,6 +622,7 @@ "delegateToReportGen": "Generate report", "errorDuringExploration": "Error during exploration", "explorationStep": "Exploration step {{step}}: {{question}}", + "emptyAnalysisInputsPrompt": "What data is available to load?", "threadExplorePrompt": "Explore interesting patterns and trends in this data", "explorationThreadDeriveDescription": "Derive from {{source}} with instruction: {{instruction}}", "explorationStepCodeComment": "# Exploration step {{step}}", @@ -851,7 +857,7 @@ "refresh": "Refresh data", "emptyTree": "No tables found", "addConnector": "Add data connector", - "configureConnector": "Edit connection", + "connectConnector": "Connect", "linkLocalFolder": "Link local folder", "newSession": "New session", "importSession": "Import session", @@ -880,7 +886,9 @@ "loadingEllipsis": "Loading...", "loadWithFilters": "Load with Filters", "load": "Load", - "disconnectConnector": "Disconnect connector", + "disconnectConnector": "Disconnect", + "connectorConnected": "Connected to \"{{name}}\"", + "failedConnectConnector": "Failed to connect", "connectorDisconnected": "Connector \"{{name}}\" disconnected", "failedDisconnectConnector": "Failed to disconnect connector", "failedSearchConnector": "Failed to search {{connector}}", diff --git a/src/i18n/locales/en/dataLoading.json b/src/i18n/locales/en/dataLoading.json index 8f70d499b..59d351f4c 100644 --- a/src/i18n/locales/en/dataLoading.json +++ b/src/i18n/locales/en/dataLoading.json @@ -92,10 +92,10 @@ "listingFiles": "Listing files", "runningPython": "Running Python", "preparingPreview": "Preparing preview", - "browsingCatalog": "Browsing catalog", - "searchingData": "Searching data", - "describingData": "Reading table metadata", - "probingData": "Probing data", + "browsingCatalog": "Browsing", + "searchingData": "Searching", + "describingData": "Reading table", + "probingData": "Probing", "proposingLoadPlan": "Proposing load plan" }, "examples": { diff --git a/src/i18n/locales/zh/common.json b/src/i18n/locales/zh/common.json index 4fd743a6c..e1fd49f2a 100644 --- a/src/i18n/locales/zh/common.json +++ b/src/i18n/locales/zh/common.json @@ -52,6 +52,7 @@ "title": "后端日志", "viewLogs": "查看后端日志", "refresh": "刷新", + "searchSavedState": "搜索保存的状态 (Cmd/Ctrl+F)", "download": "下载完整日志", "empty": "日志文件为空。" }, @@ -462,6 +463,9 @@ "rulesLoaded": "读取规则：{{rules}}", "knowledgeLoaded": "读取知识：{{knowledge}}", "searching": "搜索中...", + "listingConnectors": "检查可用连接器", + "readingConnector": "读取连接器设置", + "usingTool": "使用 {{tool}}", "producingAction": "输出 {{action}} 中...", "jumpToThreadRange": "跳转到线程 {{label}}", "collapse": "收起", @@ -617,6 +621,7 @@ "agentWorking": "Agent 努力工作中...", "attachUploadFailed": "附加 {{name}} 失败", "replyPlaceholder": "回复 Agent 的问题...", + "emptyAnalysisInputsPlaceholder": "按 Tab 询问有哪些数据可加载", "explorePlaceholder": "有什么问题，有什么想要探索的？（用 @ 添加上下文）", "explorePlaceholderSingleTable": "有什么问题，有什么想要探索的？", "addMoreData": "向工作区添加更多数据", @@ -672,6 +677,7 @@ "delegateToReportGen": "生成报告", "errorDuringExploration": "探索过程中出错", "explorationStep": "探索步骤 {{step}}：{{question}}", + "emptyAnalysisInputsPrompt": "有哪些数据可以加载？", "threadExplorePrompt": "探索这份数据中有趣的模式和趋势", "explorationThreadDeriveDescription": "从 {{source}} 派生，指令：{{instruction}}", "explorationStepCodeComment": "# 探索步骤 {{step}}", @@ -851,7 +857,7 @@ "refresh": "刷新数据", "emptyTree": "未找到表格", "addConnector": "添加数据连接器", - "configureConnector": "编辑连接", + "connectConnector": "连接", "linkLocalFolder": "链接本地文件夹", "newSession": "新建会话", "importSession": "导入会话", @@ -880,7 +886,9 @@ "loadingEllipsis": "加载中...", "loadWithFilters": "按条件筛选", "load": "加载", - "disconnectConnector": "断开连接器", + "disconnectConnector": "断开连接", + "connectorConnected": "已连接到「{{name}}」", + "failedConnectConnector": "连接失败", "connectorDisconnected": "连接器「{{name}}」已断开", "failedDisconnectConnector": "断开连接器失败", "failedSearchConnector": "搜索 {{connector}} 失败", diff --git a/src/i18n/locales/zh/dataLoading.json b/src/i18n/locales/zh/dataLoading.json index 439b79486..c6fd9645a 100644 --- a/src/i18n/locales/zh/dataLoading.json +++ b/src/i18n/locales/zh/dataLoading.json @@ -92,10 +92,10 @@ "listingFiles": "列出文件", "runningPython": "运行 Python", "preparingPreview": "准备预览", - "browsingCatalog": "浏览目录", - "searchingData": "搜索数据", - "describingData": "读取表元数据", - "probingData": "探查数据", + "browsingCatalog": "浏览", + "searchingData": "搜索", + "describingData": "读取表", + "probingData": "探查", "proposingLoadPlan": "生成加载方案" }, "examples": { diff --git a/src/views/DataSourceSidebar.tsx b/src/views/DataSourceSidebar.tsx index 83a6f074c..c82d29457 100644 --- a/src/views/DataSourceSidebar.tsx +++ b/src/views/DataSourceSidebar.tsx @@ -54,10 +54,10 @@ import ChevronLeftIcon from '@mui/icons-material/ChevronLeft'; import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; import ChevronRightIcon from '@mui/icons-material/ChevronRight'; import RefreshIcon from '@mui/icons-material/Refresh'; +import LinkOutlinedIcon from '@mui/icons-material/LinkOutlined'; import LinkOffOutlinedIcon from '@mui/icons-material/LinkOffOutlined'; import DeleteOutlineIcon from '@mui/icons-material/DeleteOutline'; import EditOutlinedIcon from '@mui/icons-material/EditOutlined'; -import SettingsOutlinedIcon from '@mui/icons-material/SettingsOutlined'; import SearchIcon from '@mui/icons-material/Search'; import ClearIcon from '@mui/icons-material/Clear'; import PushPinIcon from '@mui/icons-material/PushPin'; @@ -1636,10 +1636,8 @@ const DataSourceSidebarPanel: React.FC<{ }, [clearConnectorUiState, deleteTarget, dispatch, onConnectorsChanged, t]); // ── Disconnect connector ────────────────────────────────────────────── - // For admin (non-deletable) connectors the user can't remove the - // definition itself, but they *can* clear stored credentials and the - // active loader so they (or the next user on this identity) can - // re-authenticate via "Edit connection". + // Clear stored credentials and the active loader without removing the + // connector definition, so the user can reconnect through its form. const handleDisconnectConnector = useCallback(async (connector: ConnectorInstance) => { try { @@ -1671,6 +1669,36 @@ const DataSourceSidebarPanel: React.FC<{ } }, [clearConnectorUiState, dispatch, t]); + const handleConnectConnector = useCallback(async (connector: ConnectorInstance) => { + if (connector.auth_mode !== 'none') { + onOpenUploadDialog?.(`connector:${connector.id}`); + return; + } + try { + await apiRequest(CONNECTOR_ACTION_URLS.CONNECT, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ connector_id: connector.id }), + }); + setConnectors(prev => prev.map(c => c.id === connector.id + ? { ...c, connected: true } + : c)); + dispatch(dfActions.addMessages({ + timestamp: Date.now(), + type: 'success', + component: 'data source sidebar', + value: t('sidebar.connectorConnected', { name: connector.display_name }), + })); + } catch (e: any) { + dispatch(dfActions.addMessages({ + timestamp: Date.now(), + type: 'error', + component: 'data source sidebar', + value: e?.apiError?.message || t('sidebar.failedConnectConnector'), + })); + } + }, [dispatch, onOpenUploadDialog, t]); + // ── Render ─────────────────────────────────────────────────────────────── const panelHeaderSx = { @@ -1904,16 +1932,10 @@ const DataSourceSidebarPanel: React.FC<{ of the connector header's chevron. */} { - // No-auth connectors (auth_mode = 'none') - // are always available — clicking the - // header just toggles expansion, never - // opens a credentials dialog. - const isAlwaysOn = connector.auth_mode === 'none'; - if (connector.connected || isAlwaysOn) { + if (connector.connected) { toggleSource(connector.id); } else { - // Not connected — open config dialog for this connector - onOpenUploadDialog?.(`connector:${connector.id}`); + void handleConnectConnector(connector); } }} sx={{ @@ -1942,31 +1964,30 @@ const DataSourceSidebarPanel: React.FC<{ pointerEvents: 'none', }} > - {(connector.connected || connector.auth_mode === 'none') && isExpanded + {connector.connected && isExpanded ? : } {getConnectorIcon(connector.icon || connector.source_type, { sx: { fontSize: iconVar.md, opacity: 0.7 } })} - {/* Status dot — green for live connections - and for always-on built-ins (which are - ready by definition), warning for - disconnected. */} + {/* Status dot — green when this source is available + to the user and agent, warning when disconnected. */} - + {connector.display_name} - {(connector.connected || connector.auth_mode === 'none') && ( + {connector.connected && ( { e.stopPropagation(); @@ -1989,58 +2010,50 @@ const DataSourceSidebarPanel: React.FC<{ )} - {/* Edit connection — available for both user and admin - connectors. Admin connectors can't be deleted, but - the user still needs a way to (re)enter credentials - or trigger a fresh login after disconnecting. - No-auth connectors have no credentials to configure, - so we skip this entirely. */} - {connector.auth_mode !== 'none' && ( - - { - e.stopPropagation(); - onOpenUploadDialog?.(`connector:${connector.id}`); - }} - sx={{ color: 'text.disabled', p: 0.25, visibility: 'hidden', '&:hover': { color: 'primary.main' } }} - > - - - - )} - {connector.deletable ? ( - + {connector.connected ? ( + { e.stopPropagation(); - setDeleteTarget(connector); + void handleDisconnectConnector(connector); }} - sx={{ color: 'text.disabled', p: 0.25, visibility: 'hidden', '&:hover': { color: 'error.main' } }} + sx={{ color: 'text.disabled', p: 0.25, visibility: 'hidden', '&:hover': { color: 'warning.main' } }} > - + - ) : connector.connected && connector.auth_mode !== 'none' && ( - /* Admin connector: surface Disconnect in place of Delete. - Only meaningful when there's an active session/credentials - to clear; if already disconnected, "Edit connection" is - the path to re-authenticate. - No-auth connectors have nothing to disconnect. */ - + ) : ( + { e.stopPropagation(); - void handleDisconnectConnector(connector); + void handleConnectConnector(connector); }} - sx={{ color: 'text.disabled', p: 0.25, visibility: 'hidden', '&:hover': { color: 'warning.main' } }} + sx={{ color: 'text.disabled', p: 0.25, visibility: 'hidden', '&:hover': { color: 'primary.main' } }} > - + + + + )} + {connector.deletable && ( + + { + e.stopPropagation(); + setDeleteTarget(connector); + }} + sx={{ color: 'text.disabled', p: 0.25, visibility: 'hidden', '&:hover': { color: 'error.main' } }} + > + )} diff --git a/src/views/DataThread.tsx b/src/views/DataThread.tsx index 7e33212bc..c3640fc05 100644 --- a/src/views/DataThread.tsx +++ b/src/views/DataThread.tsx @@ -137,17 +137,21 @@ const LiveStatus: React.FC<{ startTime?: number; resetKey?: string }> = ({ start }; /** Render a multi-step thinking banner as a single block with sectioned steps. + * Steps read as progress, not a transcript, so only the active one shows. * When `startTime` is provided, the live timer is appended *inline* next to - * the active (last) step's text — same alignment grammar as the single-line + * the active step's text — same alignment grammar as the single-line * ThinkingBanner — rather than right-flushed in a separate column. * The timer resets whenever the active step changes so it shows the time * spent on the **current** action, not the cumulative wait. */ export const ThinkingStepsBanner = (steps: string[], sx?: SxProps, startTime?: number, active: boolean = true) => { - const activeStep = steps.length > 0 ? steps[steps.length - 1] : ''; + const lastStep = steps.length > 0 ? steps[steps.length - 1] : ''; + // While the run is live the latest step stays in progress even after its own + // tool returned — the agent is already working on whatever comes next. + const activeStep = active && lastStep.startsWith('✓') ? lastStep.slice(2) : lastStep; return ( : undefined} /> @@ -537,7 +541,6 @@ let SingleThreadGroupView: FC<{ let tables = useSelector(dfSelectors.getAllTables); const derivedTables = useSelector(dfSelectors.getDerivedTables); - const inferredTableNames = useSelector((state: DataFormulatorState) => state.tableSemantics); const { t } = useTranslation(); const tableById = useMemo(() => new Map(tables.map(t => [t.id, t])), [tables]); @@ -704,6 +707,23 @@ let SingleThreadGroupView: FC<{ return map; }, [loadedTableNodes]); + const highlightedTextTurnIds = useMemo(() => { + const ids = new Set(); + for (const node of loadedTableNodes) { + if (!globalHighlightedTableIds.includes(node.tableId)) continue; + let current: string | undefined = node.parentNodeId; + const seen = new Set(); + while (current && !seen.has(current)) { + seen.add(current); + const turn = turnById.get(current); + if (!turn) break; + ids.add(turn.id); + current = turn.parentNodeId; + } + } + return ids; + }, [globalHighlightedTableIds, loadedTableNodes, turnById]); + const tableAnchorOfNode = (nodeId: string | undefined): string => { let current = nodeId; const seen = new Set(); @@ -779,15 +799,13 @@ let SingleThreadGroupView: FC<{ }; let _buildTableCard = (tableId: string) => { - const inferredDisplayName = inferredTableNames.find(info => info.tableId === tableId)?.displayName; - return buildTableCard({ tableId, inferredDisplayName, ...tableCardProps }); + return buildTableCard({ tableId, ...tableCardProps }); } /** Pointer to a table whose real card lives in the shelf or a prior column. */ let _buildRefChip = (tableId: string) => { - const displayName = inferredTableNames.find(info => info.tableId === tableId)?.displayName; return buildTableRefChip({ - tableId, table: tableById.get(tableId), displayName, + tableId, table: tableById.get(tableId), focused: tableId === focusedTableId, dispatch, }); } @@ -1362,15 +1380,16 @@ let SingleThreadGroupView: FC<{ // Render a single text turn: its triggering prompt bubble (if any) then the // turn card. `keyNode` seeds prompt-entry keys. const pushSingleTurn = (turn: TextTurn, keyNode: string, highlighted: boolean, triggerType: 'trigger' | 'leaf-trigger') => { + const turnHighlighted = highlighted || highlightedTextTurnIds.has(turn.id); if (turn.prompt) { pushInteractionEntries( [{ from: 'user', to: 'data-agent', role: 'prompt', content: turn.prompt, timestamp: turn.createdAt }], - keyNode, triggerType, highlighted, `textturn-prompt-${turn.id}`, + keyNode, triggerType, turnHighlighted, `textturn-prompt-${turn.id}`, ); } - timelineItems.push(buildTextTurnTimelineItem(turn, highlighted, false)); + timelineItems.push(buildTextTurnTimelineItem(turn, turnHighlighted, false)); for (const report of reportsByParentNode.get(turn.id) || []) { - timelineItems.push(buildReportTimelineItem(report, highlighted)); + timelineItems.push(buildReportTimelineItem(report, turnHighlighted)); } // A turn that loaded tables skips the reply — the tables below already // say which option was taken. @@ -1378,7 +1397,7 @@ let SingleThreadGroupView: FC<{ if (turn.answered && turn.answer && loadedTables.length === 0) { pushInteractionEntries( [{ from: 'user', to: 'data-agent', role: 'prompt', content: turn.answer }], - keyNode, triggerType, highlighted, `textturn-answer-${turn.id}`, + keyNode, triggerType, turnHighlighted, `textturn-answer-${turn.id}`, ); } }; diff --git a/src/views/DataThreadCards.tsx b/src/views/DataThreadCards.tsx index 332bc38b7..58d219310 100644 --- a/src/views/DataThreadCards.tsx +++ b/src/views/DataThreadCards.tsx @@ -133,11 +133,10 @@ export let buildChartCards = ( export let buildTableRefChip = (props: { tableId: string; table: DictTable | undefined; - displayName?: string; focused: boolean; dispatch: any; }) => { - const { tableId, table, displayName, focused, dispatch } = props; + const { tableId, table, focused, dispatch } = props; return {displayName?.trim() || table?.displayId || tableId} + }}>{table?.displayId || tableId} @@ -202,7 +201,6 @@ export let buildTriggerCard = ( export interface BuildTableCardProps { tableId: string; tables: DictTable[]; - inferredDisplayName?: string; chartElements: { tableId: string, chartId: string, element: any }[]; usedIntermediateTableIds: string[]; highlightedTableIds: string[]; @@ -223,7 +221,7 @@ export interface BuildTableCardProps { export let buildTableCard = (props: BuildTableCardProps) => { const { - tableId, tables, inferredDisplayName, chartElements, usedIntermediateTableIds, + tableId, tables, chartElements, usedIntermediateTableIds, highlightedTableIds, focusedTableId, focusedChartId, parentTable, tableIdList, collapsed, dispatch, handleOpenTableMenu, primaryBgColor, t, showOriginalName = true, @@ -256,11 +254,8 @@ export let buildTableCard = (props: BuildTableCardProps) => { let table = tables.find(t => t.id == tableId); const originalName = getOriginalName(table); const sourceTooltip = getSourceTooltip(table); - const workspaceName = table?.displayId || tableId; + const friendlyName = table?.displayId || tableId; const normalizeTableName = (name: string) => name.toLowerCase().replace(/[\s_-]+/g, ''); - const friendlyName = inferredDisplayName?.trim() - ? inferredDisplayName.trim() - : workspaceName; const rawName = showOriginalName && originalName && normalizeTableName(originalName) !== normalizeTableName(friendlyName) diff --git a/src/views/DataView.tsx b/src/views/DataView.tsx index cbc8c3140..ab68be992 100644 --- a/src/views/DataView.tsx +++ b/src/views/DataView.tsx @@ -96,8 +96,7 @@ export const FreeDataViewFC: FC = function DataView({ maximiz const tableSemantics = useSelector((state: DataFormulatorState) => state.tableSemantics.find(info => info.tableId === focusedTableId), ); - const displayName = tableSemantics?.displayName?.trim() - || targetTable?.displayId + const displayName = targetTable?.displayId || targetTable?.id || 'table'; const realName = targetTable?.derive diff --git a/src/views/InteractionEntryCard.tsx b/src/views/InteractionEntryCard.tsx index 79f3ac573..ba026a2d0 100644 --- a/src/views/InteractionEntryCard.tsx +++ b/src/views/InteractionEntryCard.tsx @@ -50,7 +50,11 @@ const PlanStepItem: React.FC<{ const isFailed = step.startsWith('✗'); const isWarning = step.startsWith('⚠'); const isInfo = step.startsWith('📋'); - const displayLine = (isChecked || isFailed) ? step.slice(2) : (isWarning || isInfo) ? step.slice(2).trimStart() : step; + const rawLine = (isChecked || isFailed) ? step.slice(2) : (isWarning || isInfo) ? step.slice(2).trimStart() : step; + // Trailing ellipsis marks the step still in flight; some labels ship their own. + const displayLine = showShimmer && !/(\.\.\.|…)$/.test(rawLine.trim()) + ? `${rawLine}…` + : rawLine; const IconComp = getStepIconComponent(step); // Text stays in the normal muted color even for failed/warning steps — the diff --git a/src/views/LogViewerDialog.tsx b/src/views/LogViewerDialog.tsx index 7c273dcac..6d393493b 100644 --- a/src/views/LogViewerDialog.tsx +++ b/src/views/LogViewerDialog.tsx @@ -15,8 +15,23 @@ import React, { FC, useCallback, useEffect, useRef, useState } from 'react'; import CodeMirror, { EditorView } from '@uiw/react-codemirror'; -import { foldEffect, syntaxTree } from '@codemirror/language'; +import { EditorState } from '@codemirror/state'; +import { keymap, Panel } from '@codemirror/view'; +import { ensureSyntaxTree, foldEffect, forceParsing } from '@codemirror/language'; import { json } from '@codemirror/lang-json'; +import { + closeSearchPanel, + findNext, + findPrevious, + getSearchQuery, + openSearchPanel, + search, + SearchQuery, + searchKeymap, + selectMatches, + setSearchQuery, +} from '@codemirror/search'; +import { SyntaxNode } from '@lezer/common'; import { Box, CircularProgress, @@ -32,6 +47,7 @@ import { import TerminalOutlinedIcon from '@mui/icons-material/TerminalOutlined'; import RefreshIcon from '@mui/icons-material/Refresh'; import DownloadIcon from '@mui/icons-material/Download'; +import SearchIcon from '@mui/icons-material/Search'; import CloseIcon from '@mui/icons-material/Close'; import { useTranslation } from 'react-i18next'; import { useSelector } from 'react-redux'; @@ -42,7 +58,191 @@ import { DataFormulatorState } from '../app/dfSlice'; import { textVar } from '../app/layout'; const DEFAULT_TAIL_LINES = 500; -const DEFAULT_FOLD_CHARACTER_THRESHOLD = 2000; + +export function createSavedStateSearchPanel(view: EditorView): Panel { + const searchInput = document.createElement('input'); + searchInput.type = 'text'; + searchInput.className = 'cm-textfield'; + searchInput.name = 'df-saved-state-find'; + searchInput.placeholder = 'Find'; + searchInput.setAttribute('aria-label', 'Find'); + searchInput.setAttribute('main-field', 'true'); + searchInput.setAttribute('autocomplete', 'off'); + searchInput.setAttribute('autocorrect', 'off'); + searchInput.setAttribute('autocapitalize', 'off'); + searchInput.setAttribute('spellcheck', 'false'); + searchInput.setAttribute('aria-autocomplete', 'none'); + searchInput.setAttribute('data-1p-ignore', 'true'); + searchInput.setAttribute('data-lpignore', 'true'); + searchInput.value = getSearchQuery(view.state).search; + + const updateQuery = () => { + const current = getSearchQuery(view.state); + view.dispatch({ + effects: setSearchQuery.of(new SearchQuery({ + search: searchInput.value, + caseSensitive: current.caseSensitive, + literal: current.literal, + regexp: current.regexp, + wholeWord: current.wholeWord, + })), + }); + }; + searchInput.addEventListener('input', updateQuery); + searchInput.addEventListener('keydown', event => { + if (event.key === 'Enter') { + event.preventDefault(); + (event.shiftKey ? findPrevious : findNext)(view); + } else if (event.key === 'Escape') { + event.preventDefault(); + closeSearchPanel(view); + } + }); + + const makeButton = (name: string, label: string, action: () => void) => { + const button = document.createElement('button'); + button.type = 'button'; + button.className = name === 'close' ? '' : 'cm-button'; + button.name = name; + button.textContent = label; + button.setAttribute('aria-label', label); + button.addEventListener('click', action); + return button; + }; + + const panel = document.createElement('div'); + panel.className = 'cm-search'; + panel.append( + searchInput, + makeButton('next', 'Next', () => { findNext(view); }), + makeButton('prev', 'Previous', () => { findPrevious(view); }), + makeButton('select', 'All', () => { selectMatches(view); }), + makeButton('close', '×', () => { closeSearchPanel(view); }), + ); + + return { + dom: panel, + update(update) { + const query = getSearchQuery(update.state); + if (searchInput.value !== query.search) searchInput.value = query.search; + }, + destroy() { + searchInput.removeEventListener('input', updateQuery); + }, + }; +} + +const savedStateEditorTheme = EditorView.theme({ + '&': { + height: '100%', + fontSize: textVar.sm, + }, + '&.cm-focused': { outline: 'none' }, + '.cm-scroller': { fontFamily: 'var(--df-font-mono)' }, + '.cm-panels': { + backgroundColor: '#f7f8fa', + color: '#30343b', + fontFamily: 'Roboto, sans-serif', + }, + '.cm-panels.cm-panels-bottom': { + borderTop: '1px solid rgba(0, 0, 0, 0.12)', + }, + '.cm-search': { + display: 'flex', + alignItems: 'center', + gap: '6px', + padding: '7px 10px', + }, + '.cm-search label, .cm-search br': { display: 'none' }, + '.cm-search .cm-textfield': { + width: 'min(320px, 45vw)', + height: '30px', + boxSizing: 'border-box', + padding: '4px 9px', + border: '1px solid rgba(0, 0, 0, 0.18)', + borderRadius: '6px', + backgroundColor: '#fff', + color: '#202124', + fontFamily: 'var(--df-font-mono)', + fontSize: `${textVar.sm}px`, + outline: 'none', + }, + '.cm-search .cm-textfield:focus': { + borderColor: '#1976d2', + boxShadow: '0 0 0 2px rgba(25, 118, 210, 0.14)', + }, + '.cm-search .cm-button': { + height: '30px', + boxSizing: 'border-box', + margin: '0', + padding: '4px 10px', + border: '1px solid rgba(0, 0, 0, 0.14)', + borderRadius: '6px', + backgroundImage: 'none', + backgroundColor: '#fff', + color: '#3c4043', + fontFamily: 'Roboto, sans-serif', + fontSize: `${textVar.xs}px`, + cursor: 'pointer', + }, + '.cm-search .cm-button:hover': { + borderColor: 'rgba(25, 118, 210, 0.45)', + backgroundColor: 'rgba(25, 118, 210, 0.06)', + color: '#1565c0', + }, + '.cm-search button[name="close"]': { + position: 'static', + width: '30px', + height: '30px', + marginLeft: 'auto', + border: '0', + borderRadius: '6px', + backgroundColor: 'transparent', + color: '#5f6368', + fontSize: '18px', + cursor: 'pointer', + }, + '.cm-search button[name="close"]:hover': { + backgroundColor: 'rgba(0, 0, 0, 0.06)', + color: '#202124', + }, +}); + +const savedStateEditorExtensions = [ + json(), + search({ createPanel: createSavedStateSearchPanel }), + keymap.of(searchKeymap), + EditorView.lineWrapping, + savedStateEditorTheme, +]; + +const SAVED_STATE_AUTO_FOLD_PATHS = [ + // Table payloads: keep IDs, names, lineage, and virtual references visible. + ['inputTables', '*', 'snapshot'], + ['derivedTables', '*', 'rows'], + ['derivedTables', '*', 'metadata'], + // Generated derivation evidence and conversation traces. + ['derivedTables', '*', 'derive', 'dialog'], + ['derivedTables', '*', 'derive', 'explanation'], + ['derivedTables', '*', 'derive', 'trigger', 'interaction'], + ['draftNodes', '*', 'derive', 'dialog'], + ['draftNodes', '*', 'derive', 'trigger', 'interaction'], + ['draftNodes', '*', 'derive', 'pendingClarification', 'trajectory'], + // Generated visual/report payloads. + ['charts', '*', 'styleVariants'], + ['generatedReports', '*', 'inspectionSteps'], + // Structured artifacts: keep turn identity, kind, status, and parent visible. + ['textTurns', '*', 'options'], + ['textTurns', '*', 'form'], + ['textTurns', '*', 'dataOperation'], + ['textTurns', '*', 'resume', 'trajectory'], + // Embedded loading results: keep message role, content, and timestamp visible. + ['dataLoadingChatMessages', '*', 'codeBlocks'], + ['dataLoadingChatMessages', '*', 'tables'], + ['dataLoadingChatMessages', '*', 'loadPlan'], + ['dataLoadingChatMessages', '*', 'dataOperation'], + ['dataLoadingChatMessages', '*', 'connectorForm'], +]; interface LogTailResponse { path: string | null; @@ -56,27 +256,56 @@ interface SessionLoadResponse { state: Record; } -function foldLargeJsonValues(view: EditorView): void { - const effects: ReturnType[] = []; - syntaxTree(view.state).iterate({ +function jsonContainerPath(state: EditorState, node: SyntaxNode): string[] { + const path: string[] = []; + let current: SyntaxNode | null = node; + while (current?.parent) { + const parent: SyntaxNode = current.parent; + if (parent.name === 'Property') { + const propertyName = parent.getChild('PropertyName'); + if (propertyName) { + try { + path.unshift(JSON.parse(state.doc.sliceString(propertyName.from, propertyName.to))); + } catch { + return []; + } + } + } else if (parent.name === 'Array') { + path.unshift('*'); + } + current = parent; + } + return path; +} + +export function getSavedStateAutoFoldRanges(state: EditorState): { from: number; to: number }[] { + const ranges: { from: number; to: number }[] = []; + const tree = ensureSyntaxTree(state, state.doc.length, 100); + if (!tree) return ranges; + tree.iterate({ enter(node) { const isContainer = node.name === 'Array' || node.name === 'Object'; const isRoot = node.node.parent === null; - const property = node.node.parent; - const propertyPrefix = property?.name === 'Property' - ? view.state.doc.sliceString(property.from, node.from) - : ''; - const propertyName = propertyPrefix.match(/"([^"\\]+)"\s*:\s*$/)?.[1]?.toLowerCase() || ''; - const isAgentConversation = /agent|chat|message|dialog/.test(propertyName); - if (isContainer && !isRoot && ( - isAgentConversation || node.to - node.from >= DEFAULT_FOLD_CHARACTER_THRESHOLD - )) { - effects.push(foldEffect.of({ from: node.from + 1, to: node.to - 1 })); + if (!isContainer || isRoot) return undefined; + const path = jsonContainerPath(state, node.node); + const matches = SAVED_STATE_AUTO_FOLD_PATHS.some(pattern => + pattern.length === path.length && pattern.every((segment, index) => segment === path[index]) + ); + if (matches) { + if (node.to - node.from > 2) { + ranges.push({ from: node.from + 1, to: node.to - 1 }); + } return false; } return undefined; }, }); + return ranges; +} + +function foldSavedStatePaths(view: EditorView): void { + forceParsing(view, view.state.doc.length, 200); + const effects = getSavedStateAutoFoldRanges(view.state).map(range => foldEffect.of(range)); if (effects.length > 0) view.dispatch({ effects }); } @@ -108,6 +337,7 @@ export const LogViewerDialog: FC<{ const [activeTab, setActiveTab] = useState(0); const [savedState, setSavedState] = useState(''); const preRef = useRef(null); + const savedStateEditorRef = useRef(null); const fetchLogs = useCallback(async () => { setLoading(true); @@ -161,11 +391,38 @@ export const LogViewerDialog: FC<{ } }, [content, open]); + useEffect(() => { + if (activeTab === 1 && savedState && savedStateEditorRef.current) { + foldSavedStatePaths(savedStateEditorRef.current); + } + }, [activeTab, savedState]); + + useEffect(() => { + if (!open || activeTab !== 1) return; + const handleSavedStateSearchShortcut = (event: KeyboardEvent) => { + if ((event.metaKey || event.ctrlKey) && !event.altKey && event.key.toLowerCase() === 'f') { + event.preventDefault(); + event.stopPropagation(); + if (savedStateEditorRef.current) { + openSearchPanel(savedStateEditorRef.current); + } + } + }; + window.addEventListener('keydown', handleSavedStateSearchShortcut, true); + return () => window.removeEventListener('keydown', handleSavedStateSearchShortcut, true); + }, [activeTab, open]); + const handleDownload = () => { // Direct navigation triggers the browser download (attachment header). window.open(getUrls().LOGS_DOWNLOAD, '_blank'); }; + const handleSearchSavedState = () => { + if (savedStateEditorRef.current) { + openSearchPanel(savedStateEditorRef.current); + } + }; + const handleRefresh = activeTab === 0 ? fetchLogs : fetchSavedState; return ( @@ -198,6 +455,19 @@ export const LogViewerDialog: FC<{ + {activeTab === 1 && + + + + + + } {activeTab === 0 && @@ -300,7 +570,7 @@ export const LogViewerDialog: FC<{ { + savedStateEditorRef.current = view; + }} aria-label={t('logs.savedStateTab', { defaultValue: 'Saved State' })} /> diff --git a/src/views/ReportView.tsx b/src/views/ReportView.tsx index a67fbc032..735501367 100644 --- a/src/views/ReportView.tsx +++ b/src/views/ReportView.tsx @@ -42,6 +42,7 @@ export const ReportView: FC = () => { const config = useSelector((state: DataFormulatorState) => state.config); const allGeneratedReports = useSelector(dfSelectors.getAllGeneratedReports); const serverConfig = useSelector((state: DataFormulatorState) => state.serverConfig); + const activeWorkspace = useSelector((state: DataFormulatorState) => state.activeWorkspace); const focusedId = useSelector((state: DataFormulatorState) => state.focusedId); // Thumbnails live in their own slice so updates don't churn `state.charts`. const chartThumbnails = useSelector((state: DataFormulatorState) => state.chartThumbnails) || {}; @@ -143,9 +144,16 @@ export const ReportView: FC = () => { return sanitized || t('report.untitled'); }; - const getReportFileName = (extension: string): string => { + const getReportFileName = (extension: string, root?: ParentNode | null): string => { const date = new Date().toISOString().slice(0, 10); - return `${sanitizeFileName(getReportTitle())}-${date}.${extension}`; + const reportTitle = getReportTitle(root); + const sessionName = activeWorkspace?.displayName || activeWorkspace?.id || ''; + const normalizeName = (name: string) => name.toLowerCase().replace(/[\s_-]+/g, ''); + const parts = [reportTitle]; + if (sessionName && normalizeName(sessionName) !== normalizeName(reportTitle)) { + parts.push(sessionName); + } + return `${sanitizeFileName(parts.join(' - '))} - ${date}.${extension}`; }; const renderReportToCanvas = async (): Promise => { @@ -309,7 +317,7 @@ export const ReportView: FC = () => { const styles = Array.from(document.querySelectorAll('style, link[rel="stylesheet"]')) .map(node => node.outerHTML) .join('\n'); - const printTitle = sanitizeFileName(getReportTitle(exportClone.clone)); + const printTitle = getReportFileName('pdf', exportClone.clone).replace(/\.pdf$/, ''); const originalDocumentTitle = document.title; const doc = printFrame.contentDocument; const win = printFrame.contentWindow; @@ -324,7 +332,7 @@ export const ReportView: FC = () => { -${printTitle} + ${styles}