diff --git a/.gitignore b/.gitignore index 6f637365c..2277e43c1 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ design-docs/ deploy-scripts/ test-data-loader/ scripts/ +docs/esrp/* ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. diff --git a/docs/desktop-portable.md b/docs/desktop-portable.md deleted file mode 100644 index 473660530..000000000 --- a/docs/desktop-portable.md +++ /dev/null @@ -1,37 +0,0 @@ -# Portable desktop build - -Data Formulator's desktop bundle runs the existing Flask application on a -random loopback port and displays it in a native pywebview window. It is built -as a PyInstaller `onedir` bundle so users can unzip it and launch it without -installing Python or Node.js. - -## Build - -Build on each target operating system; PyInstaller does not cross-compile. - -```bash -yarn install --frozen-lockfile -yarn build # frontend -> py-src/data_formulator/dist -uv sync --extra desktop -uv run pyinstaller --noconfirm --clean packaging/data_formulator_desktop.spec -``` - -On Windows and Linux, the output is `dist/Data Formulator/`; distribute the -complete directory as a zip archive. On macOS, distribute -`dist/Data Formulator.app`. Code signing and macOS notarization should be added -before a public release. - -## Azure CLI authentication - -Kusto and other Entra-enabled connectors reuse the user's Azure CLI identity. -The desktop app does not request delegated `user_impersonation` permission for -its own app registration. - -Azure CLI remains an external prerequisite for Azure connections. Users can -sign in from Data Formulator's connector UI; the backend runs `az login` and -then Azure Identity obtains tokens from the CLI cache. Other features remain -usable when Azure CLI is absent. - -The launcher adds common Azure CLI install locations to `PATH`, including -Homebrew locations that are normally missing when a macOS app is opened from -Finder. \ No newline at end of file diff --git a/loops/model-evaluation/plan.md b/loops/model-evaluation/plan.md deleted file mode 100644 index 1fd19bf34..000000000 --- a/loops/model-evaluation/plan.md +++ /dev/null @@ -1,66 +0,0 @@ -# Loop — Open-Source (Ollama) Model Evaluation - -**High-level plan.** Execute end-to-end, making reasonable decisions when details are -ambiguous, and record them in the final report (`report.md`; all working artifacts go -under `work/`). - -## Goal - -Benchmark open-source (Ollama) models that drive Data Formulator's analyst agents — -inspect tabular data, write transformation code, and commit a visualization — and report -**two independent axes**: - -1. **Success rate** — does the agent actually produce a rendered chart? (reliability) -2. **Quality when produced** — how good is the chart when it finishes, scored 0-100 by a - code + vision grader? (competence) - -Keep them separate: a model can write good code yet fail to deliver it through the -protocol. The dominant open-model failure mode is **driving the tool/transport, not -analyzing the data**, so each model runs through more than one agent transport: - -- `analyst` — native function/tool calls (with a content-JSON salvage fallback). -- `mini` — single-decision, pure-prompt JSON contract; the production low-cost agent. - -Always include the Azure references `gpt-5.5`, `gpt-5-mini` as the baseline. - -## Data - -A frozen **45-question** set across **15 datasets** from the `../visbench` benchmark, fed -as the **raw / grouped source tables** (not VisBench's derived single-table `data.csv`) so -the agent must do its own joins: - -- **vega_datasets** single tables — 9 single-table questions. -- **TidyTuesday** multi-CSV weeks — 18 multi-table questions. -- **Spider** databases grouped by DB — 18 multi-table questions. - -Reuse VisBench's quality-filtered question and reference chart for each item. The single- -vs multi-table split (9 / 36) is the axis along which models diverge most. - -## Steps - -1. **Select & pull models** — the open roster across size tiers (1B → 120B) plus the three - Azure references. -2. **Prepare the benchmark** — materialize the 45 questions as raw/grouped tables and - freeze the VisBench questions + reference charts, reused identically across every model - and agent. -3. **Run agents** — every `(agent, model, question)` cell with `--agent` in `analyst` - and `mini`; capture the event stream and render each chart to PNG. Frozen controls: - `max_iterations = 5`, 240 s timeout, resumable. -4. **Score (two phases, GPT-5.5 grader):** - - **Phase 1 — reliability:** five sequential gates (responded → emitted action → code - ran → output → **produced chart**). The chart gate is decisive and defines the - success rate; only those runs proceed. - - **Phase 2 — quality (0-100, produced charts only):** code review vs the question - (0-50) + vision review of the rendered PNG vs the reference chart (0-50). -5. **Aggregate & report** — report the two axes separately (never collapse them); for - ranking only, derive success-weighted quality (Phase 2 over all 45, no-chart = 0) and - combined = `0.3 × (success_rate × 100) + 0.7 × success-weighted quality`. Always show - the single- vs multi-table split, the per-gate drop-off, comparison to the references, - and recommendations per size tier (with which `--agent`). - -## Principles - -- **Two axes stay separate** — `combined` is for ranking only. -- **Freeze controls** — same questions, grader, `max_iterations`, and timeout across every cell. -- **`mini` is the production low-cost agent** — `simple` was removed; don't run `--agent simple`. -- **`uv` only**, no secrets (Azure auth via Entra ID), resumable, all artifacts under `work/`. diff --git a/package.json b/package.json index 40156b843..cd0899a33 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,11 @@ "private": true, "resolutions": { "lodash": "^4.18.1", - "vite": "^7.3.3", - "dompurify": "^3.4.2", + "vite": "^7.3.5", + "dompurify": "^3.4.13", + "postcss": "^8.5.23", + "esbuild": "^0.28.1", + "tmp": "^0.2.6", "markdown-it": "^14.3.0", "linkify-it": "^5.0.2", "undici": "^7.29.0", @@ -43,14 +46,14 @@ "canvas": "^3.2.1", "chart.js": "^4.5.1", "d3": "^7.3.0", - "dompurify": "^3.4.0", - "echarts": "^6.0.0", + "dompurify": "^3.4.13", + "echarts": "^6.1.0", "exceljs": "^4.4.0", "flint-chart": ">=0.5.0", "html2canvas": "^1.4.1", "i18next": "^26.0.1", "i18next-browser-languagedetector": "^8.2.1", - "js-yaml": "^4.1.1", + "js-yaml": "^4.3.1", "katex": "^0.16.22", "localforage": "^1.10.0", "lodash": "^4.18.1", @@ -130,7 +133,7 @@ "jsdom": "^29.0.1", "sass": "^1.102.0", "typescript-eslint": "^8.65.0", - "vite": "^7.3.3", + "vite": "^7.3.5", "vitest": "^4.1.0" } } diff --git a/py-src/data_formulator/agents/agent_data_loading_chat.py b/py-src/data_formulator/agents/agent_data_loading_chat.py index 01419374a..fa505461b 100644 --- a/py-src/data_formulator/agents/agent_data_loading_chat.py +++ b/py-src/data_formulator/agents/agent_data_loading_chat.py @@ -118,17 +118,16 @@ **Workflow 3 — Find and load data from connected sources (including sample datasets):** 1. Call find_data(query="...") to search. The query is a case-insensitive regex — use alternation for synonyms ("orders|sales|revenue"), anchors ("^fact_"), word - boundaries ("\\border\\b"), or optional groups ("customers?") when helpful. Escape - "." if you mean a literal dot. Pass exclude="_staging|_test" to drop noise. - When search is ambiguous, restrict with scope="" or - scope=":". + boundaries ("\\border\\b"), or optional groups ("customers?") when helpful. + Escape "." if you mean a literal dot. Restrict with source_id and exact path + when the search location is known. 2. If find_data returns nothing useful or is ambiguous, fall back to list_data: - list_data() → which sources exist - list_data(source_id="...") → top-level folders / tables - list_data(source_id, path=[...]) → drill in - - Pass filter="..." (plain substring, not regex) when a directory has many entries. - Responses are capped at 200 entries; if truncated:true, narrow with filter or - switch back to find_data with a scope. + - Use filter_by="folder" or filter_by="table" when only one node type matters. + If a listing is truncated, continue with next_start_after or switch to a + narrower exact path. 3. For EACH promising not-imported table, call describe_data(source_id, table_key) to inspect columns and understand available values. 4. Based on column metadata, decide which columns to filter on and what values to use. @@ -171,9 +170,11 @@ When several inputs/sources are in play, reflect on the role of each attachment: is it the data to load, or context/guidance for what to extract from another source? If it's guidance, use it to steer Workflow 1/3/5 rather than transcribing it. -- User asked "what data do you have / what's available / which sources are connected" → call - list_data() — it returns the per-source summary. Drill in with list_data(source_id, ...). - Do NOT rely solely on the summary below; it only shows counts. +- User asked "what data do you have / what's available / which sources are connected" → + call list_data(), then for EACH connected source call + find_data(source_id="...", filter_by="table", limit=10). Do not stop after the + inventory and do not ask the user which source to inspect. Answer with real + tables from every inspected source and recommend concrete starting points. - Otherwise, if connected data sources are listed below AND the user is describing data they want to analyze (an entity, metric, time range, region, product, demo data, etc.) → start with Workflow 3. Try regex variants (English + the user's language, synonyms, table-name fragments, @@ -188,13 +189,15 @@ Rules: - Broad, open-ended questions ("what data do we have?", "help me connect", "how do I get started?", "what can you do?") deserve a fuller, orienting answer than a narrow task reply. - First run the relevant tool — list_data() for what's available, list_connectors for connecting — + First run the relevant tool — summarize_data_sources() for what's available, list_connectors for connecting — then give concrete guidance grounded in what you found: briefly summarize it (e.g. the connected sources with a couple of example tables, or the connector types this deployment offers), and suggest 2-3 specific next steps the user could take ("I can pull the orders table", "tell me your Postgres host and I'll set up the form"). Don't reply with a bare list or a plain "what do you want?" — help them see their options and move forward. (This does NOT override the brevity rule below, which applies only after a preview/plan card is shown.) +- For a broad availability question, call `summarize_data_sources` first and answer from its + bounded overview. Do not use `ask_user` merely to choose which connected source to inspect. - After show_user_data_preview or propose_load_plan, keep text VERY brief. The UI shows the preview automatically. - show_user_data_preview is ONLY for: (a) DataFrames you actually produced with execute_python via saved_dfs=, or (b) tables you literally extracted from a user-provided image or pasted text via tables=. NEVER use show_user_data_preview(tables=...) to narrate, describe, or invent contents of a connector-sourced table. To load ANY table from a connected source (including sample_datasets), you MUST use propose_load_plan. - For sample datasets, NEVER use execute_python or write_file to recreate them — use Workflow 3. @@ -375,16 +378,29 @@ }, }, }, + { + "type": "function", + "function": { + "name": "summarize_data_sources", + "description": ( + "Return a bounded overview of every connected data source: hierarchy stats, " + "top-level items, branch-diverse sample tables, and explicit omitted counts. " + "Use this first for broad questions about what data is available." + ), + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + }, { "type": "function", "function": { "name": "list_data", "description": ( - "Browse the catalog of connected data sources. Cache-only, fast.\n" - "- No args: per-source summary (source_id, table_count, is_hierarchical).\n" - "- source_id only: top-level entries (folders with table counts, plus root tables).\n" - "- source_id + path: direct children at that hierarchy level.\n" - "- filter: case-insensitive substring on the next path segment / table name (no regex here).\n" + "List connected-source catalogs like ls. Cache-only and fast.\n" + "- No args: immediate source nodes at the catalog root.\n" + "- source_id plus optional exact path: immediate typed children only.\n" + "- filter_by: optionally return only folders or tables.\n" + "- If truncated, continue with next_start_after as start_after.\n" + "Use summarize_data_sources instead for a broad overview.\n" "Workspace tables are already in the system prompt and are not repeated." ), "parameters": { @@ -396,7 +412,22 @@ "items": {"type": "string"}, "description": "Hierarchy path as an array of segments (e.g. ['sales', 'fy26']).", }, - "filter": {"type": "string", "description": "Substring filter on the next path segment / table name."}, + "filter_by": { + "type": "string", + "enum": ["folder", "table"], + "description": "Optional immediate-child node type.", + }, + "limit": {"type": "integer", "minimum": 1, "maximum": 500, "description": "Max items. Default 100."}, + "start_after": { + "type": "object", + "description": "Exclusive continuation reference returned as next_start_after.", + "properties": { + "type": {"type": "string", "enum": ["folder", "table"]}, + "path": {"type": "array", "items": {"type": "string"}}, + "table_key": {"type": "string"}, + }, + "required": ["type", "path"], + }, }, "required": [], }, @@ -442,30 +473,38 @@ "function": { "name": "find_data", "description": ( - "Regex search across cached catalogs for tables matching a query. " - "Searches table names, table descriptions, column names, and column descriptions.\n" - "- query: case-insensitive regex. Plain keywords work as literals; use alternation " - "(orders|sales|revenue), anchors (^fact_), word boundaries (\\border\\b), and optional " - "groups (customers?) when useful. Escape . if you mean a literal dot.\n" - "- scope: 'all' (default), 'workspace', 'connected', '', or ':' " - "to restrict to a subtree (path is /-joined segments).\n" - "- exclude: optional regex on table name to drop hits (e.g. '_staging|_test').\n" - "- fields: subset of ['name','description','columns'] to restrict matching; default is all." + "Recursively find data below an optional exact source path and return flat typed results.\n" + "- query: optional case-insensitive regex; omit to enumerate selected descendants.\n" + "- source_id plus path: exact connected-source search root.\n" + "- filter_by: optionally return only folders or tables.\n" + "- fields restrict table matching to name, description, or columns.\n" + "After list_data inventory, use one query-less table-filtered call per source " + "to ground a broad overview in real table names.\n" + "If truncated, narrow query or path rather than paging the whole source." ), "parameters": { "type": "object", "properties": { - "query": {"type": "string", "description": "Case-insensitive regex."}, - "scope": {"type": "string", "description": "Search scope. Default: all"}, - "exclude": {"type": "string", "description": "Optional regex; drops hits whose name matches."}, + "query": {"type": "string", "description": "Optional case-insensitive regex. Omit to enumerate."}, + "source_id": {"type": "string", "description": "Optional connected source identifier."}, + "path": { + "type": "array", + "items": {"type": "string"}, + "description": "Exact folder path below which to search recursively. Requires source_id.", + }, + "filter_by": { + "type": "string", + "enum": ["folder", "table"], + "description": "Optional result node type.", + }, "fields": { "type": "array", "items": {"type": "string", "enum": ["name", "description", "columns"]}, "description": "Restrict matching to these fields. Default: all.", }, - "limit": {"type": "integer", "description": "Max results. Default 50, max 200."}, + "limit": {"type": "integer", "minimum": 1, "maximum": 500, "description": "Max results. Default 100."}, }, - "required": ["query"], + "required": [], }, }, }, @@ -789,11 +828,11 @@ def _build_connector_summary_block( *, max_total_chars: int = 1200, ) -> str: - """Render a compact directory of cached connector catalogs. + """Render a compact directory of currently loadable connectors. - Only shows source IDs with table counts (and folder counts when the - catalog is hierarchical). The agent is expected to call ``list_data`` - for full inventory. + Shows connected sources even before their catalog has been cached. Retained + catalogs for disconnected sources stay on disk but are not agent-visible. + The agent is expected to call ``list_data`` for full inventory. Strictly hard-capped at ``max_total_chars``. """ if not user_home: @@ -807,9 +846,19 @@ def _build_connector_summary_block( return " none" try: - source_ids = list_cached_sources(user_home) + from data_formulator.data_connector import ( + connector_is_available, + list_available_connector_ids, + ) + cached_source_ids = set(list_cached_sources(user_home)) + available_source_ids = set(list_available_connector_ids()) + source_ids = sorted( + source_id + for source_id in cached_source_ids | available_source_ids + if connector_is_available(source_id) is not False + ) except Exception: - logger.debug("connector summary: list_cached_sources failed", exc_info=True) + logger.debug("connector summary: source inventory failed", exc_info=True) return " none" if not source_ids: @@ -817,7 +866,7 @@ def _build_connector_summary_block( user_home_path = Path(user_home) lines: list[str] = [] - for sid in sorted(source_ids): + for sid in source_ids: try: tables = load_catalog(user_home_path, sid) or [] except Exception: @@ -825,18 +874,21 @@ def _build_connector_summary_block( tables = [] n, k = _summarize_catalog_shape(tables) if n == 0: - lines.append(f"- {sid}: 0 tables cached") + status = "connected, catalog not cached" if sid in available_source_ids else "0 tables cached" + lines.append(f"- {sid}: {status}") elif k > 0: + availability = "connected" if sid in available_source_ids else "catalog available" lines.append( - f"- {sid}: {n} table{'s' if n != 1 else ''} " + f"- {sid}: {availability}; {n} table{'s' if n != 1 else ''} " f"across {k} folder{'s' if k != 1 else ''}" ) else: - lines.append(f"- {sid}: {n} table{'s' if n != 1 else ''}") + availability = "connected" if sid in available_source_ids else "catalog available" + lines.append(f"- {sid}: {availability}; {n} table{'s' if n != 1 else ''}") lines.append( - " (call list_data() for sources, list_data(source_id, ...) to drill, " - "or find_data(query=...) to search)" + " (for a broad overview: call list_data(), then find_data(source_id=..., " + "filter_by='table', limit=10) for each source; do not ask which source first)" ) output = "\n".join(lines) @@ -1150,6 +1202,8 @@ def _execute_tool(self, name, args): return self._tool_execute_python(args) elif name == "show_user_data_preview": return self._tool_show_user_data_preview(args, scratch_jail) + elif name == "summarize_data_sources": + return self._tool_summarize_data_sources(args) elif name == "list_data": return self._tool_list_data(args) elif name == "find_data": @@ -1752,17 +1806,13 @@ def _tool_list_data(self, args): from data_formulator.data_operations import DataDiscoveryService return DataDiscoveryService(self.workspace).list_data(args) - def _tool_find_data(self, args): - """Regex search across cached catalogs. - - ``scope`` accepts: 'all' (default), 'workspace', 'connected', - '', or ':'. The - path-scoped form restricts catalog search to a subtree. + def _tool_summarize_data_sources(self, args): + """Return a bounded impression of every connected data source.""" + from data_formulator.data_operations import DataDiscoveryService + return DataDiscoveryService(self.workspace).summarize_data_sources(args) - Workspace tables are searched with a plain substring match (they're - small, regex-on-name has little extra value there). Catalog cache - search is regex-based. See design-docs §3.2. - """ + def _tool_find_data(self, args): + """Recursively find matching or enumerated data below an exact scope.""" from data_formulator.data_operations import DataDiscoveryService return DataDiscoveryService(self.workspace).find_data(args) diff --git a/py-src/data_formulator/agents/agent_utils.py b/py-src/data_formulator/agents/agent_utils.py index f8e24ada9..1754b004f 100644 --- a/py-src/data_formulator/agents/agent_utils.py +++ b/py-src/data_formulator/agents/agent_utils.py @@ -583,8 +583,8 @@ def generate_data_summary( Use WorkspaceWithTempData context manager to mount temp tables to workspace. When ``primary_tables`` is provided, the output is structured into tiered sections: - - **[PRIMARY TABLE]** / **[PRIMARY TABLES]**: Full detail for the tables the user is focused on. - - **[OTHER AVAILABLE TABLES]**: Full detail for the remaining tables. + - **[PRIMARY ANALYSIS INPUTS]**: Full detail for the input tables the user is focused on. + - **[OTHER ANALYSIS INPUTS]**: Full detail for the remaining input tables. Sections are omitted when empty. Args: @@ -737,10 +737,9 @@ def assemble_table_summary(table, idx): sections = [] if primary_parts: - header = "[PRIMARY TABLE]" if len(primary_parts) == 1 else "[PRIMARY TABLES]" - sections.append(header + "\n\n" + separator.join(primary_parts)) + sections.append("[PRIMARY ANALYSIS INPUTS]\n\n" + separator.join(primary_parts)) if other_parts: - sections.append("[OTHER AVAILABLE TABLES]\n\n" + separator.join(other_parts)) + sections.append("[OTHER ANALYSIS INPUTS]\n\n" + separator.join(other_parts)) return "\n\n".join(sections) # Join with visual separators (no tiering) diff --git a/py-src/data_formulator/agents/context.py b/py-src/data_formulator/agents/context.py index 8dc743c93..e24c2ec75 100644 --- a/py-src/data_formulator/agents/context.py +++ b/py-src/data_formulator/agents/context.py @@ -159,7 +159,7 @@ def build_lightweight_table_context( """Build compact table context with schema, metadata, value samples, and rows. When ``primary_tables`` is provided, tables are grouped into - [PRIMARY TABLE(S)] and [OTHER AVAILABLE TABLES] sections. + [PRIMARY ANALYSIS INPUTS] and [OTHER ANALYSIS INPUTS] sections. """ table_desc_cache, col_desc_cache, import_opts_cache = _get_workspace_metadata_lookups(workspace) table_extra_cache: dict[str, list[str]] = {} @@ -263,7 +263,7 @@ def _table_section(table: dict[str, Any]) -> str: return _client_schema_section(table, label) load_hint = ( - "\nThe tables above are the data already loaded into this workspace, and the " + "\nThe analysis input tables above are already materialized and are the " "only data you can read directly. Anything not listed here has not been loaded " "yet: find it in a connected source and propose loading it before relying on it.\n" "To load a table in code: pd.read_parquet('file.parquet') or " @@ -278,12 +278,11 @@ def _table_section(table: dict[str, Any]) -> str: sections = [] if primary_tables_list: - header = "[PRIMARY TABLE]" if len(primary_tables_list) == 1 else "[PRIMARY TABLES]" primary_parts = [_table_section(t) for t in primary_tables_list] - sections.append(header + "\n\n" + "\n\n".join(primary_parts)) + sections.append("[PRIMARY ANALYSIS INPUTS]\n\n" + "\n\n".join(primary_parts)) if other_tables_list: other_parts = [_table_section(t) for t in other_tables_list] - sections.append("[OTHER AVAILABLE TABLES]\n\n" + "\n\n".join(other_parts)) + sections.append("[OTHER ANALYSIS INPUTS]\n\n" + "\n\n".join(other_parts)) return "\n\n".join(sections) + "\n" + load_hint sections = [_table_section(table) for table in input_tables] @@ -375,6 +374,10 @@ def handle_read_catalog_metadata( if not user_home: return "Cannot read catalog metadata: user home not available." + from data_formulator.datalake.connector_preferences import connector_is_enabled + if not connector_is_enabled(user_home, source_id): + return f"Source '{source_id}' is disconnected." + # Surface zero-config admin connectors (e.g. sample_datasets) on first use. ensure_no_auth_catalogs_cached(user_home) diff --git a/py-src/data_formulator/analyst/agent.py b/py-src/data_formulator/analyst/agent.py index a9359d4f5..2e3e22582 100644 --- a/py-src/data_formulator/analyst/agent.py +++ b/py-src/data_formulator/analyst/agent.py @@ -81,6 +81,41 @@ # emitted match — never the same text pasted by a user or echoed by the model. _SKILL_LOADED_BANNER = "[SKILL LOADED: {name}]" _SKILL_LOADED_RE = re.compile(r"^\[SKILL LOADED: ([^\]]+)\]") +_SKILL_PRELOADED_RE = re.compile( + r"\[SKILL: ([^\]]+)\] Preloaded for this run" +) + +_TOOL_PROGRESS_ARG_KEYS: dict[str, tuple[str, ...]] = { + "summarize_data_sources": (), + "list_data": ("source_id", "path", "filter_by"), + "find_data": ("query", "source_id", "path", "filter_by"), + "describe_data": ("source_id", "table_key"), + "probe_data": ("source_id", "table_key", "query"), + "describe_connector": ("source_type",), + "inspect_chart": ("chart_id",), + "search_data_tables": ("query",), + "search_knowledge": ("query",), +} + + +def _tool_progress_args(tool_name: str, args: dict[str, Any]) -> dict[str, Any]: + """Return model arguments safe and useful for user-facing progress.""" + progress_args = { + key: args[key] + for key in _TOOL_PROGRESS_ARG_KEYS.get(tool_name, ()) + if key in args + } + if tool_name == "probe_data" and isinstance(progress_args.get("query"), dict): + query = progress_args["query"] + progress_args["query"] = { + key: query[key] + for key in ("aggregates", "group_by", "limit") + if key in query + } + filters = query.get("filters") + if isinstance(filters, list) and filters: + progress_args["query"]["filter_count"] = len(filters) + return progress_args # ── Action-argument coercion ────────────────────────────────────────────── # Weaker models sometimes JSON-encode a nested action argument as a string @@ -350,6 +385,14 @@ def _legal_actions(self) -> frozenset[str]: legal.update(meta.action_names) return frozenset(legal) + @staticmethod + def _initial_loaded_skills(input_tables: list[dict[str, Any]]) -> set[str]: + """Return the skill gates that must be open before the first LLM call.""" + loaded = {_CORE_SKILL} + if not input_tables: + loaded.add("data-loading") + return loaded + # ------------------------------------------------------------------ # Public API # ------------------------------------------------------------------ @@ -388,14 +431,15 @@ def run( iteration = completed_step_count final_status = "max_iterations" - # Reset per-run skill + payload state. ``core`` is auto-loaded: its - # baseline tools + actions are always available and its SKILL.md body is - # appended to the system frame (see _build_system_prompt). Gated skills - # are added to this set as the model loads them. The payload carries + # Reset per-run skill + payload state. ``core`` is always loaded. With + # no analysis input tables, data loading is the immediate workflow, so expose + # its tools, actions, and guidance before the first model call instead + # of spending a round on load_skill. Other gated skills are added as the + # model loads them. The payload carries # everything a dispatched skill handler needs to build its own context # (e.g. the report skill rebuilds [AVAILABLE CHARTS] + thread # context). - self._loaded_skills = {_CORE_SKILL} + self._loaded_skills = self._initial_loaded_skills(input_tables) self._run_payload = { "input_tables": input_tables, "charts": charts or [], @@ -663,6 +707,10 @@ def _rehydrate_loaded_skills(self, trajectory: list[dict]) -> None: name = self.registry.canonical_name(m.group(1).strip()) if self.registry.has(name): self._loaded_skills.add(name) + for preloaded in _SKILL_PRELOADED_RE.finditer(content): + name = self.registry.canonical_name(preloaded.group(1).strip()) + if self.registry.has(name): + self._loaded_skills.add(name) def _load_skill_into_context( self, name: str, trajectory: list[dict], @@ -1165,15 +1213,18 @@ def _build_system_prompt( context_lines = [] if has_primary_tables: context_lines.append( - "- **[PRIMARY TABLE(S)]**: The table(s) the user is focused on. " - "Prioritize these, but freely use other available tables if needed." + "- **[PRIMARY ANALYSIS INPUTS]**: The analysis input table(s) the " + "user is focused on. Prioritize these, but freely use other " + "analysis inputs if needed." ) context_lines.append( - "- **[OTHER AVAILABLE TABLES]**: Additional tables in the workspace." + "- **[OTHER ANALYSIS INPUTS]**: Additional materialized input " + "tables the analyst can read directly." ) else: context_lines.append( - "- **[AVAILABLE TABLES]**: All tables in the workspace." + "- **[ANALYSIS INPUT TABLES]**: All materialized root data inputs " + "the analyst can read directly." ) context_lines.append( " Use `inspect_source_data` to get detailed stats and sample rows. " @@ -1234,6 +1285,12 @@ def _build_system_prompt( f"\n\n[SKILL: {_CORE_SKILL}] Always-on baseline — these tools and " f"actions are active for the whole run.\n\n{core_body}" ) + for name in sorted(self._loaded_skills - {_CORE_SKILL}): + body = self.registry.load_body(name) + prompt += ( + f"\n\n[SKILL: {name}] Preloaded for this run — its tools and " + f"actions are active now.\n\n{body}" + ) if self._knowledge_store: knowledge_rules = self._knowledge_store.load_always_apply_rules() @@ -1277,7 +1334,7 @@ def _build_initial_messages( if primary_tables: user_content = f"{table_summaries}\n\n" else: - user_content = f"[AVAILABLE TABLES]\n\n{table_summaries}\n\n" + user_content = f"[ANALYSIS INPUT TABLES]\n\n{table_summaries}\n\n" if focused_block: user_content += f"{focused_block}\n\n" if peripheral_block: @@ -1597,10 +1654,12 @@ def _tool_loop( yield { "type": "tool_start", "tool": tool_name, + "args": _tool_progress_args(tool_name, tool_args), "purpose": tool_args.get("purpose") if tool_name == "execute_python_script" else None, "code": tool_args.get("code") if tool_name == "execute_python_script" else None, "table_names": tool_args.get("table_names") if tool_name == "inspect_source_data" else None, "skill": tool_args.get("name") if tool_name == "load_skill" else None, + "query": tool_args.get("query") if tool_name in ("search_data_tables", "search_knowledge") else None, } tool_t0 = time.time() diff --git a/py-src/data_formulator/analyst/skills/core/SKILL.md b/py-src/data_formulator/analyst/skills/core/SKILL.md index 15e4a6294..6775e0c1a 100644 --- a/py-src/data_formulator/analyst/skills/core/SKILL.md +++ b/py-src/data_formulator/analyst/skills/core/SKILL.md @@ -31,7 +31,7 @@ to use it well. variables do NOT persist between calls, so combine related steps into a single script. - **inspect_source_data(table_names)** — get schema, stats, and sample rows for - source tables (cheaper than `execute_python_script` for basic inspection). + analysis input tables (cheaper than `execute_python_script` for basic inspection). - **load_skill(name)** — load a skill's instructions into context so you can use the action it unlocks (see the Skills section of your system instructions). @@ -44,6 +44,9 @@ requires connected data that isn't present, call `load_skill("data-loading")` and follow that skill's discovery and immutable proposal workflow in this same conversation. Do not hand off to the standalone Data Loading agent. +When `[ANALYSIS INPUT TABLES]` is empty, the `data-loading` skill is already +loaded — use it to find out what is available and tell the user what you found. + The initial context already includes sample rows and statistics for each table. If the data is straightforward, go straight to the action without calling tools. Tool results are returned to you before you act. diff --git a/py-src/data_formulator/analyst/skills/core/tools.json b/py-src/data_formulator/analyst/skills/core/tools.json index 599293a22..16bf9e1ab 100644 --- a/py-src/data_formulator/analyst/skills/core/tools.json +++ b/py-src/data_formulator/analyst/skills/core/tools.json @@ -24,14 +24,14 @@ "type": "function", "function": { "name": "inspect_source_data", - "description": "Get a detailed summary of one or more source tables — schema, field-level statistics, and sample rows. Cheaper than execute_python_script for basic data inspection.", + "description": "Get a detailed summary of one or more analysis input tables — schema, field-level statistics, and sample rows. Cheaper than execute_python_script for basic data inspection.", "parameters": { "type": "object", "properties": { "table_names": { "type": "array", "items": { "type": "string" }, - "description": "List of workspace table names, as listed in the available-tables context, to inspect." + "description": "Names listed in the analysis-input-tables context to inspect." } }, "required": ["table_names"] @@ -61,7 +61,7 @@ "input_tables": { "type": "array", "items": { "type": "string" }, - "description": "Workspace table names, as listed in the available-tables context, that the code reads." + "description": "Names listed in the analysis-input-tables context that the code reads." }, "code": { "type": "string", diff --git a/py-src/data_formulator/analyst/skills/data-loading/SKILL.md b/py-src/data_formulator/analyst/skills/data-loading/SKILL.md index a43f70906..7d11ec078 100644 --- a/py-src/data_formulator/analyst/skills/data-loading/SKILL.md +++ b/py-src/data_formulator/analyst/skills/data-loading/SKILL.md @@ -11,12 +11,13 @@ when_to_use: >- already listed in the workspace context. always_on: false tools: - - list_data - - find_data - - describe_data - - probe_data - - list_connectors - - describe_connector + - summarize_data_sources + - list_data + - find_data + - describe_data + - probe_data + - list_connectors + - describe_connector actions: - propose_data_operation - propose_connection @@ -24,15 +25,41 @@ actions: # Skill: Data discovery -The workspace tables listed in your context are the data already loaded into the -system, and the only data that can be read directly. Everything these tools -return is *not* loaded yet — it lives in a connected source and only becomes -usable after the user selects a loading option and the server materializes it. +The analysis input tables listed in your context are already materialized and +are the only data that can be read directly. Everything these tools return is +*not* loaded yet — it lives in a connected source and only becomes usable after +the user selects a loading option and the server materializes it. Use these tools to determine whether connected sources contain data needed for the user's goal. They are read-only: discovering, describing, or probing a source does not add anything to the workspace analysis inputs. +## When nothing is loaded yet + +Discovery is cheap. For a broad question such as “what data can I load?”, follow +this sequence before answering: + +1. Call `summarize_data_sources({})` for a bounded overview of every connected source. +2. Summarize its hierarchy stats, top-level items, and sample tables directly. +3. Recommend concrete starting points. Use `list_data` or `find_data` only when + deeper navigation or search is needed. + +Use `list_data({source_id, path})` when the user wants to navigate a hierarchy, +and a queried `find_data` when they name a subject. Summary samples and top-level +items are bounded; respect their `omitted` counts. + +Pick the path that fits: + +- The user named a subject → `find_data`, then propose the tables that match. +- The user asked what data exists, or asked nothing specific → summarize every + connected source using `summarize_data_sources`, then propose useful starting points. +- Nothing is connected → `list_connectors`, then `propose_connection`, or say + they can upload a file. + +Never use `ask_user` to ask which connected source to inspect for a broad +availability question. Summarize them all with one bounded call and answer directly. +Use `ask_user` only for a choice that remains necessary after discovery. + ## Adding a connector When the user wants to connect a new source, do not merely ask them to navigate @@ -62,13 +89,15 @@ seeds and are removed from persisted UI state. 1. Use `find_data` when the user names a business concept or table. Use `list_data` when you need to browse available sources or hierarchy. + `list_data` returns one level; `find_data` searches recursively and may omit + `query` to enumerate folders or tables below an exact source path. 2. Use `describe_data` before relying on columns, types, row counts, or filter values. Pass the exact `source_id` and `table_key` returned by discovery. 3. Use `probe_data` only when metadata is insufficient to choose a useful bounded result. Probes are limited, read-only, and may be approximate. -4. First reconcile discoveries with every table in `[PRIMARY TABLE(S)]`, - `[OTHER AVAILABLE TABLES]`, or `[AVAILABLE TABLES]`. If the needed data is - already loaded, use or explain that workspace table instead of proposing it. +4. First reconcile discoveries with every table in `[PRIMARY ANALYSIS INPUTS]`, + `[OTHER ANALYSIS INPUTS]`, or `[ANALYSIS INPUT TABLES]`. If the needed data is + already loaded, use or explain that analysis input instead of proposing it. 5. When there are genuinely missing useful alternatives, call `propose_data_operation` with one to three complete immutable plans. This pauses for the user's choice; it @@ -90,6 +119,8 @@ yourself. Write it as you'd say it to a colleague, not as a schema summary. and one or more tables. The labels are buttons, not sentences — the reasoning belongs in your message text. The application displays table previews separately, so don't list columns as a substitute for explaining. +- An option is one coherent choice: one or a group of tables that serve the same + analysis, and leave out the ones that don't. - Use only source IDs, table keys, columns, and values grounded by discovery. - For a whole table, omit `query`. Use the optional raw-row query only when the request needs filters, projection, ordering, or an intentional limit. It uses diff --git a/py-src/data_formulator/analyst/skills/data-loading/skill.py b/py-src/data_formulator/analyst/skills/data-loading/skill.py index 8ee921710..cfbf1ac2d 100644 --- a/py-src/data_formulator/analyst/skills/data-loading/skill.py +++ b/py-src/data_formulator/analyst/skills/data-loading/skill.py @@ -33,7 +33,9 @@ def handle_tool( ctx: SkillContext, ) -> ToolResult: service = DataDiscoveryService(ctx.workspace) - if name == "list_data": + if name == "summarize_data_sources": + result = service.summarize_data_sources(args) + elif name == "list_data": result = service.list_data(args) elif name == "find_data": result = service.find_data(args) @@ -305,7 +307,8 @@ def _propose_data_operation( names = ", ".join(dict.fromkeys(loaded_tables)) raise ValueError( f"This proposal duplicates data already loaded in the workspace: {names}. " - "Use those workspace tables directly, explain their relevance, or propose only missing data." + "Use those analysis input tables directly, explain their relevance, " + "or propose only missing data." ) DataOperationRepository.for_workspace(ctx.workspace).create( operation, diff --git a/py-src/data_formulator/analyst/skills/data-loading/tools.json b/py-src/data_formulator/analyst/skills/data-loading/tools.json index 30db03417..042f6abca 100644 --- a/py-src/data_formulator/analyst/skills/data-loading/tools.json +++ b/py-src/data_formulator/analyst/skills/data-loading/tools.json @@ -1,15 +1,34 @@ [ + { + "type": "function", + "function": { + "name": "summarize_data_sources", + "description": "Return a bounded overview of every connected data source: hierarchy stats, top-level items, branch-diverse sample tables, and explicit omitted counts. Use this first for broad questions about what data is available.", + "parameters": { "type": "object", "properties": {}, "required": [] } + } + }, { "type": "function", "function": { "name": "list_data", - "description": "Browse cached connected-source catalogs. With no arguments, list source summaries. With source_id, list its top-level entries. Add path to browse direct children and filter for a case-insensitive substring match.", + "description": "List connected-source catalogs like ls. With no arguments, return immediate source nodes at the catalog root. With source_id and optional exact path, return immediate typed children only. Use filter_by for folders or tables and start_after when truncated. Use summarize_data_sources instead for a broad overview.", "parameters": { "type": "object", "properties": { - "source_id": { "type": "string", "description": "Connected source identifier. Omit for source summaries." }, - "path": { "type": "array", "items": { "type": "string" }, "description": "Hierarchy path segments." }, - "filter": { "type": "string", "description": "Substring filter on direct children." } + "source_id": { "type": "string", "description": "Connected source identifier. Omit for catalog-root source nodes." }, + "path": { "type": "array", "items": { "type": "string" }, "description": "Exact hierarchy path segments." }, + "filter_by": { "type": "string", "enum": ["folder", "table"], "description": "Optional immediate-child node type." }, + "limit": { "type": "integer", "minimum": 1, "maximum": 500, "description": "Maximum items. Default 100." }, + "start_after": { + "type": "object", + "description": "Exclusive continuation reference returned as next_start_after.", + "properties": { + "type": { "type": "string", "enum": ["folder", "table"] }, + "path": { "type": "array", "items": { "type": "string" } }, + "table_key": { "type": "string" } + }, + "required": ["type", "path"] + } }, "required": [] } @@ -19,21 +38,22 @@ "type": "function", "function": { "name": "find_data", - "description": "Regex search across cached connected-source catalogs and optionally existing workspace tables. Returns exact source_id and table_key values for follow-up inspection.", + "description": "Recursively find data below an optional exact source path. Query is an optional case-insensitive regex; omit it to enumerate descendants. Results are flat typed nodes with exact paths. Use summarize_data_sources instead for a broad overview.", "parameters": { "type": "object", "properties": { - "query": { "type": "string", "description": "Case-insensitive regex. Plain keywords work as literals." }, - "scope": { "type": "string", "description": "all, workspace, connected, a source_id, or source_id:path/segments." }, - "exclude": { "type": "string", "description": "Optional table-name exclusion regex." }, + "query": { "type": "string", "description": "Optional case-insensitive regex. Omit to enumerate." }, + "source_id": { "type": "string", "description": "Optional connected source identifier." }, + "path": { "type": "array", "items": { "type": "string" }, "description": "Exact recursive search root. Requires source_id." }, + "filter_by": { "type": "string", "enum": ["folder", "table"], "description": "Optional result node type." }, "fields": { "type": "array", "items": { "type": "string", "enum": ["name", "description", "columns"] }, "description": "Fields to search. Omit for all." }, - "limit": { "type": "integer" } + "limit": { "type": "integer", "minimum": 1, "maximum": 500 } }, - "required": ["query"] + "required": [] } } }, @@ -179,6 +199,7 @@ }, "tables": { "type": "array", + "description": "The tables that serve the same analysis.", "minItems": 1, "items": { "type": "object", diff --git a/py-src/data_formulator/analyst/skills/data_loading/SKILL.md b/py-src/data_formulator/analyst/skills/data_loading/SKILL.md deleted file mode 100644 index a43f70906..000000000 --- a/py-src/data_formulator/analyst/skills/data_loading/SKILL.md +++ /dev/null @@ -1,109 +0,0 @@ ---- -name: data-loading -description: >- - Discover connected data sources, add new data connectors through a - user-confirmed form, inspect table metadata, and run bounded read-only probes - when the current workspace data is insufficient. -when_to_use: >- - The user's question needs data that is not already available as a workspace - input, the user asks what connected data is available, or the user wants to - connect a database, warehouse, or cloud source. Not for analyzing tables - already listed in the workspace context. -always_on: false -tools: - - list_data - - find_data - - describe_data - - probe_data - - list_connectors - - describe_connector -actions: - - propose_data_operation - - propose_connection ---- - -# Skill: Data discovery - -The workspace tables listed in your context are the data already loaded into the -system, and the only data that can be read directly. Everything these tools -return is *not* loaded yet — it lives in a connected source and only becomes -usable after the user selects a loading option and the server materializes it. - -Use these tools to determine whether connected sources contain data needed for -the user's goal. They are read-only: discovering, describing, or probing a -source does not add anything to the workspace analysis inputs. - -## Adding a connector - -When the user wants to connect a new source, do not merely ask them to navigate -to settings and do not attempt to connect on their behalf. - -1. Call `list_connectors` first because available built-ins and plugins vary by - deployment. For a broad request such as "help me connect", summarize the - concrete available types and ask which one they use. -2. Once the source type is known, call `describe_connector` when field or auth - details are useful. -3. **When the requested source type is known and available, you MUST call - `propose_connection` in this same turn.** Do not stop with text such as - "I'll open the form", "you'll need to provide", or a list of required - fields. Only the action opens the form. Include one or two helpful sentences - alongside the action call explaining what the user should review or supply; - this text appears above the chat while the form opens on the canvas. Pass - `prefilled` values the user already supplied, including values parsed from a - connection string or config snippet. Never invent missing values. -4. The form is only a proposal. The user reviews it and clicks Connect; the - action must never connect automatically. - -Prefilled values may include credentials the user deliberately supplied. Do not -repeat those values in prose or subsequent tool output. They are transient form -seeds and are removed from persisted UI state. - -## Discovery sequence - -1. Use `find_data` when the user names a business concept or table. Use - `list_data` when you need to browse available sources or hierarchy. -2. Use `describe_data` before relying on columns, types, row counts, or filter - values. Pass the exact `source_id` and `table_key` returned by discovery. -3. Use `probe_data` only when metadata is insufficient to choose a useful - bounded result. Probes are limited, read-only, and may be approximate. -4. First reconcile discoveries with every table in `[PRIMARY TABLE(S)]`, - `[OTHER AVAILABLE TABLES]`, or `[AVAILABLE TABLES]`. If the needed data is - already loaded, use or explain that workspace table instead of proposing it. -5. When there are genuinely missing useful alternatives, call - `propose_data_operation` with one - to three complete immutable plans. This pauses for the user's choice; it - does not load data yet. - -## Proposing loading options - -Write your answer as **message text alongside the call** — that prose is what -the user reads, so it carries the whole answer. Do not put it in an action -field, and do not leave the call bare. Say what you went looking for, what you -actually found, and what each option would give them — enough that they can -choose without opening a single preview. Two to four sentences; more when the -options differ in ways that matter (grain, coverage, freshness, joins needed), -fewer when the choice is obvious. Name real tables and columns you saw during -discovery, and say plainly when an option is a compromise or when you'd pick one -yourself. Write it as you'd say it to a colleague, not as a schema summary. - -- Each `option` is a complete alternative: a concise action label (2–6 words) - and one or more tables. The labels are buttons, not sentences — the - reasoning belongs in your message text. The application displays table - previews separately, so don't list columns as a substitute for explaining. -- Use only source IDs, table keys, columns, and values grounded by discovery. -- For a whole table, omit `query`. Use the optional raw-row query only when the - request needs filters, projection, ordering, or an intentional limit. It uses - the same `filters` / `columns` / `order_by` / `limit` vocabulary as - `probe_data`, without aggregation. -- Do not invent operation IDs, plan IDs, or hashes. The server creates them. -- Never propose an exact connector query already represented by a workspace - table. The server also enforces this using persisted load provenance. - -## Grounding rules - -- Never invent source IDs, table keys, columns, or category values. -- Prefer cached catalog discovery before a live probe. -- Treat probe rows as evidence for planning, not as analysis input data. -- Keep queries structured and bounded. Do not generate source-specific SQL. -- If a source is unavailable or permissions changed, report the tool result and - ask the user for the needed connection or choose another source. \ No newline at end of file diff --git a/py-src/data_formulator/analyst/skills/data_loading/skill.py b/py-src/data_formulator/analyst/skills/data_loading/skill.py deleted file mode 100644 index 8ee921710..000000000 --- a/py-src/data_formulator/analyst/skills/data_loading/skill.py +++ /dev/null @@ -1,362 +0,0 @@ -from __future__ import annotations - -import json -from typing import Any, Generator - -from data_formulator.analyst.skills.base import Event, SkillContext, ToolResult -from data_formulator.data_operations import ( - ConnectorQueryStep, - DataDiscoveryService, - DataOperation, - DataOperationExecutor, - DataOperationPlan, - DataOperationRepository, - LoadQuery, - ProbeBudget, -) - -_PROBE_BUDGET_KEY = "data_loading.probe_budget" -_CONNECTORS_LISTED_KEY = "data_loading.connectors_listed" -_CONNECTORS_DISABLED_NOTE = ( - "External data connectors are disabled in this deployment. Use file upload " - "or built-in sample datasets instead." -) - - -class DataLoadingSkill: - """Read-only connected-source discovery for the unified analyst.""" - - def handle_tool( - self, - name: str, - args: dict[str, Any], - ctx: SkillContext, - ) -> ToolResult: - service = DataDiscoveryService(ctx.workspace) - if name == "list_data": - result = service.list_data(args) - elif name == "find_data": - result = service.find_data(args) - elif name == "describe_data": - result = service.describe_data(args) - elif name == "probe_data": - result = service.probe_data(args, self._probe_budget(ctx)) - elif name == "list_connectors": - result = self._list_connectors(ctx) - elif name == "describe_connector": - result = self._describe_connector(args) - else: - result = {"error": f"data-loading has no tool '{name}'."} - return ToolResult(text=json.dumps(result, ensure_ascii=False, default=str)) - - def handle_action( - self, - action: str, - spec: dict[str, Any], - ctx: SkillContext, - ) -> Generator[Event, None, str | None]: - if action == "propose_data_operation": - return (yield from self._propose_data_operation(spec, ctx)) - if action == "propose_connection": - return (yield from self._propose_connection(spec, ctx)) - message = f"data-loading has no committing action '{action}' in this phase." - yield { - "type": "error", - "message": message, - "message_code": "agent.unknownAction", - } - return message - - @staticmethod - def _connectors_disabled() -> bool: - try: - from flask import current_app - return bool(current_app.config.get("CLI_ARGS", {}).get("disable_data_connectors")) - except Exception: - return False - - @staticmethod - def _skill_state(ctx: SkillContext) -> dict[str, Any]: - state = ctx.payload.get("skill_state") - if not isinstance(state, dict): - state = {} - ctx.payload["skill_state"] = state - return state - - def _list_connectors(self, ctx: SkillContext) -> dict[str, Any]: - self._skill_state(ctx)[_CONNECTORS_LISTED_KEY] = True - if self._connectors_disabled(): - return {"connectors": [], "unavailable": [], "note": _CONNECTORS_DISABLED_NOTE} - - from data_formulator.data_loader import DATA_LOADERS, DISABLED_LOADERS - - connectors = [] - for key, loader_class in DATA_LOADERS.items(): - if key == "sample_datasets": - continue - try: - auth_mode = loader_class.auth_mode() - except Exception: - auth_mode = None - connectors.append({ - "type": key, - "name": loader_class.DISPLAY_NAME or key.replace("_", " ").title(), - "summary": loader_class.DESCRIPTION or "", - "auth_mode": auth_mode, - "available": True, - }) - return { - "connectors": connectors, - "unavailable": [ - { - "type": key, - "name": key.replace("_", " ").title(), - "install_hint": hint, - } - for key, hint in DISABLED_LOADERS.items() - if key != "sample_datasets" - ], - "next_action": ( - "If the user requested one of these connector types, call " - "propose_connection now. Do not end the turn by saying you will open a form." - ), - } - - def _describe_connector(self, args: dict[str, Any]) -> dict[str, Any]: - if self._connectors_disabled(): - return {"error": _CONNECTORS_DISABLED_NOTE} - - from data_formulator.data_loader import DATA_LOADERS, DISABLED_LOADERS - - source_type = str(args.get("source_type") or "").strip() - loader_class = DATA_LOADERS.get(source_type) - if loader_class is None: - hint = DISABLED_LOADERS.get(source_type) - detail = f" (needs: {hint})" if hint else "" - return {"error": f"Connector {source_type!r} is unavailable{detail}. Call list_connectors."} - - def safe(callable_): - try: - return callable_() - except Exception: - return None - - return { - "type": source_type, - "name": loader_class.DISPLAY_NAME or source_type.replace("_", " ").title(), - "summary": loader_class.DESCRIPTION or "", - "auth_mode": safe(loader_class.auth_mode), - "auth_paths": safe(loader_class.auth_paths), - "auth_instructions": safe(loader_class.auth_instructions), - "params": [ - { - "name": param.get("name"), - "required": bool(param.get("required")), - "tier": param.get("tier"), - "sensitive": bool(param.get("sensitive") or param.get("type") == "password"), - "description": param.get("description"), - } - for param in (safe(loader_class.list_params) or []) - if isinstance(param, dict) - ], - "next_action": ( - "Call propose_connection now to open this form. Describing the " - "requirements in text does not open it." - ), - } - - def _propose_connection( - self, - spec: dict[str, Any], - ctx: SkillContext, - ) -> Generator[Event, None, str | None]: - if self._connectors_disabled(): - yield {"type": "error", "message": _CONNECTORS_DISABLED_NOTE, "message_code": "agent.connectorsDisabled"} - return _CONNECTORS_DISABLED_NOTE - if not self._skill_state(ctx).get(_CONNECTORS_LISTED_KEY): - message = "Call list_connectors before propose_connection." - yield {"type": "error", "message": message, "message_code": "agent.invalidConnector"} - return message - - from data_formulator.data_loader import DATA_LOADERS, DISABLED_LOADERS - - source_type = str(spec.get("source_type") or "").strip() - if source_type not in DATA_LOADERS or source_type == "sample_datasets": - hint = DISABLED_LOADERS.get(source_type) - message = f"Connector {source_type!r} is unavailable" + (f" (needs: {hint})." if hint else ".") - yield {"type": "error", "message": message, "message_code": "agent.invalidConnector"} - return message - - prefilled_raw = spec.get("prefilled") or {} - prefilled = {} - if isinstance(prefilled_raw, dict): - prefilled = { - str(key): str(value) - for key, value in prefilled_raw.items() - if value not in (None, "") - } - display_name = DATA_LOADERS[source_type].DISPLAY_NAME or source_type - response = str(ctx.payload.get("action_narration") or "").strip() - yield { - "type": "interact", - "thought": spec.get("thought", ""), - "form": { - "kind": "connector", - "title": f"Connect to {display_name}", - "response": response or f"Complete the {display_name} connection form to add this data source.", - "connector": { - "source_type": source_type, - "prefilled": prefilled, - }, - }, - } - return None - - @staticmethod - def _already_loaded_tables(steps: tuple[ConnectorQueryStep, ...], workspace) -> list[str]: - metadata = workspace.get_metadata() - if metadata is None: - return [] - loaded: list[str] = [] - for step in steps: - expected_options = DataOperationExecutor._build_import_options(step) - for table_name, table_metadata in metadata.tables.items(): - if table_metadata.source_table != step.source_table: - continue - import_options = dict(table_metadata.import_options or {}) - provenance = import_options.pop("data_operation", {}) - same_source = not provenance or ( - provenance.get("source_id") in (None, step.source_id) - and provenance.get("table_key") in (None, step.table_key) - ) - if same_source and import_options == expected_options: - loaded.append(table_name) - break - return loaded - - @staticmethod - def _propose_data_operation( - spec: dict[str, Any], - ctx: SkillContext, - ) -> Generator[Event, None, str | None]: - try: - raw_plans = spec.get("options") - if not isinstance(raw_plans, list) or not 1 <= len(raw_plans) <= 3: - raise ValueError("propose_data_operation requires one to three options") - discovery = DataDiscoveryService(ctx.workspace) - resolved_plans: list[DataOperationPlan] = [] - for raw_plan in raw_plans: - raw_steps = raw_plan.get("tables") - if not isinstance(raw_steps, list) or not raw_steps: - raise ValueError("Each loading option requires at least one table") - steps: list[ConnectorQueryStep] = [] - for raw_step in raw_steps: - source_id = str(raw_step["source_id"]) - table_key = str(raw_step["table_key"]) - if not _source_is_available(source_id): - raise ValueError( - f"source {source_id!r} is not connected, so it cannot be loaded from. " - "Propose data from a connected source, or tell the user to reconnect it first." - ) - resolved = discovery.resolve_load_table(source_id, table_key) - if resolved is None: - raise ValueError( - f"table_key {table_key!r} was not found in source {source_id!r}" - ) - steps.append(ConnectorQueryStep( - source_id=source_id, - table_key=table_key, - display_name=str(resolved["display_name"]), - source_table=str(resolved["source_table"]), - source_table_name=( - str(resolved["source_table_name"]) - if resolved.get("source_table_name") is not None - else None - ), - query=LoadQuery.from_dict(raw_step.get("query")), - )) - resolved_plans.append(DataOperationPlan( - label=str(raw_plan["label"]).strip(), - summary="", - steps=tuple(steps), - )) - plans = tuple( - resolved_plans - ) - # The agent's own prose is the answer; `response` is only a fallback - # for models that emit a bare tool call with no accompanying text. - narration = str(ctx.payload.get("action_narration") or "").strip() - response = narration or str(spec.get("response", "")).strip() - operation = DataOperation( - reason="", - plans=plans, - description=response, - ) - if not operation.description or any(not plan.label for plan in plans): - raise ValueError( - "say what you found and why in your reply text, and give each option a label" - ) - conversation_id = str(ctx.payload.get("conversation_id", "")).strip() - loaded_tables = DataLoadingSkill._already_loaded_tables( - tuple(step for plan in plans for step in plan.steps), - ctx.workspace, - ) - if loaded_tables: - names = ", ".join(dict.fromkeys(loaded_tables)) - raise ValueError( - f"This proposal duplicates data already loaded in the workspace: {names}. " - "Use those workspace tables directly, explain their relevance, or propose only missing data." - ) - DataOperationRepository.for_workspace(ctx.workspace).create( - operation, - conversation_id=conversation_id, - ) - except (KeyError, TypeError, ValueError) as exc: - message = str(exc) - yield { - "type": "error", - "message": message, - "message_code": "agent.invalidDataOperation", - } - return message - - yield { - "type": "interact", - "thought": spec.get("thought", ""), - "data_operation": operation.to_public_dict(), - "questions": [{ - "text": operation.description, - "responseType": "single_choice", - "required": True, - "options": [ - {"label": plan.label, "value": plan.id} - for plan in operation.plans - ], - }], - } - return None - - @staticmethod - def _probe_budget(ctx: SkillContext) -> ProbeBudget: - state = ctx.payload.get("skill_state") - if not isinstance(state, dict): - state = {} - ctx.payload["skill_state"] = state - budget = state.get(_PROBE_BUDGET_KEY) - if not isinstance(budget, ProbeBudget): - budget = ProbeBudget() - state[_PROBE_BUDGET_KEY] = budget - return budget - - -def _source_is_available(source_id: str) -> bool: - """Only False when we can positively tell the source is unreachable.""" - try: - from data_formulator.data_connector import connector_is_available - return connector_is_available(source_id) is not False - except Exception: - return True - - -def get_skill() -> DataLoadingSkill: - return DataLoadingSkill() \ No newline at end of file diff --git a/py-src/data_formulator/analyst/skills/data_loading/tools.json b/py-src/data_formulator/analyst/skills/data_loading/tools.json deleted file mode 100644 index 30db03417..000000000 --- a/py-src/data_formulator/analyst/skills/data_loading/tools.json +++ /dev/null @@ -1,233 +0,0 @@ -[ - { - "type": "function", - "function": { - "name": "list_data", - "description": "Browse cached connected-source catalogs. With no arguments, list source summaries. With source_id, list its top-level entries. Add path to browse direct children and filter for a case-insensitive substring match.", - "parameters": { - "type": "object", - "properties": { - "source_id": { "type": "string", "description": "Connected source identifier. Omit for source summaries." }, - "path": { "type": "array", "items": { "type": "string" }, "description": "Hierarchy path segments." }, - "filter": { "type": "string", "description": "Substring filter on direct children." } - }, - "required": [] - } - } - }, - { - "type": "function", - "function": { - "name": "find_data", - "description": "Regex search across cached connected-source catalogs and optionally existing workspace tables. Returns exact source_id and table_key values for follow-up inspection.", - "parameters": { - "type": "object", - "properties": { - "query": { "type": "string", "description": "Case-insensitive regex. Plain keywords work as literals." }, - "scope": { "type": "string", "description": "all, workspace, connected, a source_id, or source_id:path/segments." }, - "exclude": { "type": "string", "description": "Optional table-name exclusion regex." }, - "fields": { - "type": "array", - "items": { "type": "string", "enum": ["name", "description", "columns"] }, - "description": "Fields to search. Omit for all." - }, - "limit": { "type": "integer" } - }, - "required": ["query"] - } - } - }, - { - "type": "function", - "function": { - "name": "describe_data", - "description": "Read cached metadata, columns, types, description, and row count for one discovered table.", - "parameters": { - "type": "object", - "properties": { - "source_id": { "type": "string" }, - "table_key": { "type": "string" } - }, - "required": ["source_id", "table_key"] - } - } - }, - { - "type": "function", - "function": { - "name": "probe_data", - "description": "Run a bounded read-only structured query against one connected table. Use only after describe_data. Results are evidence for planning and do not become workspace inputs.", - "parameters": { - "type": "object", - "properties": { - "source_id": { "type": "string" }, - "table_key": { "type": "string" }, - "query": { - "type": "object", - "properties": { - "filters": { - "type": "array", - "items": { - "type": "object", - "properties": { - "column": { "type": "string" }, - "op": { "type": "string", "enum": ["EQ", "NEQ", "GT", "GTE", "LT", "LTE", "IN", "ILIKE", "BETWEEN", "IS_NULL"] }, - "value": {} - }, - "required": ["column", "op"] - } - }, - "columns": { "type": "array", "items": { "type": "string" } }, - "group_by": { "type": "array", "items": { "type": "string" } }, - "aggregates": { - "type": "array", - "items": { - "type": "object", - "properties": { - "op": { "type": "string", "enum": ["count", "count_distinct", "sum", "avg", "min", "max"] }, - "column": { "type": "string" }, - "as": { "type": "string" } - }, - "required": ["op"] - } - }, - "order_by": { - "type": "array", - "items": { - "type": "object", - "properties": { - "column": { "type": "string" }, - "dir": { "type": "string", "enum": ["asc", "desc"] } - }, - "required": ["column"] - } - }, - "limit": { "type": "integer" } - } - } - }, - "required": ["source_id", "table_key"] - } - } - }, - { - "type": "function", - "function": { - "name": "list_connectors", - "description": "List connector types available in this deployment. Call this before propose_connection because built-ins, plugins, and missing dependencies vary by deployment. If the user's requested type is present, you MUST call propose_connection in the same turn; do not merely say you will open a form.", - "parameters": { - "type": "object", - "properties": {} - } - } - }, - { - "type": "function", - "function": { - "name": "describe_connector", - "description": "Return setup fields and authentication choices for one source_type returned by list_connectors. After this, call propose_connection in the same turn; describing fields does not open the form.", - "parameters": { - "type": "object", - "properties": { - "source_type": { "type": "string", "description": "Connector type key returned by list_connectors." } - }, - "required": ["source_type"] - } - } - }, - { - "type": "function", - "function": { - "name": "propose_connection", - "description": "REQUIRED terminal action when the user wants an available connector and its source_type is known. This is the only operation that opens the user-confirmed add-connector form on the canvas. Call list_connectors first. Prefill only values the user supplied; never invent credentials or connect automatically.", - "parameters": { - "type": "object", - "properties": { - "source_type": { "type": "string", "description": "Connector type key returned by list_connectors." }, - "prefilled": { - "type": "object", - "description": "Optional connector field values already supplied by the user. Values seed the live form and must not be repeated in prose.", - "additionalProperties": {} - } - }, - "required": ["source_type"] - } - } - }, - { - "type": "function", - "function": { - "name": "propose_data_operation", - "description": "Offer one to three complete immutable connected-data loading alternatives and pause for the user's selection. Discovery must ground every source, table, filter, and sort field. This does not execute a load.", - "parameters": { - "type": "object", - "properties": { - "response": { - "type": "string", - "description": "Fallback only. Leave empty when you narrate in your message text, which is what the user reads." - }, - "options": { - "type": "array", - "minItems": 1, - "maxItems": 3, - "items": { - "type": "object", - "properties": { - "label": { - "type": "string", - "description": "Concise action label, ideally 2-6 words." - }, - "tables": { - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "properties": { - "source_id": { "type": "string" }, - "table_key": { "type": "string" }, - "query": { - "type": "object", - "description": "Optional raw-row subset. Omit to load the whole table subject to server limits.", - "properties": { - "filters": { - "type": "array", - "items": { - "type": "object", - "properties": { - "column": { "type": "string" }, - "op": { "type": "string", "enum": ["EQ", "NEQ", "GT", "GTE", "LT", "LTE", "IN", "ILIKE", "BETWEEN", "IS_NULL"] }, - "value": {} - }, - "required": ["column", "op"] - } - }, - "columns": { "type": "array", "items": { "type": "string" } }, - "order_by": { - "type": "array", - "maxItems": 1, - "items": { - "type": "object", - "properties": { - "column": { "type": "string" }, - "dir": { "type": "string", "enum": ["asc", "desc"] } - }, - "required": ["column"] - } - }, - "limit": { "type": "integer", "minimum": 1 } - } - } - }, - "required": ["source_id", "table_key"] - } - } - }, - "required": ["label", "tables"] - } - } - }, - "required": ["options"] - } - } - } -] \ No newline at end of file diff --git a/py-src/data_formulator/analyst/skills/report/SKILL.md b/py-src/data_formulator/analyst/skills/report/SKILL.md index 1397fcd52..245fde154 100644 --- a/py-src/data_formulator/analyst/skills/report/SKILL.md +++ b/py-src/data_formulator/analyst/skills/report/SKILL.md @@ -38,7 +38,7 @@ all chart/data inspection first — once you call `write_report`, the report is delivered as-is and the run ends. ## Context available to you -- **[PRIMARY TABLE(S)]** / **[OTHER AVAILABLE TABLES]**: Lightweight schema of datasets. +- **[PRIMARY ANALYSIS INPUTS]** / **[OTHER ANALYSIS INPUTS]**: Lightweight schema of materialized input datasets. - **[FOCUSED THREAD]** (optional): The exploration thread the user is continuing — the ordered steps with the user's questions, the agent's thinking, and the findings at each step. This is the spine of the story you are telling. diff --git a/py-src/data_formulator/analyst/tools.py b/py-src/data_formulator/analyst/tools.py index cde1f34f0..854c595ac 100644 --- a/py-src/data_formulator/analyst/tools.py +++ b/py-src/data_formulator/analyst/tools.py @@ -9,7 +9,7 @@ - ``execute_python_script`` — run a general-purpose Python script in the sandbox to inspect/compute (stdout returned). - - ``inspect_source_data`` — schema + stats + sample rows for source tables. + - ``inspect_source_data`` — schema + stats + sample rows for analysis inputs. - ``load_skill`` — pull a skill's ``SKILL.md`` body into context, unlocking its gated actions (progressive disclosure; reading a doc is read-only). @@ -53,8 +53,8 @@ "function": { "name": "inspect_source_data", "description": ( - "Get a detailed summary of one or more source tables — schema, " - "field-level statistics, and sample rows. Cheaper than explore() " + "Get a detailed summary of one or more analysis input tables — schema, " + "field-level statistics, and sample rows. Cheaper than explore() " "for basic data inspection." ), "parameters": { @@ -63,7 +63,7 @@ "table_names": { "type": "array", "items": {"type": "string"}, - "description": "List of workspace table names, as listed in the available-tables context, to inspect.", + "description": "Names listed in the analysis-input-tables context to inspect.", }, }, "required": ["table_names"], diff --git a/py-src/data_formulator/data_connector.py b/py-src/data_formulator/data_connector.py index 3fb3f837e..0b4510674 100644 --- a/py-src/data_formulator/data_connector.py +++ b/py-src/data_formulator/data_connector.py @@ -466,6 +466,7 @@ def get_frontend_config(self, include_pinned_in_form: bool = False) -> dict[str, "icon": self._icon, "params_form": form_fields, "pinned_params": pinned_params, + "connection_identity": self._loader_class.connection_identity(self._default_params), "hierarchy": _hierarchy_dicts(full_hierarchy), "effective_hierarchy": _hierarchy_dicts(effective), "auth_instructions": self._loader_class.auth_instructions(), @@ -761,15 +762,16 @@ def _try_sso_auto_connect(self, identity: str) -> ExternalDataLoader | None: def _require_loader(self) -> ExternalDataLoader: identity = self._get_identity() + from data_formulator.datalake.connector_preferences import connector_is_enabled + from data_formulator.datalake.workspace import get_user_home + if not connector_is_enabled(get_user_home(identity), self._source_id): + raise ValueError("Connector is disconnected. Please connect first.") loader = self._loaders.get(identity) if loader is not None: return loader - # No-auth connectors (e.g. built-in example datasets) are always - # available — there's nothing to connect, so lazily instantiate and - # cache the loader on first use. This mirrors the ``auth_mode == "none"`` - # special-casing in the connect/get-status/preview/import endpoints and - # keeps no-auth sources working for catalog/preview/import even when - # external data connectors are disabled (e.g. ephemeral/demo mode). + # Enabled no-auth connectors need no setup, so lazily instantiate and + # cache the loader on first use. The preference check above keeps a + # user-disconnected built-in unavailable to both UI and agent paths. if _loader_auth_mode(self._loader_class) == "none": loader = self._loader_class() self._loaders[identity] = loader @@ -844,6 +846,45 @@ def resolve_catalog_refresh_target( return loader_class, loader +def _connector_connection_status( + connector: DataConnector, + identity: str | None, + *, + sso_token: Any = None, + token_store: Any = None, +) -> tuple[bool, bool, bool]: + """Return ``(connected, has_stored_credentials, sso_auto_connect)``.""" + enabled = True + if identity: + from data_formulator.datalake.connector_preferences import connector_is_enabled + from data_formulator.datalake.workspace import get_user_home + enabled = connector_is_enabled(get_user_home(identity), connector._source_id) + if not enabled: + return False, False, False + + auth_mode = _loader_auth_mode(connector._loader_class) + if auth_mode == "none": + return True, False, False + if not identity: + return False, False, False + + has_stored = connector.has_stored_credentials(identity) + connected = connector._get_loader(identity) is not None or has_stored + if connected: + return True, has_stored, False + + sso_auto = False + if sso_token is not None and auth_mode in ("token", "sso_exchange", "delegated"): + if token_store is None: + from data_formulator.auth.token_store import TokenStore + token_store = TokenStore() + sso_auto = ( + not token_store.is_sso_reconnect_blocked(connector._source_id) + and bool(connector._default_params.get("url")) + ) + return False, has_stored, sso_auto + + def connector_is_available(source_id: str) -> bool | None: """Whether ``source_id`` could be loaded from right now, without touching it. @@ -858,26 +899,55 @@ def connector_is_available(source_id: str) -> bool | None: except Exception: return None try: - if _loader_auth_mode(connector._loader_class) == "none": - return True identity = connector._get_identity() - if connector._get_loader(identity) is not None: - return True - if connector.has_stored_credentials(identity): - return True from data_formulator.auth.identity import get_sso_token - from data_formulator.auth.token_store import TokenStore - auth_mode = _loader_auth_mode(connector._loader_class) - return ( - auth_mode in ("token", "sso_exchange", "delegated") - and not TokenStore().is_sso_reconnect_blocked(source_id) - and get_sso_token() is not None + connected, _has_stored, sso_auto = _connector_connection_status( + connector, + identity, + sso_token=get_sso_token(), ) + return connected or sso_auto except Exception: logger.debug("availability check failed for %s", source_id, exc_info=True) return None +def list_available_connector_ids() -> list[str]: + """Return connector IDs the current identity can load from.""" + try: + identity = DataConnector._get_identity() + except Exception: + return [] + + sso_token = None + token_store = None + try: + from data_formulator.auth.identity import get_sso_token + sso_token = get_sso_token() + if sso_token is not None: + from data_formulator.auth.token_store import TokenStore + token_store = TokenStore() + except Exception: + logger.debug("SSO status unavailable for connector inventory", exc_info=True) + + available: list[str] = [] + for registry_key, connector, _is_admin in _visible_connector_items(identity): + public_id = _public_connector_id(registry_key, connector) + try: + connected, _has_stored, sso_auto = _connector_connection_status( + connector, + identity, + sso_token=sso_token, + token_store=token_store, + ) + except Exception: + logger.debug("availability check failed for %s", public_id, exc_info=True) + continue + if connected or sso_auto: + available.append(public_id) + return available + + def _parse_source_table(raw: Any) -> tuple[str, str]: """Normalise the ``source_table`` value from a request body. @@ -1273,31 +1343,11 @@ def list_connectors(): result = [] for registry_key, connector, is_admin in _visible_connector_items(identity): - has_stored = False - connected = False - auth_mode = _loader_auth_mode(connector._loader_class) - if auth_mode == "none": - # No-auth connectors (e.g. built-in example datasets) are always - # available — there's no credential to store and no connection - # to establish. - connected = True - elif identity: - has_stored = connector.has_stored_credentials(identity) - connected = ( - connector._get_loader(identity) is not None - or has_stored - ) - sso_blocked = ( - token_store.is_sso_reconnect_blocked(connector._source_id) - if token_store else False - ) - # SSO auto-connect: auth-capable loader + user has SSO token + URL is pinned - sso_auto = ( - not connected - and sso_token is not None - and auth_mode in ("token", "sso_exchange", "delegated") - and not sso_blocked - and bool(connector._default_params.get("url")) + connected, has_stored, sso_auto = _connector_connection_status( + connector, + identity, + sso_token=sso_token, + token_store=token_store, ) cfg = connector.get_frontend_config(include_pinned_in_form=not is_admin) public_id = _public_connector_id(registry_key, connector) @@ -1314,6 +1364,7 @@ def list_connectors(): "sso_auto_connect": sso_auto, "params_form": cfg["params_form"], "pinned_params": cfg["pinned_params"], + "connection_identity": cfg["connection_identity"], "hierarchy": cfg["hierarchy"], "effective_hierarchy": cfg["effective_hierarchy"], "auth_mode": cfg["auth_mode"], @@ -1351,11 +1402,18 @@ def create_connector(): if not loader_class: raise AppError(ErrorCode.INVALID_REQUEST, f"Unknown loader type: {loader_type}") - display_name = data.get("display_name", loader_type.replace("_", " ").title()) + display_name = data.get("display_name") icon = data.get("icon", loader_type) raw_params = data.get("params", {}) default_params = _connector_config_params(loader_class, raw_params) + if not display_name: + # A connector is its type plus which instance it points at, so name it + # that way unless the user said otherwise. + type_name = loader_class.DISPLAY_NAME or loader_type.replace("_", " ").title() + identity = loader_class.connection_identity(default_params) + display_name = f"{type_name} · {identity}" if identity else type_name + try: identity = DataConnector._get_identity() except Exception as e: @@ -1626,12 +1684,16 @@ def connector_connect(): data = request.get_json() or {} source = _resolve_connector(data) - # No-auth connectors (e.g. built-in example datasets) have nothing to - # connect — they're always available. Return a synthetic success - # response so any (legacy) frontend code that still calls connect is - # a no-op rather than an error. + identity = source._get_identity() + from data_formulator.datalake.connector_preferences import set_connector_enabled + from data_formulator.datalake.workspace import get_user_home + + # No-auth connectors have no form to submit. Connecting simply re-enables + # access to the existing loader and preserved catalog. if _loader_auth_mode(source._loader_class) == "none": + set_connector_enabled(get_user_home(identity), source._source_id, True) loader = source._loader_class() + source._loaders[identity] = loader return json_ok({ "status": "connected", "persisted": False, @@ -1666,6 +1728,8 @@ def connector_connect(): source._loaders.pop(identity, None) raise AppError(ErrorCode.DB_CONNECTION_FAILED, "Connection test failed") + set_connector_enabled(get_user_home(identity), source._source_id, True) + persisted = False if persist: persisted = source._persist_credentials(user_params) @@ -1748,19 +1812,14 @@ def connector_disconnect(): data = request.get_json() or {} source = _resolve_connector(data) - # No-auth connectors (e.g. built-in example datasets) cannot be - # disconnected — they have no credentials to clear and are intentionally - # always available. - if _loader_auth_mode(source._loader_class) == "none": - raise AppError( - ErrorCode.INVALID_REQUEST, - "This connector is always available and cannot be disconnected.", - ) - try: identity = source._get_identity() + from data_formulator.datalake.connector_preferences import set_connector_enabled + from data_formulator.datalake.workspace import get_user_home + set_connector_enabled(get_user_home(identity), source._source_id, False) source._loaders.pop(identity, None) - source._vault_delete(identity) + if _loader_auth_mode(source._loader_class) != "none": + source._vault_delete(identity) try: from data_formulator.auth.token_store import TokenStore TokenStore().clear_service_token(source._source_id) @@ -1783,8 +1842,14 @@ def connector_get_status(): data = request.get_json() or {} source = _resolve_connector(data) - # No-auth connectors are always connected. + identity = source._get_identity() + from data_formulator.datalake.connector_preferences import connector_is_enabled + from data_formulator.datalake.workspace import get_user_home + if _loader_auth_mode(source._loader_class) == "none": + enabled = connector_is_enabled(get_user_home(identity), source._source_id) + if not enabled: + return json_ok({"connected": False, "persisted": False}) loader = source._loader_class() return json_ok({ "connected": True, diff --git a/py-src/data_formulator/data_loader/databricks_data_loader.py b/py-src/data_formulator/data_loader/databricks_data_loader.py index 99490041d..7cf6aa7bd 100644 --- a/py-src/data_formulator/data_loader/databricks_data_loader.py +++ b/py-src/data_formulator/data_loader/databricks_data_loader.py @@ -43,6 +43,9 @@ class DatabricksDataLoader(ExternalDataLoader): DISPLAY_NAME = "Databricks" DESCRIPTION = "Query Databricks Unity Catalog tables through a SQL warehouse." + # http_path routes to a warehouse; the workspace host is what names the instance. + IDENTITY_PARAMS = ("server_hostname",) + @staticmethod def list_params() -> list[dict[str, Any]]: return [ diff --git a/py-src/data_formulator/data_loader/external_data_loader.py b/py-src/data_formulator/data_loader/external_data_loader.py index 96b79f8c3..4cef27673 100644 --- a/py-src/data_formulator/data_loader/external_data_loader.py +++ b/py-src/data_formulator/data_loader/external_data_loader.py @@ -33,6 +33,31 @@ def apply_import_projection( logger = logging.getLogger(__name__) +def _concise_identity(value: str) -> str: + """Reduce a connection param to the part a human recognises. + + URLs collapse to their host (``https://x.kusto.windows.net/`` -> ``x.kusto.windows.net``) + and home directories to ``~`` so identities stay short and screenshot-safe. + """ + trimmed = value.strip().rstrip("/\\") + if not trimmed: + return "" + if "://" in trimmed: + from urllib.parse import urlparse + host = urlparse(trimmed).netloc + if host: + return host + if trimmed.startswith(("/", "~")) or (len(trimmed) > 2 and trimmed[1] == ":"): + from pathlib import Path + try: + home = str(Path.home()) + if trimmed.startswith(home): + return "~" + trimmed[len(home):] + except Exception: + pass + return trimmed + + @dataclass(frozen=True) class CatalogCachePolicy: listing_ttl_seconds: int | None = 21_600 @@ -709,6 +734,43 @@ def auth_instructions(cls) -> str: #: back to ``DISPLAY_NAME``. This is NOT the verbose ``auth_instructions``. DESCRIPTION: str | None = None + #: Params naming *which* instance of this source a connector points at + #: (cluster, host, bucket…), most significant first. When ``None`` the + #: identity is derived from the required, non-advanced connection params, + #: which is right for most loaders; override where that picks up routing + #: detail rather than identity (Databricks' ``http_path``, S3's region). + IDENTITY_PARAMS: tuple[str, ...] | None = None + + @classmethod + def identity_params(cls) -> list[str]: + """Return the param names that identify this connector's instance.""" + if cls.IDENTITY_PARAMS is not None: + return list(cls.IDENTITY_PARAMS) + return [ + p["name"] for p in cls.list_params() + if p.get("tier") == "connection" + and p.get("required") + and not p.get("advanced") + and not p.get("sensitive") + ][:2] + + @classmethod + def connection_identity(cls, params: dict[str, Any]) -> str: + """Render the connection's identity, e.g. ``"mycluster.kusto.windows.net · sales"``. + + Returns an empty string when no identifying param has a value, which + is the normal case for loaders that take no connection params at all. + """ + parts: list[str] = [] + for name in cls.identity_params(): + value = params.get(name) + if value is None: + continue + concise = _concise_identity(str(value)) + if concise and concise not in parts: + parts.append(concise) + return " · ".join(parts) + @staticmethod def delegated_login_config() -> dict[str, Any] | None: """Return config for delegated (popup-based) token login, or None. diff --git a/py-src/data_formulator/data_loader/s3_data_loader.py b/py-src/data_formulator/data_loader/s3_data_loader.py index dd8403a0a..1f4c61bb5 100644 --- a/py-src/data_formulator/data_loader/s3_data_loader.py +++ b/py-src/data_formulator/data_loader/s3_data_loader.py @@ -20,6 +20,8 @@ class S3DataLoader(ExternalDataLoader): DISPLAY_NAME = "Amazon S3" DESCRIPTION = "Load CSV, JSON, or Parquet files from an Amazon S3 bucket." + IDENTITY_PARAMS = ("bucket",) + @staticmethod def list_params() -> list[dict[str, Any]]: params_list = [ diff --git a/py-src/data_formulator/data_loader/sample_datasets_loader.py b/py-src/data_formulator/data_loader/sample_datasets_loader.py index 70172fafb..4a767564f 100644 --- a/py-src/data_formulator/data_loader/sample_datasets_loader.py +++ b/py-src/data_formulator/data_loader/sample_datasets_loader.py @@ -47,6 +47,8 @@ class SampleDatasetsLoader(ExternalDataLoader): """Browse and import the built-in sample datasets.""" + DISPLAY_NAME = "Sample Datasets" + # ------------------------------------------------------------------ # Metadata # ------------------------------------------------------------------ @@ -59,10 +61,9 @@ def list_params() -> list[dict[str, Any]]: @staticmethod def auth_mode() -> str: - # ``"none"`` declares that this loader needs no authentication and no - # connection setup. The connector framework treats such loaders as - # always-on: they cannot be connected/disconnected, expose no - # credentials UI, and are always reported as ``connected: true``. + # ``"none"`` declares that this loader needs no authentication or + # connection form. Users can still disable its availability through + # the connector preference managed by the framework. return "none" @staticmethod diff --git a/py-src/data_formulator/data_operations/discovery.py b/py-src/data_formulator/data_operations/discovery.py index f0af874f4..27a6f09fc 100644 --- a/py-src/data_formulator/data_operations/discovery.py +++ b/py-src/data_formulator/data_operations/discovery.py @@ -50,11 +50,25 @@ def ensure_catalogs_current(user_home: Any) -> dict[str, Any]: return {} snapshots: dict[str, Any] = {} try: - from data_formulator.data_connector import _ADMIN_CONNECTOR_IDS + from data_formulator.data_connector import ( + _ADMIN_CONNECTOR_IDS, + connector_is_available, + list_available_connector_ids, + ) from data_formulator.datalake.catalog_cache import list_cached_sources + from data_formulator.datalake.connector_preferences import connector_is_enabled from data_formulator.datalake.catalog_refresh import ensure_catalog_freshness - source_ids = set(list_cached_sources(user_home)) | set(_ADMIN_CONNECTOR_IDS) + source_ids = ( + set(list_cached_sources(user_home)) + | set(_ADMIN_CONNECTOR_IDS) + | set(list_available_connector_ids()) + ) + source_ids = { + source_id for source_id in source_ids + if connector_is_enabled(user_home, source_id) + and connector_is_available(source_id) is not False + } for source_id in source_ids: snapshot = ensure_catalog_freshness(Path(user_home), source_id) if snapshot is not None: @@ -79,12 +93,63 @@ def _freshness_payload(snapshot: Any) -> dict[str, Any]: } +def _source_is_discoverable(source_id: str) -> bool: + """Hide sources known to be disconnected; keep unknown status compatible.""" + try: + from data_formulator.data_connector import connector_is_available + return connector_is_available(source_id) is not False + except Exception: + logger.debug("Connector availability unavailable for %s", source_id, exc_info=True) + return True + + class DataDiscoveryService: """Read-only catalog discovery shared by data-loading entry points.""" def __init__(self, workspace: Any): self.workspace = workspace + @staticmethod + def _connected_source_inventory( + user_home: Any, + snapshots: dict[str, Any], + ) -> list[dict[str, Any]]: + from data_formulator.datalake.catalog_cache import list_sources_summary + + try: + sources = list_sources_summary(user_home) + except Exception: + logger.debug("connected source inventory failed", exc_info=True) + sources = [] + try: + from data_formulator.data_connector import list_available_connector_ids + summarized_ids = {source.get("source_id") for source in sources} + sources.extend({ + "source_id": source_id, + "table_count": 0, + "is_hierarchical": False, + "connected": True, + "catalog_status": "not_cached", + } for source_id in list_available_connector_ids() if source_id not in summarized_ids) + except Exception: + logger.debug("available connector inventory failed", exc_info=True) + + sources = [ + source for source in sources + if not source.get("source_id") + or _source_is_discoverable(source["source_id"]) + ] + for source in sources: + source_id = source.get("source_id") + snapshot = snapshots.get(source_id) + if snapshot and ( + snapshot.listing_freshness != "fresh" + or snapshot.metadata_freshness != "fresh" + or snapshot.last_refresh_error + ): + source["freshness"] = _freshness_payload(snapshot) + return sorted(sources, key=lambda source: source.get("source_id", "")) + def list_data(self, args: dict[str, Any]) -> dict[str, Any]: from data_formulator.datalake.catalog_cache import ( list_path_children, @@ -93,33 +158,41 @@ def list_data(self, args: dict[str, Any]) -> dict[str, Any]: user_home = getattr(self.workspace, "user_home", None) if not user_home: - return {"sources": []} + return {"path": [], "items": [], "total_count": 0, "truncated": False} snapshots = ensure_catalogs_current(user_home) source_id = (args.get("source_id") or "").strip() if not source_id: + sources = self._connected_source_inventory(user_home, snapshots) + items = [{ + "type": "source", + "name": source["source_id"], + "path": [source["source_id"]], + **{key: value for key, value in source.items() if key != "source_id"}, + } for source in sources] + return { + "path": [], + "items": items, + "total_count": len(items), + "truncated": False, + } + + from data_formulator.datalake.connector_preferences import connector_is_enabled + if not connector_is_enabled(user_home, source_id) or not _source_is_discoverable(source_id): + return {"error": f"Source '{source_id}' is disconnected."} + + from data_formulator.datalake.catalog_cache import list_cached_sources + if source_id not in set(list_cached_sources(user_home)): try: - sources = list_sources_summary(user_home) - except Exception: - logger.debug("list_data: list_sources_summary failed", exc_info=True) - return {"sources": []} - # Mark unreachable sources so the agent steers around them instead - # of proposing a load that can only fail. - try: - from data_formulator.data_connector import connector_is_available - for source in sources: - sid = source.get("source_id") or source.get("id") - if sid in snapshots and ( - snapshots[sid].listing_freshness != "fresh" - or snapshots[sid].metadata_freshness != "fresh" - or snapshots[sid].last_refresh_error - ): - source["freshness"] = _freshness_payload(snapshots[sid]) - if sid and connector_is_available(sid) is False: - source["connected"] = False - except Exception: - logger.debug("list_data: availability check failed", exc_info=True) - return {"sources": sources} + from data_formulator.data_connector import resolve_live_loader + from data_formulator.datalake.catalog_refresh import ensure_catalog_freshness + resolve_live_loader(source_id) + snapshot = ensure_catalog_freshness(user_home, source_id) + if snapshot is not None: + snapshots[source_id] = snapshot + except Exception as exc: + logger.debug("list_data: catalog bootstrap failed", exc_info=True) + return {"error": f"Source '{source_id}' is connected but its catalog could not be loaded: {exc}"} path = args.get("path") or [] if not isinstance(path, list): @@ -130,7 +203,9 @@ def list_data(self, args: dict[str, Any]) -> dict[str, Any]: user_home, source_id, path=path, - filter=args.get("filter"), + filter_by=args.get("filter_by"), + limit=args.get("limit") or 100, + start_after=args.get("start_after"), ) if source_id in snapshots: result["freshness"] = _freshness_payload(snapshots[source_id]) @@ -139,86 +214,141 @@ def list_data(self, args: dict[str, Any]) -> dict[str, Any]: logger.debug("list_data: list_path_children failed", exc_info=True) return {"error": f"list_data failed: {exc}"} + def summarize_data_sources(self, args: dict[str, Any]) -> dict[str, Any]: + from data_formulator.datalake.catalog_cache import summarize_catalog_sources + + user_home = getattr(self.workspace, "user_home", None) + if not user_home: + return {"sources": []} + snapshots = ensure_catalogs_current(user_home) + inventory = self._connected_source_inventory(user_home, snapshots) + try: + cached = { + source["source_id"]: source + for source in summarize_catalog_sources(user_home) + } + except Exception: + logger.debug("summarize_data_sources: catalog summary failed", exc_info=True) + cached = {} + + sources: list[dict[str, Any]] = [] + for source in inventory: + source_id = source["source_id"] + summary = cached.get(source_id, { + "source_id": source_id, + "table_count": source.get("table_count", 0), + "folder_count": 0, + "max_depth": 0, + "top_level": [], + "sample_tables": [], + "omitted": {"top_level": 0, "tables": 0}, + }) + if source.get("catalog_status"): + summary["catalog_status"] = source["catalog_status"] + if source.get("freshness"): + summary["freshness"] = source["freshness"] + sources.append(summary) + return {"sources": sources} + def find_data(self, args: dict[str, Any]) -> dict[str, Any]: from data_formulator.datalake.catalog_cache import ( CatalogSearchError, + find_catalog_cache, list_cached_sources, - search_catalog_cache, ) - query = (args.get("query") or "").strip() - if not query: - return {"error": "query is required"} + query = (args.get("query") or "").strip() or None + source_id = (args.get("source_id") or "").strip() + path = args.get("path") or [] + if not isinstance(path, list): + return {"error": "path must be an array of strings"} + path = [str(segment) for segment in path] + if path and not source_id: + return {"error": "path requires source_id"} + + filter_by = (args.get("filter_by") or "").strip() or None + if filter_by not in {None, "folder", "table"}: + return {"error": "filter_by must be 'folder' or 'table'"} - scope_raw = (args.get("scope") or "all").strip() - exclude = args.get("exclude") or None fields = args.get("fields") or None limit = args.get("limit") try: - limit = max(1, min(int(limit), 200)) if limit else 50 + limit = max(1, min(int(limit), 500)) if limit else 100 except (TypeError, ValueError): - limit = 50 - - search_workspace = False - source_ids: list[str] | None = None - path_prefix: list[str] | None = None - - if scope_raw == "all": - search_workspace = True - elif scope_raw == "workspace": - search_workspace = True - source_ids = [] - elif scope_raw == "connected": - pass - elif ":" in scope_raw: - source_id, _, path_str = scope_raw.partition(":") - source_ids = [source_id.strip()] if source_id.strip() else [] - path_prefix = [segment for segment in path_str.split("/") if segment] - else: - source_ids = [scope_raw] + limit = 100 + + search_workspace = not source_id + source_ids = [source_id] if source_id else None user_home = getattr(self.workspace, "user_home", None) snapshots = ensure_catalogs_current(user_home) results: list[dict[str, Any]] = [] + workspace_truncated = False - if search_workspace: + if search_workspace and filter_by != "folder": try: - metadata = self.workspace.get_metadata() - if metadata: - for hit in metadata.search_tables(query, limit=min(limit, 50)): + if query: + metadata = self.workspace.get_metadata() + workspace_hits = ( + metadata.search_tables(query, limit=min(limit + 1, 501)) + if metadata else [] + ) + workspace_truncated = len(workspace_hits) > limit + for hit in workspace_hits[:limit]: results.append({ + "type": "table", "source": "workspace", "name": hit["name"], + "path": [hit["name"]], "description": (hit.get("description") or "")[:120], "matched_columns": hit.get("matched_columns", []), "status": "imported", }) + else: + workspace_tables = self.workspace.list_tables() + workspace_truncated = len(workspace_tables) > limit + for table in workspace_tables[:limit]: + name = table if isinstance(table, str) else table.get("name", "") + if name: + results.append({ + "type": "table", + "source": "workspace", + "name": name, + "path": [name], + "status": "imported", + }) except Exception: logger.debug("find_data: workspace search failed", exc_info=True) - if source_ids != [] and user_home: + catalog_truncated = False + if user_home: try: + if source_ids is None: + source_ids = [ + source_id for source_id in list_cached_sources(user_home) + if _source_is_discoverable(source_id) + ] + else: + source_ids = [ + source_id for source_id in source_ids + if _source_is_discoverable(source_id) + ] imported_names = {result["name"] for result in results} - cache_hits = search_catalog_cache( + cache_hits, catalog_truncated = find_catalog_cache( user_home, query, source_ids=source_ids, - limit_per_source=min(limit, 50), + limit=limit, exclude_tables=imported_names, - exclude_pattern=exclude, + filter_by=filter_by, fields=fields, - path_prefix=path_prefix, + path_prefix=path, ) - for hit in cache_hits[:limit]: - results.append({ - "source": hit.get("source_id", "connected"), - "source_id": hit.get("source_id", ""), - "table_key": hit.get("table_key", ""), - "name": hit["name"], - "description": (hit.get("description") or "")[:120], - "matched_columns": hit.get("matched_columns", []), - "status": "not imported", - }) + for hit in cache_hits: + hit["source"] = hit.get("source_id", "connected") + if hit["type"] == "table": + hit["status"] = "not imported" + results.append(hit) except CatalogSearchError as exc: return {"error": str(exc)} except Exception: @@ -226,7 +356,11 @@ def find_data(self, args: dict[str, Any]) -> dict[str, Any]: if not results: try: - known = sorted(list_cached_sources(user_home) or []) if user_home else [] + known = sorted( + source_id + for source_id in (list_cached_sources(user_home) or []) + if _source_is_discoverable(source_id) + ) if user_home else [] except Exception: known = [] return { @@ -237,15 +371,20 @@ def find_data(self, args: dict[str, Any]) -> dict[str, Any]: for source_id, snapshot in snapshots.items() }, "note": ( - f"No tables matched query={query!r} scope={scope_raw!r}. " - "Try a broader pattern, alternation (a|b), or list_data to browse." + f"No data matched query={query!r} in the requested scope. " + "Try a broader pattern or use list_data to browse immediate children." ), + "truncated": False, } + truncated = workspace_truncated or catalog_truncated or len(results) > limit return { "results": results[:limit], "query": query, - "scope": scope_raw, + "source_id": source_id or None, + "path": path, + "filter_by": filter_by, + "truncated": truncated, "catalog_freshness": { source_id: _freshness_payload(snapshot) for source_id, snapshot in snapshots.items() @@ -257,6 +396,11 @@ def describe_data(self, args: dict[str, Any]) -> dict[str, Any]: source_id = args.get("source_id", "") table_key = args.get("table_key", "") + user_home = getattr(self.workspace, "user_home", None) + if user_home: + from data_formulator.datalake.connector_preferences import connector_is_enabled + if not connector_is_enabled(user_home, source_id) or not _source_is_discoverable(source_id): + return {"error": f"Source '{source_id}' is disconnected."} return { "result": handle_read_catalog_metadata( source_id, diff --git a/py-src/data_formulator/datalake/azure_blob_workspace_manager.py b/py-src/data_formulator/datalake/azure_blob_workspace_manager.py index cd7801347..19d3b3b5d 100644 --- a/py-src/data_formulator/datalake/azure_blob_workspace_manager.py +++ b/py-src/data_formulator/datalake/azure_blob_workspace_manager.py @@ -26,6 +26,7 @@ WorkspaceManager, SESSION_STATE_FILENAME, WORKSPACE_META_FILENAME, + _session_source_ids, _strip_sensitive, ) @@ -121,6 +122,7 @@ def _upload_meta( *, table_count: Optional[int] = None, chart_count: Optional[int] = None, + source_ids: Optional[list[str]] = None, ) -> None: """Upload a lightweight ``workspace_meta.json`` blob for fast listing. @@ -133,6 +135,7 @@ def _upload_meta( # Preserve createdAt if the meta blob already exists. created_at = now_iso + existing: dict = {} if self._blob_exists(blob_name): try: existing = json.loads(self._download_blob(blob_name)) @@ -152,8 +155,16 @@ def _upload_meta( } if table_count is not None: meta["tableCount"] = table_count + elif existing.get("tableCount") is not None: + meta["tableCount"] = existing["tableCount"] if chart_count is not None: meta["chartCount"] = chart_count + elif existing.get("chartCount") is not None: + meta["chartCount"] = existing["chartCount"] + if source_ids is not None: + meta["sourceIds"] = source_ids + elif isinstance(existing.get("sourceIds"), list): + meta["sourceIds"] = existing["sourceIds"] self._upload_blob(blob_name, json.dumps(meta, ensure_ascii=False)) def _ensure_meta(self, workspace_id: str) -> dict: @@ -209,6 +220,7 @@ def list_workspaces(self) -> list[dict]: "updated_at": meta.get("updatedAt"), "table_count": meta.get("tableCount"), "chart_count": meta.get("chartCount"), + "source_ids": meta.get("sourceIds", []), }) workspaces.sort(key=lambda w: w.get("updated_at") or "", reverse=True) @@ -331,11 +343,19 @@ def save_session_state(self, workspace_id: str, state: dict) -> None: aw = clean_state.get("activeWorkspace") dn = aw["displayName"] if isinstance(aw, dict) and aw.get("displayName") else workspace_id - tables = clean_state.get("tables") + tables = clean_state.get("inputTables") + if not isinstance(tables, list): + tables = clean_state.get("tables") tc = len(tables) if isinstance(tables, list) else None charts = clean_state.get("charts") cc = len(charts) if isinstance(charts, list) else None - self._upload_meta(workspace_id, dn, table_count=tc, chart_count=cc) + self._upload_meta( + workspace_id, + dn, + table_count=tc, + chart_count=cc, + source_ids=_session_source_ids(clean_state), + ) logger.debug(f"Saved session state to blob {blob_name}") diff --git a/py-src/data_formulator/datalake/catalog_cache.py b/py-src/data_formulator/datalake/catalog_cache.py index 68ea9810d..bfddca059 100644 --- a/py-src/data_formulator/datalake/catalog_cache.py +++ b/py-src/data_formulator/datalake/catalog_cache.py @@ -378,188 +378,292 @@ def list_cached_sources(workspace_root: Path | str) -> list[str]: sources = [s for s in sources if s in allowed] except Exception: logger.debug("Failed to filter cached sources by admin set", exc_info=True) + try: + from data_formulator.datalake.connector_preferences import disabled_connector_ids + disabled_sources = disabled_connector_ids(workspace_root) + sources = [source for source in sources if source not in disabled_sources] + except Exception: + logger.debug("Failed to filter disabled cached sources", exc_info=True) return sources -def _search_python( +def find_catalog_cache( workspace_root: Path | str, - needle: str, - all_ids: list[str], - exclude: set[str], - limit_per_source: int, + query: str | None = None, + source_ids: list[str] | None = None, + limit: int = 100, *, - exclude_pattern: re.Pattern | None = None, - fields: set[str] | None = None, + filter_by: str | None = None, + fields: list[str] | None = None, path_prefix: list[str] | None = None, -) -> list[dict[str, Any]]: - """Structured field search over the on-disk catalog cache. + exclude_tables: set[str] | None = None, +) -> tuple[list[dict[str, Any]], bool]: + """Recursively find typed catalog nodes below an exact path. - ``needle`` is always a regex pattern (case-insensitive). Callers who - want literal substring matching should ``re.escape`` first. Invalid - patterns raise :class:`CatalogSearchError`. + ``query`` is an optional case-insensitive regex. Omitting it enumerates all + selected descendants. Results are flat and include exact source paths. """ - match_fields = fields if fields is not None else {"name", "description", "columns"} - - try: - compiled = re.compile(needle, re.IGNORECASE) - except re.error as exc: - raise CatalogSearchError(f"Invalid query regex: {exc}") from exc + node_filter = (filter_by or "").strip().lower() or None + if node_filter not in {None, "folder", "table"}: + raise ValueError("filter_by must be 'folder' or 'table'") - def _matches(text: str) -> bool: - return bool(text) and compiled.search(text) is not None + pattern = None + if query and query.strip(): + try: + pattern = re.compile(query.strip(), re.IGNORECASE) + except re.error as exc: + raise CatalogSearchError(f"Invalid query regex: {exc}") from exc + match_fields = set(fields or ["name", "description", "columns"]) + prefix = [str(segment) for segment in (path_prefix or [])] + excluded_tables = exclude_tables or set() + all_ids = source_ids if source_ids is not None else list_cached_sources(workspace_root) + try: + from data_formulator.datalake.connector_preferences import disabled_connector_ids + disabled_sources = disabled_connector_ids(workspace_root) + all_ids = [source_id for source_id in all_ids if source_id not in disabled_sources] + except Exception: + logger.debug("Failed to filter disabled catalog finder sources", exc_info=True) + cap = max(1, min(int(limit or 100), 500)) results: list[dict[str, Any]] = [] - plen = len(path_prefix) if path_prefix else 0 - prefix = list(path_prefix or []) - for sid in all_ids: - raw = _load_catalog_raw(workspace_root, sid) + for source_id in all_ids: + raw = _load_catalog_raw(workspace_root, source_id) if not raw: continue + original_source_id = raw.get("source_id", source_id) + tables = raw.get("tables", []) or [] + normalized_tables: list[tuple[dict[str, Any], list[str]]] = [] + folder_stats: dict[tuple[str, ...], dict[str, Any]] = {} + + for table in tables: + table_name = str(table.get("name", "")) + raw_path = table.get("path") + table_path = [str(segment) for segment in raw_path] if isinstance(raw_path, list) else [] + if not table_path and table_name: + table_path = [table_name] + normalized_tables.append((table, table_path)) + + for depth in range(1, len(table_path)): + folder_path = tuple(table_path[:depth]) + stats = folder_stats.setdefault( + folder_path, + {"children": set(), "descendant_table_count": 0}, + ) + child_type = "folder" if depth < len(table_path) - 1 else "table" + stats["children"].add((child_type, table_path[depth])) + stats["descendant_table_count"] += 1 + + if node_filter != "table": + for folder_path, stats in folder_stats.items(): + if len(folder_path) <= len(prefix) or list(folder_path[:len(prefix)]) != prefix: + continue + name = folder_path[-1] + if pattern is not None and pattern.search(name) is None: + continue + results.append({ + "type": "folder", + "source_id": original_source_id, + "name": name, + "path": list(folder_path), + "child_count": len(stats["children"]), + "descendant_table_count": stats["descendant_table_count"], + "score": 10 if pattern is not None else 0, + "match_reasons": ["folder_name"] if pattern is not None else [], + }) - original_source_id = raw.get("source_id", sid) - tables = raw.get("tables", []) + if node_filter == "folder": + continue - source_hits: list[dict[str, Any]] = [] - for t in tables: - tname = t.get("name", "") - if tname in exclude: + for table, table_path in normalized_tables: + if len(table_path) <= len(prefix) or table_path[:len(prefix)] != prefix: continue - # Path-prefix filter - if plen: - tpath = t.get("path") or [] - if not isinstance(tpath, list) or len(tpath) < plen: - continue - if [str(s) for s in tpath[:plen]] != prefix: - continue - - # Exclude pattern (regex on name) - if exclude_pattern is not None and exclude_pattern.search(tname): + table_name = str(table.get("name", "")) + leaf_name = table_path[-1] + if table_name in excluded_tables: continue + metadata = table.get("metadata") or {} + description = str(metadata.get("description", "")) score = 0 - matched_cols: list[str] = [] + matched_columns: list[str] = [] match_reasons: list[str] = [] - meta = t.get("metadata") or {} - table_key = t.get("table_key", "") - - if "name" in match_fields and _matches(tname): - score += 10 - match_reasons.append("table_name") - - # Source description - src_desc = meta.get("description", "") - if "description" in match_fields and src_desc and _matches(src_desc): - score += 5 - match_reasons.append("source_description") - - # Source columns - if "columns" in match_fields: - for col in meta.get("columns", []): - cname = col.get("name", "") - if cname and _matches(cname): - matched_cols.append(cname) - score += 2 - if "column_name" not in match_reasons: - match_reasons.append("column_name") - cdesc = col.get("description", "") - if cdesc and _matches(cdesc): - matched_cols.append(cname) - score += 1 - if "source_column_description" not in match_reasons: - match_reasons.append("source_column_description") - - if score > 0: - source_hits.append({ - "source_id": original_source_id, - "table_key": table_key, - "name": tname, - "description": src_desc, - "matched_columns": list(dict.fromkeys(matched_cols)), - "score": score, - "match_reasons": match_reasons, - "metadata_status": meta.get("source_metadata_status", ""), - }) - - source_hits.sort(key=lambda r: -r["score"]) - results.extend(source_hits[:limit_per_source]) + if pattern is not None: + if "name" in match_fields and ( + pattern.search(leaf_name) or pattern.search(table_name) + ): + score += 10 + match_reasons.append("table_name") + if "description" in match_fields and pattern.search(description): + score += 5 + match_reasons.append("source_description") + if "columns" in match_fields: + for column in metadata.get("columns", []): + column_name = str(column.get("name", "")) + column_description = str(column.get("description", "")) + if pattern.search(column_name): + score += 2 + matched_columns.append(column_name) + if "column_name" not in match_reasons: + match_reasons.append("column_name") + if pattern.search(column_description): + score += 1 + matched_columns.append(column_name) + if "source_column_description" not in match_reasons: + match_reasons.append("source_column_description") + if score == 0: + continue - results.sort(key=lambda r: -r["score"]) - return results + results.append({ + "type": "table", + "source_id": original_source_id, + "name": leaf_name, + "path": table_path, + "table_key": table.get("table_key", "") or "", + "description": description[:120], + "matched_columns": list(dict.fromkeys(matched_columns)), + "score": score, + "match_reasons": match_reasons, + "metadata_status": metadata.get("source_metadata_status", ""), + }) + + results.sort(key=lambda item: ( + -item["score"], + item["source_id"].casefold(), + 0 if item["type"] == "folder" else 1, + [segment.casefold() for segment in item["path"]], + item["path"], + )) + return results[:cap], len(results) > cap -def search_catalog_cache( - workspace_root: Path | str, - query: str, - source_ids: list[str] | None = None, - limit_per_source: int = 20, - exclude_tables: set[str] | None = None, - *, - exclude_pattern: str | None = None, - fields: list[str] | None = None, - path_prefix: list[str] | None = None, -) -> list[dict[str, Any]]: - """Search across cached catalogs for tables matching a regex pattern. +# --------------------------------------------------------------------------- +# Hierarchy navigation (used by the data loading agent's list_data tool) +# --------------------------------------------------------------------------- - ``query`` is treated as a case-insensitive regex. Callers passing - user-typed keywords should ``re.escape`` the input first. Invalid - patterns raise :class:`CatalogSearchError`. +# Directory listings default to 100 immediate children and allow callers to +# request at most 500. +LIST_DATA_DEFAULT_LIMIT = 100 +LIST_DATA_MAX_LIMIT = 500 - Returns a flat list of match dicts with fields: - ``source_id``, ``table_key``, ``name``, ``description``, - ``matched_columns``, ``score``, ``match_reasons``, ``metadata_status``. +# Compact orientation only; agents inspect a source before describing its data. +SOURCE_TOP_LEVEL_PREVIEW = 12 +SUMMARY_TOP_LEVEL_LIMIT = 5 +SUMMARY_TABLE_LIMIT = 5 - ``exclude_pattern``, ``fields``, and ``path_prefix`` further constrain - the search. - """ - needle_raw = (query or "").strip() - if not needle_raw: - return [] - exclude = exclude_tables or set() - all_ids = source_ids or list_cached_sources(workspace_root) +def summarize_catalog_sources( + workspace_root: Path | str, + top_level_limit: int = SUMMARY_TOP_LEVEL_LIMIT, + table_limit: int = SUMMARY_TABLE_LIMIT, +) -> list[dict[str, Any]]: + """Return bounded, branch-diverse impressions of cached sources.""" + summaries: list[dict[str, Any]] = [] + for source_id in list_cached_sources(workspace_root): + raw = _load_catalog_raw(workspace_root, source_id) + if not raw: + continue - # Compile exclude pattern up-front so a bad pattern surfaces clearly. - excl_re = None - if exclude_pattern: - try: - excl_re = re.compile(exclude_pattern, re.IGNORECASE) - except re.error as exc: - raise CatalogSearchError(f"Invalid exclude regex: {exc}") from exc - - fields_set = set(fields) if fields else None - - return _search_python( - workspace_root, - needle_raw, - all_ids, - exclude, - limit_per_source, - exclude_pattern=excl_re, - fields=fields_set, - path_prefix=list(path_prefix or []), - ) + original_source_id = raw.get("source_id", source_id) + tables = raw.get("tables", []) or [] + folder_paths: set[tuple[str, ...]] = set() + top_folders: dict[str, int] = {} + root_tables: list[dict[str, Any]] = [] + tables_by_branch: dict[str, list[dict[str, Any]]] = {} + max_depth = 0 + + for table in tables: + name = str(table.get("name", "")) + raw_path = table.get("path") + path = [str(segment) for segment in raw_path] if isinstance(raw_path, list) else [] + if not path and name: + path = [name] + if not path: + continue + max_depth = max(max_depth, len(path) - 1) + for depth in range(1, len(path)): + folder_paths.add(tuple(path[:depth])) -# --------------------------------------------------------------------------- -# Hierarchy navigation (used by the data loading agent's list_data tool) -# --------------------------------------------------------------------------- + item = { + "type": "table", + "name": path[-1], + "path": path, + "table_key": table.get("table_key", "") or "", + } + description = str((table.get("metadata") or {}).get("description", "")) + if description: + item["description"] = description[:80] + + if len(path) == 1: + root_tables.append(item) + branch = "" + else: + branch = path[0] + top_folders[branch] = top_folders.get(branch, 0) + 1 + tables_by_branch.setdefault(branch, []).append(item) + + top_level: list[dict[str, Any]] = [ + { + "type": "folder", + "name": name, + "path": [name], + "descendant_table_count": count, + } + for name, count in sorted( + top_folders.items(), key=lambda entry: (-entry[1], entry[0].casefold(), entry[0]) + ) + ] + root_tables.sort(key=lambda item: (item["name"].casefold(), item["name"])) + top_level.extend(root_tables) + + for branch_tables in tables_by_branch.values(): + branch_tables.sort(key=lambda item: ( + [segment.casefold() for segment in item["path"]], item["path"] + )) + sample_tables: list[dict[str, Any]] = [] + branch_names = sorted(tables_by_branch, key=lambda name: (name.casefold(), name)) + sample_index = 0 + while len(sample_tables) < table_limit: + added = False + for branch in branch_names: + branch_tables = tables_by_branch[branch] + if sample_index < len(branch_tables): + sample_tables.append(branch_tables[sample_index]) + added = True + if len(sample_tables) == table_limit: + break + if not added: + break + sample_index += 1 -# Hard cap on entries returned in one list_path_children response. See -# design-docs/32-data-loading-agent-navigation.md §5. Truncation pushes the -# agent toward find_data or a tighter filter rather than pagination. -LIST_DATA_LIMIT = 200 + summaries.append({ + "source_id": original_source_id, + "table_count": len(tables), + "folder_count": len(folder_paths), + "max_depth": max_depth, + "top_level": top_level[:top_level_limit], + "sample_tables": sample_tables, + "omitted": { + "top_level": max(0, len(top_level) - top_level_limit), + "tables": max(0, len(tables) - len(sample_tables)), + }, + }) + summaries.sort(key=lambda summary: summary["source_id"]) + return summaries def list_sources_summary( workspace_root: Path | str, ) -> list[dict[str, Any]]: """Return a per-source summary suitable for ``list_data()`` with no args. - Each entry: ``{source_id, table_count, is_hierarchical}``. Sources whose - cache file is missing or unreadable are skipped silently — the agent - treats the cache as ground truth (see design-docs §8). + Each entry includes a bounded ``top_level`` preview and an explicit + ``top_level_truncated`` signal. The preview is orientation, not a substitute + for listing or finding data within the source. + Sources whose cache file is missing or unreadable are skipped silently — the + agent treats the cache as ground truth (see design-docs §8). """ out: list[dict[str, Any]] = [] for sid in list_cached_sources(workspace_root): @@ -568,15 +672,28 @@ def list_sources_summary( continue tables = raw.get("tables", []) or [] is_hier = False + folders: list[str] = [] + seen_folders: set[str] = set() + leaves: list[str] = [] for t in tables: p = t.get("path") - if isinstance(p, list) and len(p) >= 2: + p = [str(s) for s in p] if isinstance(p, list) else [] + if len(p) >= 2: is_hier = True - break + if p[0] not in seen_folders: + seen_folders.add(p[0]) + folders.append(p[0]) + else: + leaf = p[0] if p else str(t.get("name", "")) + if leaf: + leaves.append(leaf) + top_level = folders + leaves out.append({ "source_id": raw.get("source_id", sid), "table_count": len(tables), "is_hierarchical": is_hier, + "top_level": top_level[:SOURCE_TOP_LEVEL_PREVIEW], + "top_level_truncated": len(top_level) > SOURCE_TOP_LEVEL_PREVIEW, }) out.sort(key=lambda r: r["source_id"]) return out @@ -586,8 +703,9 @@ def list_path_children( workspace_root: Path | str, source_id: str, path: list[str] | None = None, - filter: str | None = None, - limit: int = LIST_DATA_LIMIT, + filter_by: str | None = None, + limit: int = LIST_DATA_DEFAULT_LIMIT, + start_after: dict[str, Any] | None = None, ) -> dict[str, Any]: """List direct children at a hierarchy level within a source's catalog. @@ -601,35 +719,31 @@ def list_path_children( equal the input path. At depth 0 we additionally surface records with empty path, using their ``name`` as the leaf. - ``filter`` is a case-insensitive substring match on the immediate child - segment / table name (the *next* segment after the prefix), equivalent to - ``ls /**``. Not a regex — keep this primitive cheap. - - Returns ``{source_id, path, folders, tables, total_folders, total_tables, - truncated, hint?}``. Combined ``folders + tables`` are capped at ``limit`` - (folders take precedence to preserve drill-down). + ``filter_by`` may be ``folder`` or ``table``. Results use deterministic + folder-first ordering and ``start_after`` is an exclusive node reference. """ path = [str(p) for p in (path or [])] K = len(path) - cap = max(1, min(int(limit or LIST_DATA_LIMIT), LIST_DATA_LIMIT)) - filt = (filter or "").strip().lower() or None + cap = max(1, min(int(limit or LIST_DATA_DEFAULT_LIMIT), LIST_DATA_MAX_LIMIT)) + node_filter = (filter_by or "").strip().lower() or None + if node_filter not in {None, "folder", "table"}: + raise ValueError("filter_by must be 'folder' or 'table'") raw = _load_catalog_raw(workspace_root, source_id) if not raw: return { "source_id": source_id, "path": path, - "folders": [], - "tables": [], - "total_folders": 0, - "total_tables": 0, + "items": [], + "total_count": 0, "truncated": False, } original_sid = raw.get("source_id", source_id) tables_raw = raw.get("tables", []) or [] - folder_counts: dict[str, int] = {} + folder_table_counts: dict[str, int] = {} + folder_child_names: dict[str, set[tuple[str, str]]] = {} leaf_tables: list[dict[str, Any]] = [] for t in tables_raw: @@ -649,9 +763,10 @@ def list_path_children( # Folder: at least one more segment after the prefix beyond the leaf. if plen >= K + 2: seg = tpath[K] - if filt and filt not in seg.lower(): - continue - folder_counts[seg] = folder_counts.get(seg, 0) + 1 + folder_table_counts[seg] = folder_table_counts.get(seg, 0) + 1 + child_type = "folder" if plen >= K + 3 else "table" + child_name = tpath[K + 1] + folder_child_names.setdefault(seg, set()).add((child_type, child_name)) continue # Table at this level. @@ -663,53 +778,62 @@ def list_path_children( else: continue - if filt and filt not in leaf.lower(): - continue - - meta = t.get("metadata") or {} - desc = (meta.get("description") or "")[:120] leaf_tables.append({ + "type": "table", "name": leaf, + "path": [*path, leaf], "table_key": t.get("table_key", "") or "", - "description": desc, }) - # Sort folders by table_count desc then name; tables by name. folders = [ - {"name": name, "table_count": cnt} - for name, cnt in sorted( - folder_counts.items(), key=lambda kv: (-kv[1], kv[0]) - ) + { + "type": "folder", + "name": name, + "path": [*path, name], + "child_count": len(folder_child_names[name]), + "descendant_table_count": table_count, + } + for name, table_count in folder_table_counts.items() ] - leaf_tables.sort(key=lambda r: r["name"]) - - total_folders = len(folders) - total_tables = len(leaf_tables) - total = total_folders + total_tables - truncated = total > cap + folders.sort(key=lambda item: (item["name"].casefold(), item["name"])) + leaf_tables.sort(key=lambda item: (item["name"].casefold(), item["name"])) + items = ( + folders if node_filter == "folder" + else leaf_tables if node_filter == "table" + else folders + leaf_tables + ) + total_count = len(items) - # Combined cap: folders first (drill-down has higher value), then tables. - if total_folders >= cap: - folders = folders[:cap] - leaf_tables = [] - else: - leaf_tables = leaf_tables[: cap - total_folders] + if start_after is not None: + try: + start_index = next( + index for index, item in enumerate(items) + if item["type"] == start_after.get("type") + and item["path"] == start_after.get("path") + and ( + item["type"] == "folder" + or item["table_key"] == start_after.get("table_key") + ) + ) + except (AttributeError, StopIteration) as exc: + raise ValueError("start_after does not identify an immediate child") from exc + items = items[start_index + 1:] + + page_items = items[:cap] + truncated = len(items) > len(page_items) result: dict[str, Any] = { "source_id": original_sid, "path": path, - "folders": folders, - "tables": leaf_tables, - "total_folders": total_folders, - "total_tables": total_tables, + "items": page_items, + "total_count": total_count, "truncated": truncated, } if truncated: - remaining = total - len(folders) - len(leaf_tables) - result["hint"] = ( - f"{remaining} more entries not shown. Use list_path_children(filter=...) " - f"to narrow, or find_data(query=..., scope='{original_sid}" - + (":" + "/".join(path) if path else "") - + "') to search this subtree." - ) + last_item = page_items[-1] + result["next_start_after"] = { + key: last_item[key] + for key in ("type", "path", "table_key") + if key in last_item + } return result diff --git a/py-src/data_formulator/datalake/connector_preferences.py b/py-src/data_formulator/datalake/connector_preferences.py new file mode 100644 index 000000000..db4eca6ba --- /dev/null +++ b/py-src/data_formulator/datalake/connector_preferences.py @@ -0,0 +1,60 @@ +"""Per-user connector availability preferences.""" + +from __future__ import annotations + +import json +import logging +import os +from pathlib import Path +from threading import Lock +from uuid import uuid4 + +from data_formulator.security.path_safety import ConfinedDir + +logger = logging.getLogger(__name__) + +_PREFERENCES_FILE = "connector_preferences.json" +_PREFERENCES_LOCK = Lock() + + +def disabled_connector_ids(user_home: Path | str) -> set[str]: + jail = ConfinedDir(user_home, mkdir=False) + if not jail.exists(_PREFERENCES_FILE): + return set() + try: + raw = json.loads(jail.read_text(_PREFERENCES_FILE)) + values = raw.get("disabled_connector_ids", []) if isinstance(raw, dict) else [] + return {value for value in values if isinstance(value, str) and value} + except Exception: + logger.warning("Failed to read connector preferences", exc_info=True) + return set() + + +def connector_is_enabled(user_home: Path | str, source_id: str) -> bool: + return source_id not in disabled_connector_ids(user_home) + + +def set_connector_enabled( + user_home: Path | str, + source_id: str, + enabled: bool, +) -> None: + jail = ConfinedDir(user_home, mkdir=True) + with _PREFERENCES_LOCK: + disabled = disabled_connector_ids(user_home) + if enabled: + disabled.discard(source_id) + else: + disabled.add(source_id) + + target = jail.resolve(_PREFERENCES_FILE) + temporary = jail.resolve(f".{_PREFERENCES_FILE}.{os.getpid()}.{uuid4().hex}.tmp") + try: + with open(temporary, "w", encoding="utf-8") as file: + json.dump({"disabled_connector_ids": sorted(disabled)}, file) + file.flush() + os.fsync(file.fileno()) + os.replace(temporary, target) + finally: + if temporary.exists(): + temporary.unlink() \ No newline at end of file diff --git a/py-src/data_formulator/datalake/workspace_manager.py b/py-src/data_formulator/datalake/workspace_manager.py index c5af79d12..99f2dfdbc 100644 --- a/py-src/data_formulator/datalake/workspace_manager.py +++ b/py-src/data_formulator/datalake/workspace_manager.py @@ -46,6 +46,49 @@ def _strip_sensitive(state: dict) -> dict: return {k: v for k, v in state.items() if k not in _SENSITIVE_FIELDS} +def _session_source_ids(state: dict) -> list[str]: + """Summarize input-table origins for lightweight session grouping.""" + tables = state.get("inputTables") + if not isinstance(tables, list): + tables = state.get("tables") + if not isinstance(tables, list): + return [] + + source_ids: set[str] = set() + for table in tables: + if not isinstance(table, dict): + continue + source = table.get("source") + source_config = table.get("sourceConfig") + + if isinstance(source, dict) and source.get("kind") == "connector": + connector_id = source.get("connectorId") or source.get("connector_id") + if isinstance(connector_id, str) and connector_id: + source_ids.add(connector_id) + continue + + config = source_config if isinstance(source_config, dict) else source + if not isinstance(config, dict): + continue + connector_id = ( + config.get("connectorId") + or config.get("connector_id") + or config.get("sourceId") + or config.get("source_id") + ) + if isinstance(connector_id, str) and connector_id: + source_ids.add(connector_id) + continue + + source_type = config.get("type") + if source_type == "example": + source_ids.add("sample_datasets") + elif source_type in {"file", "paste", "url", "stream", "extract"}: + source_ids.add("upload") + + return sorted(source_ids) + + class WorkspaceManager: """ Manages the set of workspaces for a single user. @@ -87,6 +130,7 @@ def _write_meta( *, table_count: Optional[int] = None, chart_count: Optional[int] = None, + source_ids: Optional[list[str]] = None, provisional: Optional[bool] = None, ) -> None: """Write a lightweight ``workspace_meta.json`` used by list_workspaces. @@ -101,6 +145,7 @@ def _write_meta( # Preserve createdAt if the meta file already exists. created_at = now_iso + existing: dict = {} if meta_file.exists(): try: existing = json.loads(meta_file.read_text(encoding="utf-8")) @@ -121,8 +166,16 @@ def _write_meta( } if table_count is not None: meta["tableCount"] = table_count + elif existing.get("tableCount") is not None: + meta["tableCount"] = existing["tableCount"] if chart_count is not None: meta["chartCount"] = chart_count + elif existing.get("chartCount") is not None: + meta["chartCount"] = existing["chartCount"] + if source_ids is not None: + meta["sourceIds"] = source_ids + elif isinstance(existing.get("sourceIds"), list): + meta["sourceIds"] = existing["sourceIds"] if provisional: meta["provisional"] = True meta_file.write_text( @@ -217,6 +270,7 @@ def list_workspaces(self) -> list[dict]: "updated_at": meta.get("updatedAt"), "table_count": tc, "chart_count": cc, + "source_ids": meta.get("sourceIds", []), }) workspaces.sort(key=lambda w: w.get("updated_at") or "", reverse=True) @@ -497,12 +551,20 @@ def save_session_state(self, workspace_id: str, state: dict) -> None: aw = clean_state.get("activeWorkspace") dn = aw["displayName"] if isinstance(aw, dict) and aw.get("displayName") else workspace_id - tables = clean_state.get("tables") + tables = clean_state.get("inputTables") + if not isinstance(tables, list): + tables = clean_state.get("tables") tc = len(tables) if isinstance(tables, list) else None charts = clean_state.get("charts") cc = len(charts) if isinstance(charts, list) else None # Saving state is the moment a session stops being provisional. - self._write_meta(workspace_id, dn, table_count=tc, chart_count=cc) + self._write_meta( + workspace_id, + dn, + table_count=tc, + chart_count=cc, + source_ids=_session_source_ids(clean_state), + ) logger.debug(f"Saved session state to {state_file}") diff --git a/py-src/data_formulator/routes/agents.py b/py-src/data_formulator/routes/agents.py index 23b49aafd..eec77fc0e 100644 --- a/py-src/data_formulator/routes/agents.py +++ b/py-src/data_formulator/routes/agents.py @@ -135,6 +135,8 @@ def preview_data_operation(): "source_id": step.source_id, **({"table_description": str(table_description).strip()} if table_description else {}), "error": str(exc), + "columns": [], + "rows": [], }) continue previews.append({ diff --git a/py-src/data_formulator/routes/sessions.py b/py-src/data_formulator/routes/sessions.py index 626afd4e0..81efcabab 100644 --- a/py-src/data_formulator/routes/sessions.py +++ b/py-src/data_formulator/routes/sessions.py @@ -126,6 +126,7 @@ def list_sessions(): entry["table_count"] = w["table_count"] if w.get("chart_count") is not None: entry["chart_count"] = w["chart_count"] + entry["source_ids"] = w.get("source_ids", []) sessions.append(entry) return json_ok({"sessions": sessions}) diff --git a/pyproject.toml b/pyproject.toml index 457f80706..44daf9bca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,7 @@ dependencies = [ "databricks-sql-connector", # databricks # SSO / Auth deps "PyJWT[crypto]>=2.8.0", # OIDC JWT verification (includes cryptography) + "cryptography>=50.0.0", # Security floor for auth and local vault encryption "requests", # GitHub OAuth code exchange, Superset API calls "flask-session>=0.8.0", # Server-side session (SQLite) for TokenStore ] diff --git a/src/app/App.tsx b/src/app/App.tsx index 881fde203..4c04c76f6 100644 --- a/src/app/App.tsx +++ b/src/app/App.tsx @@ -1129,7 +1129,7 @@ const AppShell: FC = () => { - + {isCompactToolbar ? ( @@ -1137,15 +1137,16 @@ const AppShell: FC = () => { <> diff --git a/src/app/dfSlice.tsx b/src/app/dfSlice.tsx index 5c919acc9..c77f380f5 100644 --- a/src/app/dfSlice.tsx +++ b/src/app/dfSlice.tsx @@ -10,13 +10,13 @@ import { getChartTemplate, getChartChannels } from "../components/ChartTemplates import { vlAdaptChart, vlRecommendEncodings } from 'flint-chart'; import { migrateState } from './stateMigrations'; import { getDataTable } from '../views/ChartUtils'; -import { getTriggers, getUrls, computeContentHash } from './utils'; +import { getUrls, computeContentHash } from './utils'; import { apiRequest, ApiRequestError } from './apiClient'; import { deleteTablesFromWorkspace } from './workspaceService'; import i18n from '../i18n'; import { Type } from '../data/types'; -import { createTableFromFromObjectArray, inferTypeFromValueArray, refineTemporalType } from '../data/utils'; -import { Identity, IdentityType, getBrowserId } from './identity'; +import { inferTypeFromValueArray, refineTemporalType } from '../data/utils'; +import { Identity, getBrowserId } from './identity'; import { REHYDRATE } from 'redux-persist'; import { setInputTablePreview } from './inputTablePreviewCache'; import { materializeInputTablePreview, materializeTables } from './tableResolution'; @@ -89,6 +89,7 @@ export interface ServerConfig { icon: string; params_form: Array<{name: string; type: string; required: boolean; default?: string; options?: string[]; advanced?: boolean; description?: string; sensitive?: boolean; tier?: 'connection' | 'auth' | 'filter'}>; pinned_params: Record; + connection_identity?: string; hierarchy: Array<{key: string; label: string}>; effective_hierarchy: Array<{key: string; label: string}>; auth_instructions: string; @@ -2345,8 +2346,17 @@ export const dataFormulatorSlice = createSlice({ }; } - const displayName = data["result"][0]["suggested_table_name"] as string | undefined; - const info = { tableId, ...(displayName ? { displayName } : {}), fields }; + const suggestedName = data["result"][0]["suggested_table_name"] as string | undefined; + const normalizeName = (name: string) => name.toLowerCase().replace(/[\s_-]+/g, ''); + if (suggestedName && normalizeName(table.displayId || table.id) === normalizeName(table.id)) { + state.inputTables = state.inputTables.map(item => + item.id === tableId ? { ...item, displayId: suggestedName } : item + ); + state.derivedTables = state.derivedTables.map(item => + item.id === tableId ? { ...item, displayId: suggestedName } : item + ); + } + const info = { tableId, fields }; const existingIndex = state.tableSemantics.findIndex(item => item.tableId === tableId); if (existingIndex >= 0) state.tableSemantics[existingIndex] = info; else state.tableSemantics.push(info); @@ -2681,7 +2691,7 @@ export const dfSelectors = { seen.add(cur.id); const p: string | undefined = cur.parentNodeId; if (!p) break; - const parentTurn = textTurns.find(tt => tt.id === p); + const parentTurn: TextTurn | undefined = textTurns.find(tt => tt.id === p); if (parentTurn?.dataOperation || parentTurn?.form) { return { type: 'text', textId: parentTurn.id }; } diff --git a/src/app/stateMigrations.ts b/src/app/stateMigrations.ts index 7c4134340..618226b6b 100644 --- a/src/app/stateMigrations.ts +++ b/src/app/stateMigrations.ts @@ -26,7 +26,7 @@ */ /** Current persisted-state schema version. Bump when adding a migration. */ -export const DF_STATE_VERSION = 4; +export const DF_STATE_VERSION = 6; type SavedState = Record; @@ -312,6 +312,43 @@ const MIGRATIONS: Migration[] = [ }; }, }, + { + // Table labels have one owner: `displayId`. Older states also stored an + // inferred table label on `tableSemantics`; preserve that suggestion + // only when the table still has its default label, then remove it from + // the field-semantics collection. + to: 6, + migrate: (s) => { + const semantics = Array.isArray(s.tableSemantics) ? s.tableSemantics : []; + const suggestedNames = new Map(); + const tableSemantics = semantics.map(({ displayName, ...info }: any) => { + if (info?.tableId && typeof displayName === 'string' && displayName.trim()) { + suggestedNames.set(info.tableId, displayName.trim()); + } + return info; + }); + const normalizeName = (name: string) => name.toLowerCase().replace(/[\s_-]+/g, ''); + const migrateTableName = (table: any) => { + if (!table?.id) return table; + const suggestion = suggestedNames.get(table.id); + const currentName = table.displayId || table.id; + return suggestion && normalizeName(currentName) === normalizeName(table.id) + ? { ...table, displayId: suggestion } + : table; + }; + return { + ...s, + inputTables: Array.isArray(s.inputTables) + ? s.inputTables.map(migrateTableName) + : s.inputTables, + derivedTables: Array.isArray(s.derivedTables) + ? s.derivedTables.map(migrateTableName) + : s.derivedTables, + tableSemantics, + __stateVersion: 6, + }; + }, + }, ]; /** diff --git a/src/app/useAutoSave.tsx b/src/app/useAutoSave.tsx index 0289bcb86..5142fa7c8 100644 --- a/src/app/useAutoSave.tsx +++ b/src/app/useAutoSave.tsx @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { useEffect, useRef } from 'react'; +import { useCallback, useEffect, useRef } from 'react'; import { useSelector } from 'react-redux'; import { DataFormulatorState, dfSelectors } from './dfSlice'; import { saveWorkspaceState } from './workspaceService'; @@ -65,6 +65,35 @@ export function useAutoSave() { const isSavingRef = useRef(false); const pendingRef = useRef(false); const lastErrorNotifyRef = useRef(0); + const latestStateRef = useRef(state); + latestStateRef.current = state; + + const saveLatestState = useCallback(async () => { + if (isSavingRef.current) { + pendingRef.current = true; + return; + } + + isSavingRef.current = true; + try { + do { + pendingRef.current = false; + try { + await saveWorkspaceState(getSerializableState(latestStateRef.current)); + } catch (err) { + const now = Date.now(); + if (now - lastErrorNotifyRef.current >= AUTO_SAVE_ERROR_NOTIFY_MS) { + lastErrorNotifyRef.current = now; + handleApiError(err, 'Auto-save'); + } else { + console.warn('[auto-save] failed:', err); + } + } + } while (pendingRef.current); + } finally { + isSavingRef.current = false; + } + }, []); useEffect(() => { // Nothing to save while a session is loading, read-only, workspace-less, @@ -79,36 +108,8 @@ export function useAutoSave() { clearTimeout(timerRef.current); } - timerRef.current = setTimeout(async () => { - // Skip if a save is already in flight - if (isSavingRef.current) { - pendingRef.current = true; - return; - } - - isSavingRef.current = true; - try { - const serializable = getSerializableState(state); - await saveWorkspaceState(serializable); - } catch (err) { - const now = Date.now(); - if (now - lastErrorNotifyRef.current >= AUTO_SAVE_ERROR_NOTIFY_MS) { - lastErrorNotifyRef.current = now; - handleApiError(err, 'Auto-save'); - } else { - console.warn('[auto-save] failed:', err); - } - } finally { - isSavingRef.current = false; - // If state changed while we were saving, trigger another save - if (pendingRef.current) { - pendingRef.current = false; - // Re-trigger by scheduling another timeout - timerRef.current = setTimeout(() => { - // This will be picked up by the next effect cycle - }, AUTO_SAVE_DEBOUNCE_MS); - } - } + timerRef.current = setTimeout(() => { + void saveLatestState(); }, AUTO_SAVE_DEBOUNCE_MS); return () => { @@ -116,5 +117,5 @@ export function useAutoSave() { clearTimeout(timerRef.current); } }; - }, [state]); + }, [saveLatestState, state]); } diff --git a/src/app/workspaceService.ts b/src/app/workspaceService.ts index d4c9fd4d4..5e4c0d2e3 100644 --- a/src/app/workspaceService.ts +++ b/src/app/workspaceService.ts @@ -23,6 +23,7 @@ export interface WorkspaceSummary { saved_at: string | null; table_count?: number | null; chart_count?: number | null; + source_ids?: string[]; read_only?: boolean; } diff --git a/src/components/ComponentType.tsx b/src/components/ComponentType.tsx index 598c5dac4..c520fb9dc 100644 --- a/src/components/ComponentType.tsx +++ b/src/components/ComponentType.tsx @@ -370,7 +370,6 @@ export interface FieldSemanticsInfo { export interface TableSemanticsInfo { tableId: string; - displayName?: string; fields: Record; } @@ -677,6 +676,8 @@ export interface ConnectorInstance { deletable?: boolean; params_form: Array<{name: string; type: string; required: boolean; default?: string | number | boolean; options?: string[]; advanced?: boolean; description?: string; sensitive?: boolean; tier?: 'connection' | 'auth' | 'filter'}>; pinned_params: Record; + /** Which instance this connector points at (cluster, host, bucket…), resolved by the loader. */ + connection_identity?: string; hierarchy: Array<{key: string; label: string}>; effective_hierarchy: Array<{key: string; label: string}>; auth_mode?: string; diff --git a/src/i18n/locales/en/common.json b/src/i18n/locales/en/common.json index 457334ffb..24ad9f50b 100644 --- a/src/i18n/locales/en/common.json +++ b/src/i18n/locales/en/common.json @@ -52,6 +52,7 @@ "title": "Backend Log", "viewLogs": "View backend log", "refresh": "Refresh", + "searchSavedState": "Search saved state (Cmd/Ctrl+F)", "download": "Download full log", "empty": "Log file is empty." }, @@ -462,6 +463,9 @@ "rulesLoaded": "Reading rules: {{rules}}", "knowledgeLoaded": "Reading knowledge: {{knowledge}}", "searching": "searching...", + "listingConnectors": "Checking available connectors", + "readingConnector": "Reading connector setup", + "usingTool": "Using {{tool}}", "producingAction": "outputting {{action}}...", "jumpToThreadRange": "Jump to thread(s) {{label}}", "collapse": "collapse", @@ -562,6 +566,7 @@ "agentWorking": "Agent is working...", "attachUploadFailed": "Failed to attach {{name}}", "replyPlaceholder": "Reply to agent's question...", + "emptyAnalysisInputsPlaceholder": "Press Tab to ask what data are available to load", "explorePlaceholder": "Ask questions or describe what to explore (add context with @)", "explorePlaceholderSingleTable": "Ask questions or describe what to explore", "addMoreData": "Add more data to the workspace", @@ -617,6 +622,7 @@ "delegateToReportGen": "Generate report", "errorDuringExploration": "Error during exploration", "explorationStep": "Exploration step {{step}}: {{question}}", + "emptyAnalysisInputsPrompt": "What data is available to load?", "threadExplorePrompt": "Explore interesting patterns and trends in this data", "explorationThreadDeriveDescription": "Derive from {{source}} with instruction: {{instruction}}", "explorationStepCodeComment": "# Exploration step {{step}}", @@ -851,9 +857,10 @@ "refresh": "Refresh data", "emptyTree": "No tables found", "addConnector": "Add data connector", - "configureConnector": "Edit connection", + "connectConnector": "Connect", "linkLocalFolder": "Link local folder", "newSession": "New session", + "importSession": "Import session", "noSessions": "No saved sessions", "tableCount": "{{count}} table(s)", "chartCount": "{{count}} chart(s)", @@ -879,7 +886,9 @@ "loadingEllipsis": "Loading...", "loadWithFilters": "Load with Filters", "load": "Load", - "disconnectConnector": "Disconnect connector", + "disconnectConnector": "Disconnect", + "connectorConnected": "Connected to \"{{name}}\"", + "failedConnectConnector": "Failed to connect", "connectorDisconnected": "Connector \"{{name}}\" disconnected", "failedDisconnectConnector": "Failed to disconnect connector", "failedSearchConnector": "Failed to search {{connector}}", @@ -921,6 +930,15 @@ "sortRecentlyModifiedFirst": "recently modified", "sortNameAsc": "name (a–z)", "sortSessions": "Sort sessions", + "organizeSessions": "Group and sort sessions", + "groupSessions": "Group", + "groupBySource": "Data source", + "groupSourceShort": "Source", + "noGrouping": "No grouping", + "sourceUpload": "Upload", + "sourceExampleDatasets": "Example datasets", + "sourceNoData": "No data", + "sourceOther": "Other", "runCatalogSearch": "Search", "clearCatalogSearch": "Clear search", "timeJustNow": "just now", diff --git a/src/i18n/locales/en/dataLoading.json b/src/i18n/locales/en/dataLoading.json index 8f70d499b..f3864dbc2 100644 --- a/src/i18n/locales/en/dataLoading.json +++ b/src/i18n/locales/en/dataLoading.json @@ -92,10 +92,11 @@ "listingFiles": "Listing files", "runningPython": "Running Python", "preparingPreview": "Preparing preview", - "browsingCatalog": "Browsing catalog", - "searchingData": "Searching data", - "describingData": "Reading table metadata", - "probingData": "Probing data", + "summarizingSources": "Summarizing connected data", + "browsingCatalog": "Browsing", + "searchingData": "Searching", + "describingData": "Reading table", + "probingData": "Probing", "proposingLoadPlan": "Proposing load plan" }, "examples": { diff --git a/src/i18n/locales/en/messages.json b/src/i18n/locales/en/messages.json index d5f894050..fa212ef41 100644 --- a/src/i18n/locales/en/messages.json +++ b/src/i18n/locales/en/messages.json @@ -22,10 +22,11 @@ "changesDiscarded": "Changes discarded", "formulate": "Formulate", "formulateAndOverride": "Formulate and override", - "viewSystemMessages": "view system messages", - "systemMessagesWithCount": "system messages ({{count}})", - "clearAllMessages": "clear all messages", - "details": "[details]", + "viewSystemMessages": "View system messages", + "systemMessagesWithCount": "System messages ({{count}})", + "showingLatest": "Showing the latest {{count}}", + "clearAllMessages": "Clear all messages", + "details": "Details", "generatedCode": "[generated code]", "chatWithAgents": "Dialog with Agents", "you": "You", diff --git a/src/i18n/locales/en/upload.json b/src/i18n/locales/en/upload.json index 10323a3d1..3aad0fe29 100644 --- a/src/i18n/locales/en/upload.json +++ b/src/i18n/locales/en/upload.json @@ -43,10 +43,10 @@ "agentChatSuggestionsLabel": "Try asking", "agentChatSendTooltip": "Start chatting with the agent", "dataSourcesLabel": "Connected to:", - "addSourceLabel": "Or add data directly:", + "addSourceLabel": "Add data:", "agentChatQuickAction": { - "connect": "Help me connect to my data source", - "askConnected": "What data do we have from connected sources?" + "connect": "Help me connect my data source", + "askConnected": "What data are available from my sources?" }, "agentChatSuggestion": { "askConnected": "What datasets do we have from connected sources?", @@ -69,6 +69,7 @@ "addConnectionDesc": "Connect to a live database", "connectorConnected": "Connected", "connectorDisconnected": "Click to connect", + "connectorNotConnected": "Not connected", "pickDataSourceType": "Choose a data source type to create a new connection.", "nameYourConnection": "Name your {{type}} connection.", "connectionName": "Connection name", diff --git a/src/i18n/locales/zh/common.json b/src/i18n/locales/zh/common.json index 1d20a31d0..e1fd49f2a 100644 --- a/src/i18n/locales/zh/common.json +++ b/src/i18n/locales/zh/common.json @@ -52,6 +52,7 @@ "title": "后端日志", "viewLogs": "查看后端日志", "refresh": "刷新", + "searchSavedState": "搜索保存的状态 (Cmd/Ctrl+F)", "download": "下载完整日志", "empty": "日志文件为空。" }, @@ -462,6 +463,9 @@ "rulesLoaded": "读取规则：{{rules}}", "knowledgeLoaded": "读取知识：{{knowledge}}", "searching": "搜索中...", + "listingConnectors": "检查可用连接器", + "readingConnector": "读取连接器设置", + "usingTool": "使用 {{tool}}", "producingAction": "输出 {{action}} 中...", "jumpToThreadRange": "跳转到线程 {{label}}", "collapse": "收起", @@ -617,6 +621,7 @@ "agentWorking": "Agent 努力工作中...", "attachUploadFailed": "附加 {{name}} 失败", "replyPlaceholder": "回复 Agent 的问题...", + "emptyAnalysisInputsPlaceholder": "按 Tab 询问有哪些数据可加载", "explorePlaceholder": "有什么问题，有什么想要探索的？（用 @ 添加上下文）", "explorePlaceholderSingleTable": "有什么问题，有什么想要探索的？", "addMoreData": "向工作区添加更多数据", @@ -672,6 +677,7 @@ "delegateToReportGen": "生成报告", "errorDuringExploration": "探索过程中出错", "explorationStep": "探索步骤 {{step}}：{{question}}", + "emptyAnalysisInputsPrompt": "有哪些数据可以加载？", "threadExplorePrompt": "探索这份数据中有趣的模式和趋势", "explorationThreadDeriveDescription": "从 {{source}} 派生，指令：{{instruction}}", "explorationStepCodeComment": "# 探索步骤 {{step}}", @@ -851,9 +857,10 @@ "refresh": "刷新数据", "emptyTree": "未找到表格", "addConnector": "添加数据连接器", - "configureConnector": "编辑连接", + "connectConnector": "连接", "linkLocalFolder": "链接本地文件夹", "newSession": "新建会话", + "importSession": "导入会话", "noSessions": "暂无已保存的会话", "tableCount": "{{count}} 个表格", "chartCount": "{{count}} 个图表", @@ -879,7 +886,9 @@ "loadingEllipsis": "加载中...", "loadWithFilters": "按条件筛选", "load": "加载", - "disconnectConnector": "断开连接器", + "disconnectConnector": "断开连接", + "connectorConnected": "已连接到「{{name}}」", + "failedConnectConnector": "连接失败", "connectorDisconnected": "连接器「{{name}}」已断开", "failedDisconnectConnector": "断开连接器失败", "failedSearchConnector": "搜索 {{connector}} 失败", @@ -921,6 +930,15 @@ "sortRecentlyModifiedFirst": "最近修改优先", "sortNameAsc": "名称 (a–z)", "sortSessions": "排序会话", + "organizeSessions": "分组和排序会话", + "groupSessions": "分组", + "groupBySource": "数据源", + "groupSourceShort": "数据源", + "noGrouping": "不分组", + "sourceUpload": "上传", + "sourceExampleDatasets": "示例数据集", + "sourceNoData": "无数据", + "sourceOther": "其他", "runCatalogSearch": "搜索", "clearCatalogSearch": "清除搜索", "timeJustNow": "刚刚", diff --git a/src/i18n/locales/zh/dataLoading.json b/src/i18n/locales/zh/dataLoading.json index 439b79486..06f340602 100644 --- a/src/i18n/locales/zh/dataLoading.json +++ b/src/i18n/locales/zh/dataLoading.json @@ -92,10 +92,11 @@ "listingFiles": "列出文件", "runningPython": "运行 Python", "preparingPreview": "准备预览", - "browsingCatalog": "浏览目录", - "searchingData": "搜索数据", - "describingData": "读取表元数据", - "probingData": "探查数据", + "summarizingSources": "汇总已连接数据", + "browsingCatalog": "浏览", + "searchingData": "搜索", + "describingData": "读取表", + "probingData": "探查", "proposingLoadPlan": "生成加载方案" }, "examples": { diff --git a/src/i18n/locales/zh/messages.json b/src/i18n/locales/zh/messages.json index 59185a446..fafcd7a73 100644 --- a/src/i18n/locales/zh/messages.json +++ b/src/i18n/locales/zh/messages.json @@ -24,8 +24,9 @@ "formulateAndOverride": "生成并覆盖", "viewSystemMessages": "查看系统消息", "systemMessagesWithCount": "系统消息（{{count}}）", + "showingLatest": "显示最近 {{count}} 条", "clearAllMessages": "清空全部消息", - "details": "[详情]", + "details": "详情", "generatedCode": "[生成代码]", "chatWithAgents": "与 Agent 对话", "you": "你", diff --git a/src/i18n/locales/zh/upload.json b/src/i18n/locales/zh/upload.json index 124ca5874..86140fdc4 100644 --- a/src/i18n/locales/zh/upload.json +++ b/src/i18n/locales/zh/upload.json @@ -43,10 +43,10 @@ "agentChatSuggestionsLabel": "试试这样问", "agentChatSendTooltip": "开始与助手对话", "dataSourcesLabel": "已连接：", - "addSourceLabel": "或直接添加数据：", + "addSourceLabel": "添加数据：", "agentChatQuickAction": { "connect": "帮我连接数据源", - "askConnected": "已连接的数据源里有哪些数据？" + "askConnected": "我的数据源中有哪些可用数据？" }, "agentChatSuggestion": { "askConnected": "已连接的数据源里有哪些数据集？", @@ -69,6 +69,7 @@ "addConnectionDesc": "连接到实时数据库", "connectorConnected": "已连接", "connectorDisconnected": "点击连接", + "connectorNotConnected": "未连接", "pickDataSourceType": "选择数据源类型以创建新连接。", "nameYourConnection": "为您的 {{type}} 连接命名。", "connectionName": "连接名称", diff --git a/src/views/DataFormulator.tsx b/src/views/DataFormulator.tsx index cc3cb9d0e..fcb8415bb 100644 --- a/src/views/DataFormulator.tsx +++ b/src/views/DataFormulator.tsx @@ -797,28 +797,39 @@ export const DataFormulatorFC = ({ }) => { {/* Hero — fills the viewport so title + input own the first screen; Demos/Sessions live below the fold and just peek up. */} - - + + + + {toolName} + + + - {toolName} + {t('landing.tagline')} - - {t('landing.tagline')} - {/* Hosted-demo notice — borderless strip (it's prose, not a button) placed before the Import Data section. The rocket @@ -906,7 +917,7 @@ export const DataFormulatorFC = ({ }) => { )} - + openUploadDialog(tab)} onSelectConnector={(conn) => { @@ -932,7 +943,7 @@ export const DataFormulatorFC = ({ }) => { demo, since first-time visitors won't have any sessions yet and demos are the most engaging entry point. */} - + {t('landing.demos')} { {/* ── Saved workspaces section ──────────────────────────── */} - {/* Section header — left-aligned label with the sort control - on the right, aligned to the card grid. */} - + {t('workspace.yourSessions')} - )[sessionSort]}`} placement="bottom"> - + + + + + + + + {t('sidebar.groupSessions', { defaultValue: 'Group' })} + + {([ + ['source', t('sidebar.groupBySource', { defaultValue: 'Data source' })], + ['none', t('sidebar.noGrouping', { defaultValue: 'No grouping' })], + ] as [SessionGroupKey, string][]).map(([key, label]) => ( + { + setSessionGroup(key); + setSessionSortAnchor(null); + }} + sx={{ fontSize: textVar.sm, py: 0.75 }} + > + + {sessionGroup === key && } + + + + ))} + + + {t('sidebar.sortSessions', { defaultValue: 'Sort' })} + {([ - ['updated_desc', t('sidebar.sortRecentlyModifiedFirst')], ['created_desc', t('sidebar.sortNewestFirst')], ['created_asc', t('sidebar.sortOldestFirst')], + ['updated_desc', t('sidebar.sortRecentlyModifiedFirst')], ['name_asc', t('sidebar.sortNameAsc')], ] as [SessionSortKey, string][]).map(([key, label]) => ( ))} - {pinAction} - - - - - {sessions.length === 0 ? ( @@ -2273,7 +2368,24 @@ const DataSourceSidebarPanel: React.FC<{ ) : ( - sortedSessions.map((s) => { + sessionSections.map((section, sectionIndex) => ( + + {sessionGroup === 'source' && ( + + + {section.label} + + + + )} + {section.sessions.map((s) => { const isRenaming = renamingSession === s.id; return ( ); - }) + })} + + )) )} diff --git a/src/views/DataThread.tsx b/src/views/DataThread.tsx index 7e33212bc..c3640fc05 100644 --- a/src/views/DataThread.tsx +++ b/src/views/DataThread.tsx @@ -137,17 +137,21 @@ const LiveStatus: React.FC<{ startTime?: number; resetKey?: string }> = ({ start }; /** Render a multi-step thinking banner as a single block with sectioned steps. + * Steps read as progress, not a transcript, so only the active one shows. * When `startTime` is provided, the live timer is appended *inline* next to - * the active (last) step's text — same alignment grammar as the single-line + * the active step's text — same alignment grammar as the single-line * ThinkingBanner — rather than right-flushed in a separate column. * The timer resets whenever the active step changes so it shows the time * spent on the **current** action, not the cumulative wait. */ export const ThinkingStepsBanner = (steps: string[], sx?: SxProps, startTime?: number, active: boolean = true) => { - const activeStep = steps.length > 0 ? steps[steps.length - 1] : ''; + const lastStep = steps.length > 0 ? steps[steps.length - 1] : ''; + // While the run is live the latest step stays in progress even after its own + // tool returned — the agent is already working on whatever comes next. + const activeStep = active && lastStep.startsWith('✓') ? lastStep.slice(2) : lastStep; return ( : undefined} /> @@ -537,7 +541,6 @@ let SingleThreadGroupView: FC<{ let tables = useSelector(dfSelectors.getAllTables); const derivedTables = useSelector(dfSelectors.getDerivedTables); - const inferredTableNames = useSelector((state: DataFormulatorState) => state.tableSemantics); const { t } = useTranslation(); const tableById = useMemo(() => new Map(tables.map(t => [t.id, t])), [tables]); @@ -704,6 +707,23 @@ let SingleThreadGroupView: FC<{ return map; }, [loadedTableNodes]); + const highlightedTextTurnIds = useMemo(() => { + const ids = new Set(); + for (const node of loadedTableNodes) { + if (!globalHighlightedTableIds.includes(node.tableId)) continue; + let current: string | undefined = node.parentNodeId; + const seen = new Set(); + while (current && !seen.has(current)) { + seen.add(current); + const turn = turnById.get(current); + if (!turn) break; + ids.add(turn.id); + current = turn.parentNodeId; + } + } + return ids; + }, [globalHighlightedTableIds, loadedTableNodes, turnById]); + const tableAnchorOfNode = (nodeId: string | undefined): string => { let current = nodeId; const seen = new Set(); @@ -779,15 +799,13 @@ let SingleThreadGroupView: FC<{ }; let _buildTableCard = (tableId: string) => { - const inferredDisplayName = inferredTableNames.find(info => info.tableId === tableId)?.displayName; - return buildTableCard({ tableId, inferredDisplayName, ...tableCardProps }); + return buildTableCard({ tableId, ...tableCardProps }); } /** Pointer to a table whose real card lives in the shelf or a prior column. */ let _buildRefChip = (tableId: string) => { - const displayName = inferredTableNames.find(info => info.tableId === tableId)?.displayName; return buildTableRefChip({ - tableId, table: tableById.get(tableId), displayName, + tableId, table: tableById.get(tableId), focused: tableId === focusedTableId, dispatch, }); } @@ -1362,15 +1380,16 @@ let SingleThreadGroupView: FC<{ // Render a single text turn: its triggering prompt bubble (if any) then the // turn card. `keyNode` seeds prompt-entry keys. const pushSingleTurn = (turn: TextTurn, keyNode: string, highlighted: boolean, triggerType: 'trigger' | 'leaf-trigger') => { + const turnHighlighted = highlighted || highlightedTextTurnIds.has(turn.id); if (turn.prompt) { pushInteractionEntries( [{ from: 'user', to: 'data-agent', role: 'prompt', content: turn.prompt, timestamp: turn.createdAt }], - keyNode, triggerType, highlighted, `textturn-prompt-${turn.id}`, + keyNode, triggerType, turnHighlighted, `textturn-prompt-${turn.id}`, ); } - timelineItems.push(buildTextTurnTimelineItem(turn, highlighted, false)); + timelineItems.push(buildTextTurnTimelineItem(turn, turnHighlighted, false)); for (const report of reportsByParentNode.get(turn.id) || []) { - timelineItems.push(buildReportTimelineItem(report, highlighted)); + timelineItems.push(buildReportTimelineItem(report, turnHighlighted)); } // A turn that loaded tables skips the reply — the tables below already // say which option was taken. @@ -1378,7 +1397,7 @@ let SingleThreadGroupView: FC<{ if (turn.answered && turn.answer && loadedTables.length === 0) { pushInteractionEntries( [{ from: 'user', to: 'data-agent', role: 'prompt', content: turn.answer }], - keyNode, triggerType, highlighted, `textturn-answer-${turn.id}`, + keyNode, triggerType, turnHighlighted, `textturn-answer-${turn.id}`, ); } }; diff --git a/src/views/DataThreadCards.tsx b/src/views/DataThreadCards.tsx index 332bc38b7..58d219310 100644 --- a/src/views/DataThreadCards.tsx +++ b/src/views/DataThreadCards.tsx @@ -133,11 +133,10 @@ export let buildChartCards = ( export let buildTableRefChip = (props: { tableId: string; table: DictTable | undefined; - displayName?: string; focused: boolean; dispatch: any; }) => { - const { tableId, table, displayName, focused, dispatch } = props; + const { tableId, table, focused, dispatch } = props; return {displayName?.trim() || table?.displayId || tableId} + }}>{table?.displayId || tableId} @@ -202,7 +201,6 @@ export let buildTriggerCard = ( export interface BuildTableCardProps { tableId: string; tables: DictTable[]; - inferredDisplayName?: string; chartElements: { tableId: string, chartId: string, element: any }[]; usedIntermediateTableIds: string[]; highlightedTableIds: string[]; @@ -223,7 +221,7 @@ export interface BuildTableCardProps { export let buildTableCard = (props: BuildTableCardProps) => { const { - tableId, tables, inferredDisplayName, chartElements, usedIntermediateTableIds, + tableId, tables, chartElements, usedIntermediateTableIds, highlightedTableIds, focusedTableId, focusedChartId, parentTable, tableIdList, collapsed, dispatch, handleOpenTableMenu, primaryBgColor, t, showOriginalName = true, @@ -256,11 +254,8 @@ export let buildTableCard = (props: BuildTableCardProps) => { let table = tables.find(t => t.id == tableId); const originalName = getOriginalName(table); const sourceTooltip = getSourceTooltip(table); - const workspaceName = table?.displayId || tableId; + const friendlyName = table?.displayId || tableId; const normalizeTableName = (name: string) => name.toLowerCase().replace(/[\s_-]+/g, ''); - const friendlyName = inferredDisplayName?.trim() - ? inferredDisplayName.trim() - : workspaceName; const rawName = showOriginalName && originalName && normalizeTableName(originalName) !== normalizeTableName(friendlyName) diff --git a/src/views/DataView.tsx b/src/views/DataView.tsx index cbc8c3140..ab68be992 100644 --- a/src/views/DataView.tsx +++ b/src/views/DataView.tsx @@ -96,8 +96,7 @@ export const FreeDataViewFC: FC = function DataView({ maximiz const tableSemantics = useSelector((state: DataFormulatorState) => state.tableSemantics.find(info => info.tableId === focusedTableId), ); - const displayName = tableSemantics?.displayName?.trim() - || targetTable?.displayId + const displayName = targetTable?.displayId || targetTable?.id || 'table'; const realName = targetTable?.derive diff --git a/src/views/InteractionEntryCard.tsx b/src/views/InteractionEntryCard.tsx index 79f3ac573..ba026a2d0 100644 --- a/src/views/InteractionEntryCard.tsx +++ b/src/views/InteractionEntryCard.tsx @@ -50,7 +50,11 @@ const PlanStepItem: React.FC<{ const isFailed = step.startsWith('✗'); const isWarning = step.startsWith('⚠'); const isInfo = step.startsWith('📋'); - const displayLine = (isChecked || isFailed) ? step.slice(2) : (isWarning || isInfo) ? step.slice(2).trimStart() : step; + const rawLine = (isChecked || isFailed) ? step.slice(2) : (isWarning || isInfo) ? step.slice(2).trimStart() : step; + // Trailing ellipsis marks the step still in flight; some labels ship their own. + const displayLine = showShimmer && !/(\.\.\.|…)$/.test(rawLine.trim()) + ? `${rawLine}…` + : rawLine; const IconComp = getStepIconComponent(step); // Text stays in the normal muted color even for failed/warning steps — the diff --git a/src/views/LogViewerDialog.tsx b/src/views/LogViewerDialog.tsx index 7c273dcac..6d393493b 100644 --- a/src/views/LogViewerDialog.tsx +++ b/src/views/LogViewerDialog.tsx @@ -15,8 +15,23 @@ import React, { FC, useCallback, useEffect, useRef, useState } from 'react'; import CodeMirror, { EditorView } from '@uiw/react-codemirror'; -import { foldEffect, syntaxTree } from '@codemirror/language'; +import { EditorState } from '@codemirror/state'; +import { keymap, Panel } from '@codemirror/view'; +import { ensureSyntaxTree, foldEffect, forceParsing } from '@codemirror/language'; import { json } from '@codemirror/lang-json'; +import { + closeSearchPanel, + findNext, + findPrevious, + getSearchQuery, + openSearchPanel, + search, + SearchQuery, + searchKeymap, + selectMatches, + setSearchQuery, +} from '@codemirror/search'; +import { SyntaxNode } from '@lezer/common'; import { Box, CircularProgress, @@ -32,6 +47,7 @@ import { import TerminalOutlinedIcon from '@mui/icons-material/TerminalOutlined'; import RefreshIcon from '@mui/icons-material/Refresh'; import DownloadIcon from '@mui/icons-material/Download'; +import SearchIcon from '@mui/icons-material/Search'; import CloseIcon from '@mui/icons-material/Close'; import { useTranslation } from 'react-i18next'; import { useSelector } from 'react-redux'; @@ -42,7 +58,191 @@ import { DataFormulatorState } from '../app/dfSlice'; import { textVar } from '../app/layout'; const DEFAULT_TAIL_LINES = 500; -const DEFAULT_FOLD_CHARACTER_THRESHOLD = 2000; + +export function createSavedStateSearchPanel(view: EditorView): Panel { + const searchInput = document.createElement('input'); + searchInput.type = 'text'; + searchInput.className = 'cm-textfield'; + searchInput.name = 'df-saved-state-find'; + searchInput.placeholder = 'Find'; + searchInput.setAttribute('aria-label', 'Find'); + searchInput.setAttribute('main-field', 'true'); + searchInput.setAttribute('autocomplete', 'off'); + searchInput.setAttribute('autocorrect', 'off'); + searchInput.setAttribute('autocapitalize', 'off'); + searchInput.setAttribute('spellcheck', 'false'); + searchInput.setAttribute('aria-autocomplete', 'none'); + searchInput.setAttribute('data-1p-ignore', 'true'); + searchInput.setAttribute('data-lpignore', 'true'); + searchInput.value = getSearchQuery(view.state).search; + + const updateQuery = () => { + const current = getSearchQuery(view.state); + view.dispatch({ + effects: setSearchQuery.of(new SearchQuery({ + search: searchInput.value, + caseSensitive: current.caseSensitive, + literal: current.literal, + regexp: current.regexp, + wholeWord: current.wholeWord, + })), + }); + }; + searchInput.addEventListener('input', updateQuery); + searchInput.addEventListener('keydown', event => { + if (event.key === 'Enter') { + event.preventDefault(); + (event.shiftKey ? findPrevious : findNext)(view); + } else if (event.key === 'Escape') { + event.preventDefault(); + closeSearchPanel(view); + } + }); + + const makeButton = (name: string, label: string, action: () => void) => { + const button = document.createElement('button'); + button.type = 'button'; + button.className = name === 'close' ? '' : 'cm-button'; + button.name = name; + button.textContent = label; + button.setAttribute('aria-label', label); + button.addEventListener('click', action); + return button; + }; + + const panel = document.createElement('div'); + panel.className = 'cm-search'; + panel.append( + searchInput, + makeButton('next', 'Next', () => { findNext(view); }), + makeButton('prev', 'Previous', () => { findPrevious(view); }), + makeButton('select', 'All', () => { selectMatches(view); }), + makeButton('close', '×', () => { closeSearchPanel(view); }), + ); + + return { + dom: panel, + update(update) { + const query = getSearchQuery(update.state); + if (searchInput.value !== query.search) searchInput.value = query.search; + }, + destroy() { + searchInput.removeEventListener('input', updateQuery); + }, + }; +} + +const savedStateEditorTheme = EditorView.theme({ + '&': { + height: '100%', + fontSize: textVar.sm, + }, + '&.cm-focused': { outline: 'none' }, + '.cm-scroller': { fontFamily: 'var(--df-font-mono)' }, + '.cm-panels': { + backgroundColor: '#f7f8fa', + color: '#30343b', + fontFamily: 'Roboto, sans-serif', + }, + '.cm-panels.cm-panels-bottom': { + borderTop: '1px solid rgba(0, 0, 0, 0.12)', + }, + '.cm-search': { + display: 'flex', + alignItems: 'center', + gap: '6px', + padding: '7px 10px', + }, + '.cm-search label, .cm-search br': { display: 'none' }, + '.cm-search .cm-textfield': { + width: 'min(320px, 45vw)', + height: '30px', + boxSizing: 'border-box', + padding: '4px 9px', + border: '1px solid rgba(0, 0, 0, 0.18)', + borderRadius: '6px', + backgroundColor: '#fff', + color: '#202124', + fontFamily: 'var(--df-font-mono)', + fontSize: `${textVar.sm}px`, + outline: 'none', + }, + '.cm-search .cm-textfield:focus': { + borderColor: '#1976d2', + boxShadow: '0 0 0 2px rgba(25, 118, 210, 0.14)', + }, + '.cm-search .cm-button': { + height: '30px', + boxSizing: 'border-box', + margin: '0', + padding: '4px 10px', + border: '1px solid rgba(0, 0, 0, 0.14)', + borderRadius: '6px', + backgroundImage: 'none', + backgroundColor: '#fff', + color: '#3c4043', + fontFamily: 'Roboto, sans-serif', + fontSize: `${textVar.xs}px`, + cursor: 'pointer', + }, + '.cm-search .cm-button:hover': { + borderColor: 'rgba(25, 118, 210, 0.45)', + backgroundColor: 'rgba(25, 118, 210, 0.06)', + color: '#1565c0', + }, + '.cm-search button[name="close"]': { + position: 'static', + width: '30px', + height: '30px', + marginLeft: 'auto', + border: '0', + borderRadius: '6px', + backgroundColor: 'transparent', + color: '#5f6368', + fontSize: '18px', + cursor: 'pointer', + }, + '.cm-search button[name="close"]:hover': { + backgroundColor: 'rgba(0, 0, 0, 0.06)', + color: '#202124', + }, +}); + +const savedStateEditorExtensions = [ + json(), + search({ createPanel: createSavedStateSearchPanel }), + keymap.of(searchKeymap), + EditorView.lineWrapping, + savedStateEditorTheme, +]; + +const SAVED_STATE_AUTO_FOLD_PATHS = [ + // Table payloads: keep IDs, names, lineage, and virtual references visible. + ['inputTables', '*', 'snapshot'], + ['derivedTables', '*', 'rows'], + ['derivedTables', '*', 'metadata'], + // Generated derivation evidence and conversation traces. + ['derivedTables', '*', 'derive', 'dialog'], + ['derivedTables', '*', 'derive', 'explanation'], + ['derivedTables', '*', 'derive', 'trigger', 'interaction'], + ['draftNodes', '*', 'derive', 'dialog'], + ['draftNodes', '*', 'derive', 'trigger', 'interaction'], + ['draftNodes', '*', 'derive', 'pendingClarification', 'trajectory'], + // Generated visual/report payloads. + ['charts', '*', 'styleVariants'], + ['generatedReports', '*', 'inspectionSteps'], + // Structured artifacts: keep turn identity, kind, status, and parent visible. + ['textTurns', '*', 'options'], + ['textTurns', '*', 'form'], + ['textTurns', '*', 'dataOperation'], + ['textTurns', '*', 'resume', 'trajectory'], + // Embedded loading results: keep message role, content, and timestamp visible. + ['dataLoadingChatMessages', '*', 'codeBlocks'], + ['dataLoadingChatMessages', '*', 'tables'], + ['dataLoadingChatMessages', '*', 'loadPlan'], + ['dataLoadingChatMessages', '*', 'dataOperation'], + ['dataLoadingChatMessages', '*', 'connectorForm'], +]; interface LogTailResponse { path: string | null; @@ -56,27 +256,56 @@ interface SessionLoadResponse { state: Record; } -function foldLargeJsonValues(view: EditorView): void { - const effects: ReturnType[] = []; - syntaxTree(view.state).iterate({ +function jsonContainerPath(state: EditorState, node: SyntaxNode): string[] { + const path: string[] = []; + let current: SyntaxNode | null = node; + while (current?.parent) { + const parent: SyntaxNode = current.parent; + if (parent.name === 'Property') { + const propertyName = parent.getChild('PropertyName'); + if (propertyName) { + try { + path.unshift(JSON.parse(state.doc.sliceString(propertyName.from, propertyName.to))); + } catch { + return []; + } + } + } else if (parent.name === 'Array') { + path.unshift('*'); + } + current = parent; + } + return path; +} + +export function getSavedStateAutoFoldRanges(state: EditorState): { from: number; to: number }[] { + const ranges: { from: number; to: number }[] = []; + const tree = ensureSyntaxTree(state, state.doc.length, 100); + if (!tree) return ranges; + tree.iterate({ enter(node) { const isContainer = node.name === 'Array' || node.name === 'Object'; const isRoot = node.node.parent === null; - const property = node.node.parent; - const propertyPrefix = property?.name === 'Property' - ? view.state.doc.sliceString(property.from, node.from) - : ''; - const propertyName = propertyPrefix.match(/"([^"\\]+)"\s*:\s*$/)?.[1]?.toLowerCase() || ''; - const isAgentConversation = /agent|chat|message|dialog/.test(propertyName); - if (isContainer && !isRoot && ( - isAgentConversation || node.to - node.from >= DEFAULT_FOLD_CHARACTER_THRESHOLD - )) { - effects.push(foldEffect.of({ from: node.from + 1, to: node.to - 1 })); + if (!isContainer || isRoot) return undefined; + const path = jsonContainerPath(state, node.node); + const matches = SAVED_STATE_AUTO_FOLD_PATHS.some(pattern => + pattern.length === path.length && pattern.every((segment, index) => segment === path[index]) + ); + if (matches) { + if (node.to - node.from > 2) { + ranges.push({ from: node.from + 1, to: node.to - 1 }); + } return false; } return undefined; }, }); + return ranges; +} + +function foldSavedStatePaths(view: EditorView): void { + forceParsing(view, view.state.doc.length, 200); + const effects = getSavedStateAutoFoldRanges(view.state).map(range => foldEffect.of(range)); if (effects.length > 0) view.dispatch({ effects }); } @@ -108,6 +337,7 @@ export const LogViewerDialog: FC<{ const [activeTab, setActiveTab] = useState(0); const [savedState, setSavedState] = useState(''); const preRef = useRef(null); + const savedStateEditorRef = useRef(null); const fetchLogs = useCallback(async () => { setLoading(true); @@ -161,11 +391,38 @@ export const LogViewerDialog: FC<{ } }, [content, open]); + useEffect(() => { + if (activeTab === 1 && savedState && savedStateEditorRef.current) { + foldSavedStatePaths(savedStateEditorRef.current); + } + }, [activeTab, savedState]); + + useEffect(() => { + if (!open || activeTab !== 1) return; + const handleSavedStateSearchShortcut = (event: KeyboardEvent) => { + if ((event.metaKey || event.ctrlKey) && !event.altKey && event.key.toLowerCase() === 'f') { + event.preventDefault(); + event.stopPropagation(); + if (savedStateEditorRef.current) { + openSearchPanel(savedStateEditorRef.current); + } + } + }; + window.addEventListener('keydown', handleSavedStateSearchShortcut, true); + return () => window.removeEventListener('keydown', handleSavedStateSearchShortcut, true); + }, [activeTab, open]); + const handleDownload = () => { // Direct navigation triggers the browser download (attachment header). window.open(getUrls().LOGS_DOWNLOAD, '_blank'); }; + const handleSearchSavedState = () => { + if (savedStateEditorRef.current) { + openSearchPanel(savedStateEditorRef.current); + } + }; + const handleRefresh = activeTab === 0 ? fetchLogs : fetchSavedState; return ( @@ -198,6 +455,19 @@ export const LogViewerDialog: FC<{ + {activeTab === 1 && + + + + + + } {activeTab === 0 && @@ -300,7 +570,7 @@ export const LogViewerDialog: FC<{ { + savedStateEditorRef.current = view; + }} aria-label={t('logs.savedStateTab', { defaultValue: 'Saved State' })} /> diff --git a/src/views/MessageSnackbar.tsx b/src/views/MessageSnackbar.tsx index 48d6bb9e4..43b4d0797 100644 --- a/src/views/MessageSnackbar.tsx +++ b/src/views/MessageSnackbar.tsx @@ -7,14 +7,20 @@ import IconButton from '@mui/material/IconButton'; import CloseIcon from '@mui/icons-material/Close'; import { DataFormulatorState, dfActions } from '../app/dfSlice'; import { useDispatch, useSelector } from 'react-redux'; -import { Alert, Box, Paper, Tooltip, Typography } from '@mui/material'; +import { Alert, Box, Button, Paper, Tooltip, Typography, alpha, useTheme } from '@mui/material'; import InfoIcon from '@mui/icons-material/Info'; -import DeleteIcon from '@mui/icons-material/Delete'; +import InfoOutlinedIcon from '@mui/icons-material/InfoOutlined'; +import DeleteOutlineIcon from '@mui/icons-material/DeleteOutline'; import CheckCircleIcon from '@mui/icons-material/CheckCircle'; +import CheckCircleOutlineIcon from '@mui/icons-material/CheckCircleOutline'; import ErrorOutlineIcon from '@mui/icons-material/ErrorOutline'; +import WarningAmberOutlinedIcon from '@mui/icons-material/WarningAmberOutlined'; import ContentCopyIcon from '@mui/icons-material/ContentCopy'; +import ChevronRightIcon from '@mui/icons-material/ChevronRight'; +import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; import { useTranslation } from 'react-i18next'; import { iconVar, textVar } from '../app/layout'; +import { borderColor, radius, shadow } from '../app/tokens'; export interface Message { type: "success" | "info" | "error" | "warning", @@ -26,18 +32,11 @@ export interface Message { diagnostics?: any, // full diagnostic payload from the backend agent pipeline } -const TYPE_SYMBOLS: Record = { - error: '✗', - warning: '⚠', - info: 'ℹ', - success: '✓', -}; - -const TYPE_COLORS: Record = { - error: '#d32f2f', - warning: '#ed6c02', - info: '#0288d1', - success: '#2e7d32', +const SeverityIcon: React.FC<{ type: Message['type'] }> = ({ type }) => { + if (type === 'error') return ; + if (type === 'warning') return ; + if (type === 'success') return ; + return ; }; // Helper function to format timestamp @@ -51,6 +50,7 @@ const formatTimestamp = (timestamp: number) => { }; const DiagnosticsViewer: React.FC<{ diagnostics: any }> = React.memo(({ diagnostics }) => { + const theme = useTheme(); const [expanded, setExpanded] = React.useState(false); const [copied, setCopied] = React.useState(false); const jsonStr = React.useMemo(() => JSON.stringify(diagnostics, null, 2), [diagnostics]); @@ -63,40 +63,48 @@ const DiagnosticsViewer: React.FC<{ diagnostics: any }> = React.memo(({ diagnost }, [jsonStr]); return ( -
- - + + {expanded && ( - - + + )} - + {expanded && ( - 
                     {jsonStr}
-
+ )} -
+ ); }); @@ -107,6 +115,7 @@ export const MessageSnackbar = React.memo(function MessageSnackbar() { const dispatch = useDispatch(); const { t } = useTranslation(); + const theme = useTheme(); const [openLastMessage, setOpenLastMessage] = React.useState(false); const [latestMessage, setLatestMessage] = React.useState(); @@ -184,170 +193,284 @@ export const MessageSnackbar = React.memo(function MessageSnackbar() { return ( - setOpenMessages(true)} + aria-label={t('messages.viewSystemMessages')} + onClick={() => { + setOpenLastMessage(false); + setOpenMessages(open => !open); + }} > - {buttonSeverity === "error" ? : - buttonSeverity === "warning" ? : - buttonSeverity === "success" ? : - } + {buttonSeverity === 'error' ? : + buttonSeverity === 'warning' ? : + buttonSeverity === 'success' ? : + } - {/* Header */} - - - {t('messages.systemMessagesWithCount', { count: messages.length })}{messages.length > MAX_DISPLAY_MESSAGES ? ` — showing latest ${MAX_DISPLAY_MESSAGES}` : ''} - + + + + {t('messages.systemMessagesWithCount', { count: messages.length })} + + {messages.length > MAX_DISPLAY_MESSAGES && ( + + {t('messages.showingLatest', { + count: MAX_DISPLAY_MESSAGES, + defaultValue: 'Showing the latest {{count}}', + })} + + )} + { dispatch(dfActions.clearMessages()); dispatch(dfActions.setDisplayedMessageIndex(0)); setOpenMessages(false); }} + sx={{ color: 'text.secondary', '&:hover': { color: 'error.main' } }} > - + setOpenMessages(false)} + sx={{ color: 'text.secondary' }} > - + - {/* Message list — plain text, no MUI Alert per row */} -
{messages.length === 0 && ( - {t('messages.noMessages')} + + + + {t('messages.noMessages')} + + )} {groupedMessages.map((msg, index) => { - const color = TYPE_COLORS[msg.type] || '#333'; - const symbol = TYPE_SYMBOLS[msg.type] || '•'; + const color = theme.palette[msg.type].main; const hasDetails = !!(msg.detail || msg.code || msg.diagnostics); const isExpanded = expandedMessages.has(index); return ( -
- - {symbol} - [{formatTimestamp(msg.timestamp)}] - ({msg.component}) {msg.value} - {msg.count > 1 && ( - ×{msg.count} - )} - {hasDetails && ( - toggleExpand(index)} - > - {isExpanded ? `▾ ${t('messages.details')}` : `▸ ${t('messages.details')}`} - - )} - - {hasDetails && isExpanded && ( -
- {msg.detail && ( -
- — details — - {msg.detail} -
+ + + + + + + {msg.value} + + + + {msg.component} + + + {formatTimestamp(msg.timestamp)} + + {msg.count > 1 && ( + + ×{msg.count} + )} - {msg.code && ( -
- — code — - 
 : }
+                                                    onClick={() => toggleExpand(index)}
+                                                    sx={{
+                                                        minWidth: 0, p: 0,
+                                                        textTransform: 'none', fontSize: textVar.xxs,
+                                                        color: 'text.secondary',
+                                                        '& .MuiButton-startIcon': { mr: 0.125 },
+                                                        '&:hover': { color: 'primary.main', backgroundColor: 'transparent' },
+                                                    }}
+                                                >
+                                                    {t('messages.details')}
+                                                
+                                            )}
+                                        
+                                        {hasDetails && isExpanded && (
+                                            
+                                                {msg.detail && (
+                                                    
+                                                        {msg.detail}
+                                                    
+                                                )}
+                                                {msg.code && (
+                                                    
                                                         {msg.code.split('\n').filter(line => line.trim() !== '').join('\n')}
-
-
- )} - {msg.diagnostics && ( - - )} -
- )} -
+ + )} + {msg.diagnostics && } + + )} + + ); })} -
+ - {/* Last message toast — keep the single Alert for latest message popup */} - {latestMessage != undefined ? - - - [{formatTimestamp(latestMessage.timestamp)}] ({latestMessage.component}) {latestMessage?.value} - - {latestMessage?.detail && <> -
{latestMessage.detail}
- } - {latestMessage?.code && - 
+                
+                    
+                        {latestMessage.component} · {formatTimestamp(latestMessage.timestamp)}
+                    
+                    
+                        {latestMessage.value}
+                    
+                    {latestMessage.detail && (
+                        
+                            {latestMessage.detail}
+                        
+                    )}
+                    {latestMessage.code && (
+                        
                             {latestMessage.code.split('\n').filter(line => line.trim() !== '').join('\n')}
-
- } + + )} - : ""} + + ) : null} ); }); \ No newline at end of file diff --git a/src/views/ReportView.tsx b/src/views/ReportView.tsx index a67fbc032..735501367 100644 --- a/src/views/ReportView.tsx +++ b/src/views/ReportView.tsx @@ -42,6 +42,7 @@ export const ReportView: FC = () => { const config = useSelector((state: DataFormulatorState) => state.config); const allGeneratedReports = useSelector(dfSelectors.getAllGeneratedReports); const serverConfig = useSelector((state: DataFormulatorState) => state.serverConfig); + const activeWorkspace = useSelector((state: DataFormulatorState) => state.activeWorkspace); const focusedId = useSelector((state: DataFormulatorState) => state.focusedId); // Thumbnails live in their own slice so updates don't churn `state.charts`. const chartThumbnails = useSelector((state: DataFormulatorState) => state.chartThumbnails) || {}; @@ -143,9 +144,16 @@ export const ReportView: FC = () => { return sanitized || t('report.untitled'); }; - const getReportFileName = (extension: string): string => { + const getReportFileName = (extension: string, root?: ParentNode | null): string => { const date = new Date().toISOString().slice(0, 10); - return `${sanitizeFileName(getReportTitle())}-${date}.${extension}`; + const reportTitle = getReportTitle(root); + const sessionName = activeWorkspace?.displayName || activeWorkspace?.id || ''; + const normalizeName = (name: string) => name.toLowerCase().replace(/[\s_-]+/g, ''); + const parts = [reportTitle]; + if (sessionName && normalizeName(sessionName) !== normalizeName(reportTitle)) { + parts.push(sessionName); + } + return `${sanitizeFileName(parts.join(' - '))} - ${date}.${extension}`; }; const renderReportToCanvas = async (): Promise => { @@ -309,7 +317,7 @@ export const ReportView: FC = () => { const styles = Array.from(document.querySelectorAll('style, link[rel="stylesheet"]')) .map(node => node.outerHTML) .join('\n'); - const printTitle = sanitizeFileName(getReportTitle(exportClone.clone)); + const printTitle = getReportFileName('pdf', exportClone.clone).replace(/\.pdf$/, ''); const originalDocumentTitle = document.title; const doc = printFrame.contentDocument; const win = printFrame.contentWindow; @@ -324,7 +332,7 @@ export const ReportView: FC = () => { -${printTitle} + ${styles}