diff --git a/ci/.nim_models_used.json b/ci/.nim_models_used.json index a38fe13517..2055ebf564 100644 --- a/ci/.nim_models_used.json +++ b/ci/.nim_models_used.json @@ -7,7 +7,7 @@ }, { "model": "nvidia/nemotron-3-nano-30b-a3b", - "num_configs": 43 + "num_configs": 44 }, { "model": "meta/llama-3.3-70b-instruct", diff --git a/docs/source/run-workflows/observe/observe-workflow-with-mlflow.md b/docs/source/run-workflows/observe/observe-workflow-with-mlflow.md new file mode 100644 index 0000000000..d1177af339 --- /dev/null +++ b/docs/source/run-workflows/observe/observe-workflow-with-mlflow.md @@ -0,0 +1,62 @@ + + +# NVIDIA NeMo Agent Toolkit: Observing a Workflow with MLflow + +This guide shows how to send **OpenTelemetry** traces from NeMo Agent Toolkit to an [MLflow](https://mlflow.org/docs/latest/tracing/) tracking server using the built-in `mlflow` exporter (`nvidia-nat[opentelemetry]`). MLflow 3.6+ ingests OTLP spans at `/v1/traces` and routes them to an experiment via the `x-mlflow-experiment-id` header. For field reference and custom OTLP endpoints, see [Adding Telemetry Exporters](../../extend/custom-components/telemetry-exporters.md). + +## Step 1: Start an MLflow tracking server + +Trace ingestion requires a tracking server backed by a database store. Install MLflow (`pip install "mlflow>=3.6"`) and start it: + +```bash +mlflow server --backend-store-uri sqlite:///mlflow.db --host 127.0.0.1 --port 5000 +``` + +The OTLP ingestion endpoint is then `http://localhost:5000/v1/traces`, and the MLflow UI is at `http://localhost:5000`. + +## Step 2: Configure the environment + +```bash +# Optional: overrides the defaults in config-mlflow.yml +export MLFLOW_OTLP_ENDPOINT="http://localhost:5000/v1/traces" +export MLFLOW_EXPERIMENT_ID="0" # the MLflow experiment ID to route traces to (default experiment is "0") +``` + +## Step 3: Install the OpenTelemetry extra + +```bash +uv pip install -e ".[opentelemetry]" +# or, from PyPI: uv pip install "nvidia-nat[opentelemetry]" +``` + +## Step 4: Run the simple calculator observability example + +From the root of the NeMo Agent Toolkit repository: + +```bash +uv pip install -e examples/observability/simple_calculator_observability/ + +nat run --config_file examples/observability/simple_calculator_observability/configs/config-mlflow.yml --input "What is 2 * 4?" +``` + +You should see a log line such as `Started exporter 'mlflow'`. Open the MLflow UI at `http://localhost:5000`, select the experiment matching `MLFLOW_EXPERIMENT_ID`, and view the workflow trace under the **Traces** tab. + +## Related configuration + +- Example config: `examples/observability/simple_calculator_observability/configs/config-mlflow.yml` +- [Telemetry exporters reference](../../extend/custom-components/telemetry-exporters.md) diff --git a/docs/source/run-workflows/observe/observe.md b/docs/source/run-workflows/observe/observe.md index fc53bb1939..fc07ed3474 100644 --- a/docs/source/run-workflows/observe/observe.md +++ b/docs/source/run-workflows/observe/observe.md @@ -77,6 +77,7 @@ The following table lists each exporter with its supported features and configur | [Galileo](https://galileo.ai/) | [Observing with Galileo](?provider=Galileo#provider-integration-guides){.external} | Logging, Tracing | | [Langfuse](https://langfuse.com/) | Refer to the `examples/observability/simple_calculator_observability` example for usage details | Logging, Tracing | | [LangSmith](https://www.langchain.com/langsmith) | [Observing with LangSmith](?provider=LangSmith#provider-integration-guides){.external} | Logging, Tracing, Evaluation Metrics | +| [MLflow](https://mlflow.org/docs/latest/tracing/) | [Observing with MLflow](?provider=MLflow#provider-integration-guides){.external} | Logging, Tracing | | [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) | [Observing with OTel Collector](?provider=OTel-collector#provider-integration-guides){.external} | Logging, Tracing | | [Patronus](https://www.patronus.ai/) | Refer to the `examples/observability/simple_calculator_observability` example for usage details | Logging, Tracing | | [Phoenix](http://arize.com/phoenix/) | [Observing with Phoenix](?provider=Phoenix#provider-integration-guides){.external} | Logging, Tracing | @@ -227,6 +228,12 @@ For complete information about developing and integrating custom telemetry expor ::: + :::{tab-item} MLflow + :sync: MLflow + + :::{include} ./observe-workflow-with-mlflow.md + ::: + :::{tab-item} OTel Collector :sync: OTel-collector diff --git a/examples/observability/simple_calculator_observability/README.md b/examples/observability/simple_calculator_observability/README.md index 98ba9d9f7b..4914446e0a 100644 --- a/examples/observability/simple_calculator_observability/README.md +++ b/examples/observability/simple_calculator_observability/README.md @@ -23,7 +23,7 @@ This example demonstrates how to implement **observability and tracing capabilit ## Key Features -- **Multi-Platform Observability Integration:** Demonstrates integration with multiple observability platforms including Phoenix (local), Arize AX (hosted OTLP), Langfuse, LangSmith, Weave, and Patronus for comprehensive monitoring options. +- **Multi-Platform Observability Integration:** Demonstrates integration with multiple observability platforms including Phoenix (local), Arize AX (hosted OTLP), MLflow (local), Langfuse, LangSmith, Weave, and Patronus for comprehensive monitoring options. - **Distributed Tracing Implementation:** Shows how to track agent execution flow across components with detailed trace visualization including agent reasoning, tool calls, and LLM interactions. - **Performance Monitoring:** Demonstrates capturing latency metrics, token usage, resource consumption, and error tracking for production-ready AI system monitoring. - **Development and Production Patterns:** Provides examples for both local development tracing (Phoenix) and production monitoring setups with various enterprise observability platforms. @@ -43,7 +43,7 @@ Before starting this example, you need: 1. **Agent toolkit**: Ensure you have the Agent toolkit installed. If you have not already done so, follow the instructions in the [Install Guide](../../../docs/source/get-started/installation.md#install-from-source) to create the development environment and install NeMo Agent Toolkit. 2. **Base workflow**: This example builds upon the Getting Started [Simple Calculator](../../getting_started/simple_calculator/) example. Make sure you are familiar with the example before proceeding. -3. **Observability platform**: Access to at least one of the supported platforms (Phoenix, Arize AX, Langfuse, LangSmith, Weave, or Patronus) +3. **Observability platform**: Access to at least one of the supported platforms (Phoenix, Arize AX, MLflow, Langfuse, LangSmith, Weave, or Patronus) ## Installation @@ -120,6 +120,24 @@ Send traces to [Arize AX](https://arize.com/docs/ax/) using the `arize_ax` expor 3. Open the Arize project matching your project name to view traces. For **EU** residency, set `use_eu_region: true` under `arize_ax` in the config file. +### MLflow + +Send traces to a local [MLflow](https://mlflow.org/docs/latest/tracing/) tracking server using the `mlflow` exporter (`nvidia-nat[opentelemetry]`). MLflow 3.6+ ingests OTLP spans at `/v1/traces`, routed to an experiment via `x-mlflow-experiment-id`. + +1. Start a DB-backed tracking server: + + ```bash + mlflow server --backend-store-uri sqlite:///mlflow.db --host 127.0.0.1 --port 5000 + ``` + +2. Run the example: + + ```bash + nat run --config_file examples/observability/simple_calculator_observability/configs/config-mlflow.yml --input "What is 2 * 4?" + ``` + +3. Open the MLflow UI at `http://localhost:5000` and view the trace under the experiment's **Traces** tab. Override the target with `MLFLOW_OTLP_ENDPOINT` / `MLFLOW_EXPERIMENT_ID`. + ### File-Based Tracing For simple local development and debugging, you can export traces directly to a local file without requiring any external services. @@ -280,6 +298,7 @@ The example includes multiple configuration files for different observability pl | `config-phoenix.yml` | Phoenix | Tracing with Phoenix | | `config-phoenix-nested.yml` | Phoenix | Testing parent-child span tracking with nested tool calls | | `config-arize-ax.yml` | Arize AX | Hosted OTLP tracing to Arize AX (requires `ARIZE_*` environment variables) | +| `config-mlflow.yml` | MLflow | Local OTLP tracing to an MLflow tracking server | | `config-otel-file.yml` | File Export | Local file-based tracing for development and debugging | | `config-langfuse.yml` | Langfuse | Langfuse monitoring and analytics | | `config-langsmith.yml` | LangSmith | LangChain/LangGraph ecosystem integration | diff --git a/examples/observability/simple_calculator_observability/configs/config-mlflow.yml b/examples/observability/simple_calculator_observability/configs/config-mlflow.yml new file mode 100644 index 0000000000..7b63a06194 --- /dev/null +++ b/examples/observability/simple_calculator_observability/configs/config-mlflow.yml @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# MLflow: start a tracking server with a database backend store (required for trace ingestion), e.g. +# mlflow server --backend-store-uri sqlite:///mlflow.db --host 127.0.0.1 --port 5000 +# Traces are sent to ${MLFLOW_OTLP_ENDPOINT:-http://localhost:5000/v1/traces} and routed to experiment +# ${MLFLOW_EXPERIMENT_ID:-0}. See: https://mlflow.org/docs/latest/tracing/ + +general: + telemetry: + logging: + console: + _type: console + level: WARN + file: + _type: file + path: ./.tmp/nat_simple_calculator.log + level: DEBUG + tracing: + mlflow: + _type: mlflow + endpoint: ${MLFLOW_OTLP_ENDPOINT:-http://localhost:5000/v1/traces} + experiment_id: "${MLFLOW_EXPERIMENT_ID:-0}" + + front_end: + _type: fastapi + endpoints: + - path: /get_time + method: POST + description: Gets the current time + function_name: current_datetime + cors: + allow_origins: ['*'] + +function_groups: + calculator: + _type: calculator + +functions: + current_datetime: + _type: current_datetime + +llms: + nim_llm: + _type: nim + model_name: nvidia/nemotron-3-nano-30b-a3b + temperature: 0.0 + max_tokens: 1024 + + +workflow: + _type: react_agent + tool_names: [calculator, current_datetime] + llm_name: nim_llm + verbose: true + parse_agent_response_max_retries: 3 diff --git a/packages/nvidia_nat_opentelemetry/src/nat/plugins/opentelemetry/register.py b/packages/nvidia_nat_opentelemetry/src/nat/plugins/opentelemetry/register.py index 653f5ae5e3..cd6f062dec 100644 --- a/packages/nvidia_nat_opentelemetry/src/nat/plugins/opentelemetry/register.py +++ b/packages/nvidia_nat_opentelemetry/src/nat/plugins/opentelemetry/register.py @@ -233,6 +233,55 @@ async def galileo_telemetry_exporter(config: GalileoTelemetryExporter, builder: ) +# MLflow routes OTLP traces to an experiment via this header +# (matches mlflow.tracing.utils.otlp.MLFLOW_EXPERIMENT_ID_HEADER). +_MLFLOW_EXPERIMENT_ID_HEADER = "x-mlflow-experiment-id" + + +def _mlflow_experiment_headers(experiment_id: str) -> dict[str, str]: + """Build the OTLP header that routes traces to an MLflow experiment (matches MLflow's own ingestion).""" + return {_MLFLOW_EXPERIMENT_ID_HEADER: experiment_id} + + +class MLflowTelemetryExporter(BatchConfigMixin, TelemetryExporterBaseConfig, name="mlflow"): + """Export traces to an MLflow tracking server over OTLP/HTTP. + + MLflow 3.6+ ingests OpenTelemetry spans at ``/v1/traces`` and routes them to an + experiment via the ``x-mlflow-experiment-id`` header. Point ``endpoint`` at that path and set + ``experiment_id`` to the target experiment. The tracking server must run with a database backend + store for trace ingestion. + """ + + endpoint: str = Field( + default="http://localhost:5000/v1/traces", + description="The MLflow OTLP/HTTP trace ingestion endpoint (the tracking server URL plus /v1/traces).", + ) + experiment_id: str = Field( + default="0", + description="MLflow experiment ID that traces are routed to. If empty, uses the MLFLOW_EXPERIMENT_ID " + "environment variable, otherwise the default experiment (\"0\").", + ) + + +@register_telemetry_exporter(config_type=MLflowTelemetryExporter) +async def mlflow_telemetry_exporter(config: MLflowTelemetryExporter, builder: Builder): + """Create a telemetry exporter that sends OTLP traces to an MLflow tracking server.""" + + from nat.plugins.opentelemetry import OTLPSpanAdapterExporter + + experiment_id = (config.experiment_id or os.environ.get("MLFLOW_EXPERIMENT_ID") or "0").strip() + + yield OTLPSpanAdapterExporter( + endpoint=config.endpoint, + headers=_mlflow_experiment_headers(experiment_id), + batch_size=config.batch_size, + flush_interval=config.flush_interval, + max_queue_size=config.max_queue_size, + drop_on_overflow=config.drop_on_overflow, + shutdown_timeout=config.shutdown_timeout, + ) + + class ArizeAxTelemetryExporter(BatchConfigMixin, CollectorConfigMixin, TelemetryExporterBaseConfig, name="arize_ax"): """Export traces to Arize AX over OTLP. diff --git a/packages/nvidia_nat_opentelemetry/tests/observability/test_mlflow_telemetry_exporter.py b/packages/nvidia_nat_opentelemetry/tests/observability/test_mlflow_telemetry_exporter.py new file mode 100644 index 0000000000..bb6bb49eed --- /dev/null +++ b/packages/nvidia_nat_opentelemetry/tests/observability/test_mlflow_telemetry_exporter.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for the MLflow OTLP telemetry exporter routing header and config defaults.""" + +import nat.plugins.opentelemetry.register as otel_register + + +def test_mlflow_experiment_headers_match_mlflow_ingestion_contract(): + """The routing header key/value matches MLflow's OTLP ingestion (x-mlflow-experiment-id).""" + assert otel_register._mlflow_experiment_headers("42") == {"x-mlflow-experiment-id": "42"} + + +def test_mlflow_exporter_config_field_defaults(): + """Defaults target a local MLflow tracking server and the default experiment.""" + fields = otel_register.MLflowTelemetryExporter.model_fields + assert fields["endpoint"].default == "http://localhost:5000/v1/traces" + assert fields["experiment_id"].default == "0"