diff --git a/Snippets/Core/Core_10/OpenTelemetry/ExceptionRecording.cs b/Snippets/Core/Core_10/OpenTelemetry/ExceptionRecording.cs new file mode 100644 index 00000000000..b9901acca86 --- /dev/null +++ b/Snippets/Core/Core_10/OpenTelemetry/ExceptionRecording.cs @@ -0,0 +1,16 @@ +namespace Core.OpenTelemetry; + +using NServiceBus; + +public static class ExceptionRecording +{ + public static void ConfigureLogsOnly(EndpointConfiguration endpointConfiguration) + { + #region opentelemetry-exception-recording-logs + + var options = endpointConfiguration.Tracing(); + options.ExceptionRecordingMode = ExceptionRecordingMode.Logs; + + #endregion + } +} diff --git a/Snippets/Core/Core_10/OpenTelemetry/MetersConfiguration.cs b/Snippets/Core/Core_10/OpenTelemetry/MetersConfiguration.cs new file mode 100644 index 00000000000..b1a5080d660 --- /dev/null +++ b/Snippets/Core/Core_10/OpenTelemetry/MetersConfiguration.cs @@ -0,0 +1,16 @@ +namespace Core.OpenTelemetry; + +using NServiceBus; + +public static class MetersConfiguration +{ + public static void DisableExecutionResultTags(EndpointConfiguration endpointConfiguration) + { + #region opentelemetry-meters-disable-execution-result-tags + + var options = endpointConfiguration.Tracing(); + options.Meters.EmitExecutionResultTags = false; + + #endregion + } +} diff --git a/Snippets/Core/Core_10/OpenTelemetry/Metrics.cs b/Snippets/Core/Core_10/OpenTelemetry/Metrics.cs index 86f97b5e70c..63ee2bf355e 100644 --- a/Snippets/Core/Core_10/OpenTelemetry/Metrics.cs +++ b/Snippets/Core/Core_10/OpenTelemetry/Metrics.cs @@ -1,5 +1,6 @@ namespace Core.OpenTelemetry; +using System.Collections.Generic; using global::OpenTelemetry; using global::OpenTelemetry.Metrics; @@ -17,4 +18,25 @@ public static void EnableMetrics() #endregion } + + public static void FilterMetrics(MeterProviderBuilder metrics) + { + #region opentelemetry-metrics-filter-view + + // Only these NServiceBus metrics are collected; everything else the meter emits is dropped. + var enabledNServiceBusMetrics = new HashSet(StringComparer.Ordinal) + { + "nservicebus.messaging.deserialize_time", + "nservicebus.messaging.serialize_time", + }; + + metrics.AddMeter("NServiceBus.*") + .AddView(instrument => + instrument.Meter.Name.StartsWith("NServiceBus", StringComparison.Ordinal) + && !enabledNServiceBusMetrics.Contains(instrument.Name) + ? MetricStreamConfiguration.Drop + : null); + + #endregion + } } \ No newline at end of file diff --git a/Snippets/Core/Core_10/OpenTelemetry/TraceConfiguration.cs b/Snippets/Core/Core_10/OpenTelemetry/TraceConfiguration.cs new file mode 100644 index 00000000000..d8824c66947 --- /dev/null +++ b/Snippets/Core/Core_10/OpenTelemetry/TraceConfiguration.cs @@ -0,0 +1,102 @@ +namespace Core.OpenTelemetry; + +using global::OpenTelemetry; +using global::OpenTelemetry.Trace; +using NServiceBus; + +public static class TraceConfiguration +{ + public static void ConfigureSendTraceMode(EndpointConfiguration endpointConfiguration) + { + #region opentelemetry-trace-mode-send + + var options = endpointConfiguration.Tracing(); + // Default: ContinueExisting - the receiver continues the sender's trace. + // Set to StartNew to always start a new linked trace on the receiver. + options.SendTraceMode = TraceMode.StartNew; + + #endregion + } + + public static void ConfigurePublishTraceMode(EndpointConfiguration endpointConfiguration) + { + #region opentelemetry-trace-mode-publish + + var options = endpointConfiguration.Tracing(); + // Default: StartNew - subscribers start a new trace linked to the publish span. + // Set to ContinueExisting to continue the publisher's trace in the subscriber. + options.PublishTraceMode = TraceMode.ContinueExisting; + + #endregion + } + + public static void ConfigureDelayedTraceMode(EndpointConfiguration endpointConfiguration) + { + #region opentelemetry-trace-mode-delayed + + var options = endpointConfiguration.Tracing(); + // Default for all three is StartNew (new linked trace at delivery time). + // Set to ContinueExisting to continue the originating trace instead. + options.DelayedDelivery.SendOperationTraceMode = TraceMode.ContinueExisting; + options.DelayedDelivery.SagaTimeoutTraceMode = TraceMode.ContinueExisting; + options.Recoverability.DelayedRetryTraceMode = TraceMode.ContinueExisting; + + #endregion + } + + public static void SubscribeToAllTraceSources() + { + #region opentelemetry-enabletracing-all-sources + + var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource("NServiceBus.Core") + .AddSource("NServiceBus.Core.Handler") + .AddSource("NServiceBus.Core.Recoverability") + // ... Add exporters + .Build(); + + #endregion + } + + public static void EnableHandlerActivitySource() + { + #region opentelemetry-handler-activity-source-switch + + // Must be set before the endpoint starts. + AppContext.SetSwitch("NServiceBus.Core.OpenTelemetry.UseHandlerActivitySource", true); + + #endregion + } + + public static void UseDestinationInSpanNames(EndpointConfiguration endpointConfiguration) + { + #region opentelemetry-span-names-destination + + var options = endpointConfiguration.Tracing(); + options.UseMessageDestinationInSpanNames = true; + + #endregion + } + + public static void DisableDispatchingEvents(EndpointConfiguration endpointConfiguration) + { + #region opentelemetry-dispatching-events-disable + + var options = endpointConfiguration.Tracing(); + options.EmitMessageDispatchingEvents = false; + + #endregion + } + + public static void EnableDistributedContextPropagator() + { + #region opentelemetry-distributed-context-propagator-switch + + // Must be set before the endpoint starts. + // Opts in to W3C DistributedContextPropagator-based trace context propagation. + // This becomes the default in version 11. + AppContext.SetSwitch("NServiceBus.Core.OpenTelemetry.UseDistributedContextPropagator", true); + + #endregion + } +} diff --git a/nservicebus/operations/opentelemetry_logs_core_[8,).partial.md b/nservicebus/operations/opentelemetry_logs_core_[8,).partial.md index d67b9190d8a..1269582c4d6 100644 --- a/nservicebus/operations/opentelemetry_logs_core_[8,).partial.md +++ b/nservicebus/operations/opentelemetry_logs_core_[8,).partial.md @@ -1,3 +1,16 @@ ## Logging NServiceBus supports logging out of the box. To collect OpenTelemetry-compatible logging in NServiceBus endpoints, it's possible to configure the endpoint to connect traces and logging when using `Microsoft.Extensions.Logging` package. See the [_Connecting OpenTelemetry traces and logs_ sample](/samples/open-telemetry/logging) for more details. + +### Recoverability structured log properties + +Recoverability action log entries are emitted via `Microsoft.Extensions.Logging` and include named structured properties. When using a structured logging backend such as Serilog, Seq, or Application Insights, these properties are captured as queryable key-value pairs rather than text embedded in the message string: + +| Recoverability action | Structured properties | +|---|---| +| Immediate retry | `MessageId` | +| Delayed retry | `MessageId`, `Delay` | +| Move to error queue | `MessageId`, `ErrorQueue` | +| Discard | `MessageId`, `Reason` | + +For example, all messages moved to a specific error queue can be queried by filtering on `ErrorQueue`, or all delayed retries for a specific message can be found by filtering on `MessageId`, without needing to parse log message strings. diff --git a/nservicebus/operations/opentelemetry_metrics_core_[10,).partial.md b/nservicebus/operations/opentelemetry_metrics_core_[10,).partial.md index 1dc6bca2072..7acd49a992e 100644 --- a/nservicebus/operations/opentelemetry_metrics_core_[10,).partial.md +++ b/nservicebus/operations/opentelemetry_metrics_core_[10,).partial.md @@ -7,6 +7,20 @@ NServiceBus endpoints can be configured to expose metrics related to message pro snippet: opentelemetry-enablemeters +Because all metrics from a meter source are emitted by default, use an OpenTelemetry view to drop any that are not needed: + +snippet: opentelemetry-metrics-filter-view + +### Meter options + +#### execution.result tag + +Several metrics include an `execution.result` tag with a value of `"success"` or `"failure"`. This tag increases metric cardinality and the associated ingestion cost. To opt out, set `EmitExecutionResultTags` to `false`: + +snippet: opentelemetry-meters-disable-execution-result-tags + +The `execution.result` tag is emitted by default. Disabling it affects all metrics that carry it: `nservicebus.messaging.successes`, `nservicebus.messaging.failures`, `nservicebus.messaging.processing_time`, `nservicebus.messaging.critical_time`, `nservicebus.messaging.handler_time`, `nservicebus.messaging.deserialize_time`, `nservicebus.messaging.serialize_time`, and `nservicebus.sagas.fetch_time`. + ### Emitted meters Meter source `NServiceBus.Core.Pipeline.Incoming`: @@ -17,9 +31,17 @@ Meter source `NServiceBus.Core.Pipeline.Incoming`: - [`nservicebus.messaging.handler_time`](/monitoring/metrics/definitions.md#metrics-captured-handler-time) - The time the user handling code takes to handle a message - [`nservicebus.messaging.processing_time`](/monitoring/metrics/definitions.md#metrics-captured-processing-time) - The time the endpoint takes to process a message - [`nservicebus.messaging.critical_time`](/monitoring/metrics/definitions.md#metrics-captured-critical-time) - The time between when a message is sent and when it is fully processed +- `nservicebus.messaging.active_messages` (UpDownCounter) - Number of messages currently being processed by the endpoint. Tags: `nservicebus.queue`, `nservicebus.discriminator`, `nservicebus.enclosed_message_types` +- `nservicebus.messaging.deserialize_time` - The time in seconds for deserializing an incoming message. Tags: `nservicebus.queue`, `nservicebus.discriminator`, `nservicebus.enclosed_message_types`, `execution.result`, `error.type` +- `nservicebus.messaging.serialize_time` - The time in seconds for serializing an outgoing message. Tags: `nservicebus.message_type`, `execution.result`, `error.type` - [`nservicebus.recoverability.immediate`](/monitoring/metrics/definitions.md#metrics-captured-immediate-retries) - Total number of immediate retries requested - [`nservicebus.recoverability.delayed`](/monitoring/metrics/definitions.md#metrics-captured-delayed-retries) - Total number of delayed retries requested - [`nservicebus.recoverability.error`](/monitoring/metrics/definitions.md#metrics-captured-moved-to-error-queue) - Total number of messages sent to the error queue +- `nservicebus.sagas.fetch_time` - The time in seconds for loading saga data from the persister. Tags: `nservicebus.queue`, `nservicebus.discriminator`, `nservicebus.message_type`, `nservicebus.saga_type`, `execution.result`, `error.type` +- `nservicebus.outbox.duplicates` - Total number of messages deduplicated by the outbox. Tags: `nservicebus.queue`, `nservicebus.discriminator`, `nservicebus.message_type` +- `nservicebus.outbox.fetch_time` - The time in seconds for querying the outbox storage for deduplication. Tags: `nservicebus.queue`, `nservicebus.discriminator` +- `nservicebus.outbox.store_time` - The time in seconds for storing a message in the outbox storage. Tags: `nservicebus.queue`, `nservicebus.discriminator` +- `nservicebus.persistence.commit_time` - The time in seconds for completing the synchronized storage session. Tags: `nservicebus.queue`, `nservicebus.discriminator`, `nservicebus.message_type`, `nservicebus.message_handler_types` Starting NServiceBus V10.1 envelope unwrapping metrics are emitted as part of the `NServiceBus.Core.Pipeline.Incoming` source: @@ -38,4 +60,5 @@ Meter source `NServiceBus.Envelope.CloudEvents`: - [`nservicebus.envelope.cloud_events.received.invalid_message`](/monitoring/metrics/definitions.md#metrics-captured-envelope-handling-metrics-cloudevents-specific-metrics) - Total number of received messages not conforming to the specification - [`nservicebus.envelope.cloud_events.received.unexpected_version`](/monitoring/metrics/definitions.md#metrics-captured-envelope-handling-metrics-cloudevents-specific-metrics) - Total number of received messages with unexpected version field value + See the [OpenTelemetry samples](/samples/open-telemetry/) for instructions on how to send metric information to different tools. diff --git a/nservicebus/operations/opentelemetry_traces_core_[10,).partial.md b/nservicebus/operations/opentelemetry_traces_core_[10,).partial.md new file mode 100644 index 00000000000..45ac876b640 --- /dev/null +++ b/nservicebus/operations/opentelemetry_traces_core_[10,).partial.md @@ -0,0 +1,223 @@ +### Trace sources + +NServiceBus emits spans from three ActivitySources: + +| Source | Description | +|---|---| +| `NServiceBus.Core` | Pipeline spans: send, publish, process | +| `NServiceBus.Core.Handler` | Handler invocation spans (one per handler per message) | +| `NServiceBus.Core.Recoverability` | Recoverability action spans (immediate retry, delayed retry, move to error, discard) | + +Subscribe to the sources needed for the endpoint's observability requirements: + +snippet: opentelemetry-enabletracing-all-sources + +> [!NOTE] +> In version 10, `NServiceBus.Core.Handler` must be opted into via an AppContext switch before the endpoint starts: +> +> snippet: opentelemetry-handler-activity-source-switch +> +> Without this switch, handler spans are emitted from `NServiceBus.Core` instead. In version 11, `NServiceBus.Core.Handler` is the default and the switch is removed. + +Subscribing to `NServiceBus.Core.Handler` without subscribing to `NServiceBus.Core` suppresses handler spans - `Activity.Current` inside handlers and behaviors becomes the pipeline span. This enables a flattened trace view where handler work appears directly on the process span. + +### Span relationships + +#### Send operations + +A span is emitted for each message sent by an NServiceBus endpoint. When the message is received, a receive span is created as a child to the send span. + +```mermaid +flowchart LR; + subgraph SENDER + direction TB + NSBM1[NServiceBus Send span] + end + subgraph RECEIVER + direction TB + PRM1[NServiceBus Process span] + + end + NSBM1--child--> PRM1 +``` + +The default trace behavior for sends is to continue the existing trace: the receiver span is a child of the sender span. To override this for a specific message, use `SendOptions`: + +snippet: opentelemetry-sendoptions-start-new-trace + +This creates a new trace on the receiver and links the send and receive spans: + +```mermaid +flowchart LR; + subgraph SENDER + direction TB + NSBM1[NServiceBus Send span] + end + subgraph RECEIVER + direction TB + PRM1[NServiceBus Receive span] + + end + NSBM1-. link .-PRM1; +``` + +To change the default for all sends from an endpoint, set `SendTraceMode`: + +snippet: opentelemetry-trace-mode-send + +#### Publish operations + +A span is emitted for each message published by an NServiceBus endpoint. When the message is processed by a subscriber, a process span is created in a new trace, which is linked to the publish span. + + +```mermaid +flowchart LR; + subgraph PRODUCER + direction TB + NSBM1[NServiceBus Publish span] + end + subgraph CONSUMER + direction TB + PRM1[NServiceBus Process span] + + end + NSBM1-. link .-PRM1; +``` + +The default trace behavior for publishes is to start a new linked trace on each subscriber. To override this for a specific event, use `PublishOptions`: + +snippet: opentelemetry-publishoptions-continue-trace + +This continues the publisher's trace in the subscriber: + +```mermaid +flowchart LR; + subgraph PRODUCER + direction TB + NSBM1[NServiceBus Publish span] + end + subgraph CONSUMER + direction TB + PRM1[NServiceBus Process span] + + end + NSBM1--child--> PRM1 +``` + +To change the default for all publishes from an endpoint, set `PublishTraceMode`: + +snippet: opentelemetry-trace-mode-publish + +Per-message overrides (`StartNewTraceOnReceive`, `ContinueExistingTraceOnReceive`) always take precedence over the endpoint-level defaults. + +### Delayed messages + +When a message is delayed - whether by explicit delay (`SendOptions.DelayDeliveryWith`), saga timeout, or delayed retry - a new linked trace is started at delivery time by default. This reflects that the receive operation happens at a different moment in time than the send or retry decision. + +The trace behavior for each category of delayed message is configurable independently: + +snippet: opentelemetry-trace-mode-delayed + +| Option | Default | Applies to | +|---|---|---| +| `DelayedDelivery.SendOperationTraceMode` | `StartNew` | `SendOptions.DelayDeliveryWith` / `DoNotDeliverBefore` | +| `DelayedDelivery.SagaTimeoutTraceMode` | `StartNew` | Saga timeouts (`Saga.RequestTimeout`) | +| `Recoverability.DelayedRetryTraceMode` | `StartNew` | Delayed retries driven by recoverability policy | + +### Recoverability spans + +When a message cannot be processed successfully, NServiceBus emits a recoverability span from the `NServiceBus.Core.Recoverability` ActivitySource. The span carries a `nservicebus.recoverability_action` tag indicating the outcome: + +| Tag value | Meaning | +|---|---| +| `immediate_retry` | Message will be retried immediately | +| `delayed_retry` | Message will be retried after a delay | +| `move_to_error` | Message is moved to the error queue | +| `discard` | Message is discarded without further processing | + +Recoverability spans are children of the process span. To receive them, subscribe to the `NServiceBus.Core.Recoverability` ActivitySource. + +### Span names + +By default, NServiceBus uses generic operation names for spans: `"send message"`, `"process message"`, `"publish event"`, `"reply"`, etc. To include the destination or source queue in the span name - following the OpenTelemetry messaging semantic convention format `{operation} {destination}` - enable `UseMessageDestinationInSpanNames`: + +snippet: opentelemetry-span-names-destination + +With this enabled: + +| Operation | Default span name | With destination | +|---|---|---| +| Receive | `process message` | `process {receiveAddress}` | +| Send | `send message` | `send message {destination}` | +| Reply | `reply` | `reply {destination}` | +| Move to error | `move to error` | `move to {errorQueue}` | + +### Dispatching events + +When outgoing messages are dispatched during message processing, NServiceBus adds two span events to the incoming pipeline span: + +- `"Start dispatching"` - emitted before dispatch, includes a `message-count` event tag +- `"Finished dispatching"` - emitted after dispatch completes + +To suppress these events: + +snippet: opentelemetry-dispatching-events-disable + +These events are emitted by default. Disabling them reduces observability ingestion cost when dispatch timing is not needed. + +### Context propagation + +NServiceBus propagates the [W3C Trace Context](https://www.w3.org/TR/trace-context/) and [W3C Baggage](https://www.w3.org/TR/baggage/) headers between endpoints. In version 10, NServiceBus uses a custom propagator by default. To opt in to propagation via the built-in .NET `DistributedContextPropagator` instead, set the following AppContext switch before the endpoint starts: + +snippet: opentelemetry-distributed-context-propagator-switch + +This is the default behavior in version 11, where the custom propagator and the switch are removed. See the [version 10 to 11 upgrade guide](/nservicebus/upgrades/10to11/) for details on baggage serialization changes introduced with this switch. + +### Failed spans and the error.type tag + +When a span fails, NServiceBus sets the span status to `Error` and adds an `error.type` tag containing the fully qualified exception type name. This tag is set on the innermost span where the exception was thrown. + +### Exception recording + +When a span fails, NServiceBus must decide where to record the exception details - the type, message, and stack trace. The `ExceptionRecordingMode` property on `InstrumentationOptions` controls this behavior. + +> [!NOTE] +> The [OpenTelemetry semantic conventions for exceptions](https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-logs/) are moving away from span events toward log records as the canonical signal for exception details. NServiceBus follows this transition path. The `Logs` mode is the future direction; `SpanAndLogs` is provided for backward compatibility during the migration period. + +#### SpanAndLogs mode (default) + +In the default `SpanAndLogs` mode, NServiceBus records exception details in two places: + +- **As a span event** on the activity that failed. The event includes the `exception.type`, `exception.message`, and `exception.stacktrace` attributes. This makes exception details visible directly in trace backends such as Jaeger or Zipkin without needing to correlate with log output. +- **In log output**, when a recoverability decision is made. Log entries are written for immediate retries, delayed retries, moves to the error queue, and discards, and each includes the full exception. + +This mode preserves the behavior from earlier NServiceBus versions and is appropriate during a migration period, or when trace backends are the primary tool for investigating failures. It corresponds to the `logs/dup` value defined in the OpenTelemetry [transition guidance](https://opentelemetry.io/docs/specs/semconv/exceptions/). + +#### Logs mode + +To record exception details only via logging and not as span events, configure `ExceptionRecordingMode` to `Logs`: + +snippet: opentelemetry-exception-recording-logs + +In `Logs` mode, NServiceBus logs the exception exactly once, at the point where the exception was thrown - on the innermost span where the failure originated. Recoverability decisions (immediate retry, delayed retry, move to error queue, discard) are still logged, but those log entries contain only the action metadata such as message ID, destination queue, and retry delay. The exception details are not repeated. + +This is the mode recommended by the OpenTelemetry semantic conventions, which define exceptions as log records rather than span events. It is a good fit when: + +- Log aggregation (such as structured logging sent to Elasticsearch or Azure Monitor) is the primary tool for investigating failures. +- Span event storage is expensive or not supported in the observability backend being used. +- Teams prefer a single, authoritative log entry per failure rather than exception details appearing in both trace and log outputs. + +#### Environment variable override + +The exception recording mode can also be set via the [`OTEL_SEMCONV_EXCEPTION_SIGNAL_OPT_IN`](https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-logs/) environment variable, which is part of the standard OpenTelemetry transition mechanism for migrating from span events to log records. Because the environment variable takes precedence over any value configured in code, operators can drive the entire migration through deployment configuration - without requiring code changes at each step. For example, `logs/dup` can be set first to emit exceptions to both signals simultaneously, giving teams time to verify that their log aggregation pipeline captures exception details correctly before switching to `logs` to stop emitting span events entirely. + +This also gives ops teams independent control over observability behavior in each environment. If the application hardcodes `ExceptionRecordingMode.SpanAndLogs`, the `Logs` mode can be forced in production by setting the environment variable, without the need for a redeploy. + +| Environment variable value | Equivalent `ExceptionRecordingMode` | +|---|---| +| `logs` | `Logs` | +| `logs/dup` | `SpanAndLogs` | + +When the environment variable is not set, the value configured in code is used, defaulting to `SpanAndLogs` if not explicitly configured. + +See the [OpenTelemetry samples](/samples/open-telemetry/) for instructions on how to send trace information to different tools. diff --git a/nservicebus/operations/opentelemetry_traces_core_[9,).partial.md b/nservicebus/operations/opentelemetry_traces_core_[9,10).partial.md similarity index 100% rename from nservicebus/operations/opentelemetry_traces_core_[9,).partial.md rename to nservicebus/operations/opentelemetry_traces_core_[9,10).partial.md diff --git a/nservicebus/upgrades/10to11/index.md b/nservicebus/upgrades/10to11/index.md index c3c90900f02..6c63b24efc9 100644 --- a/nservicebus/upgrades/10to11/index.md +++ b/nservicebus/upgrades/10to11/index.md @@ -253,3 +253,139 @@ Or via MSBuild in the project file: > The legacy MD5-based host identifier algorithm and the `UseV2DeterministicGuid` AppContext switch will be removed in version 12. If an endpoint must keep a specific host identifier beyond version 11, configure the host identifier explicitly instead of relying on the legacy algorithm switch. For example, an endpoint can be configured with its existing host identifier to keep the value stable after the legacy algorithm is removed. See [Overriding the host identifier](/nservicebus/hosting/override-hostid.md). + +## OpenTelemetry + +### ActivitySources + +In version 11, NServiceBus emits spans from three ActivitySources: + +| Source | Spans | +|---|---| +| `NServiceBus.Core` | Pipeline spans (send, publish, process) | +| `NServiceBus.Core.Handler` | Handler invocation spans | +| `NServiceBus.Core.Recoverability` | Recoverability action spans | + +In version 10, handler spans were emitted from `NServiceBus.Core` by default, with `NServiceBus.Core.Handler` available as an opt-in via the `NServiceBus.Core.OpenTelemetry.UseHandlerActivitySource` AppContext switch. In version 11, handler spans are always emitted from `NServiceBus.Core.Handler`. + +Any OpenTelemetry configuration that only subscribes to `NServiceBus.Core` will no longer receive handler spans after upgrading. Update the tracer configuration to subscribe to all required sources: + +```csharp +Sdk.CreateTracerProviderBuilder() + .AddSource("NServiceBus.Core") + .AddSource("NServiceBus.Core.Handler") + .AddSource("NServiceBus.Core.Recoverability") + // ... + .Build(); +``` + +### Deprecated span attributes + +#### otel.status_code and otel.status_description + +Earlier versions of NServiceBus set `otel.status_code` and `otel.status_description` as explicit span attributes on failed spans, in addition to setting the span status via the OpenTelemetry API. These are NServiceBus-specific tags that predate reliable support for `Activity.SetStatus` in .NET. They are now redundant: the standard `Activity.SetStatus` call is the canonical way to convey span status, and exporters surface it correctly without these extra attributes. + +In version 10, these attributes are still emitted for backward compatibility. They will be removed in version 11. + +If dashboards, alerts, or queries rely on `otel.status_code` or `otel.status_description` span attributes set by NServiceBus, migrate to using the span status provided by the OpenTelemetry exporter before upgrading to version 11. + +#### exception.escaped + +The `exception.escaped` attribute on exception span events is deprecated in the [OpenTelemetry semantic conventions](https://opentelemetry.io/docs/specs/semconv/attributes-registry/exception/). The spec notes that it is no longer recommended to record exceptions that are handled and do not escape the scope of a span. + +In version 10, `exception.escaped` is still included in exception events for backward compatibility. It will be removed in version 11. + +#### `start_dispatch` and `end_dispatch` + +NServiceBus adds two span events to the incoming message pipeline span whenever outgoing messages are dispatched during message processing: + +- `"Start dispatching"` - emitted before the outgoing messages are handed to the transport, with a `message-count` event tag indicating how many messages are being dispatched. +- `"Finished dispatching"` - emitted after the dispatch completes. + +In version 10, these events are always emitted when OpenTelemetry instrumentation is enabled. In version 11, they are opt-out via the `EmitMessageDispatchingEvents` property on `InstrumentationOptions`: + +```csharp +var options = endpointConfiguration.Tracing(); +options.EmitMessageDispatchingEvents = false; +``` + +The default remains `true` for backward compatibility. Consider disabling these events when they add no diagnostic value in order to reduce observability ingestion cost. + +### Context propagation + +In version 11, NServiceBus propagates the [W3C Trace Context](https://www.w3.org/TR/trace-context/) and [W3C Baggage](https://www.w3.org/TR/baggage/) using the built-in .NET `DistributedContextPropagator` instead of the custom propagation logic used in version 10. This aligns the on-the-wire format with the W3C specifications and improves interoperability with standard OpenTelemetry tooling and non-NServiceBus systems that participate in the same trace. See [OpenTelemetry](/nservicebus/operations/opentelemetry.md) for an overview of the feature. + +#### Trace correlation is unaffected + +The `traceparent` and `tracestate` headers continue to be emitted in the W3C format. Distributed traces still correlate correctly between version 10 and version 11 endpoints in both directions, so upgrading does not break trace continuity. + +#### Baggage serialization change + +The change affects how the `baggage` header is serialized on the wire: + +- Version 10 emitted a compact form with no optional whitespace (`key1=value1,key2=value2`) and percent-encoded baggage values aggressively. +- Version 11 emits the W3C form with optional whitespace around the delimiters (`key1 = value1, key2 = value2`) and percent-encodes only the characters that are structurally significant (such as `,`, `;`, and `%`). + +Both versions decode percent-encoding when reading, so a baggage value written by one version is generally decoded correctly by the other - with the exception described below. + +#### Mixed-version incompatibility + +> [!WARNING] +> When [baggage](https://www.w3.org/TR/baggage/) is used, a **version 11 endpoint sending to a version 10 endpoint corrupts every baggage value by prepending a single space**. Version 11 writes baggage using the W3C optional-whitespace format (`key = value`), and the version 10 reader does not trim that whitespace from the value when parsing. The opposite direction (a version 10 endpoint sending to a version 11 endpoint) is not affected. + +This only matters when both of the following are true: + +- The application adds baggage to activities. Baggage is opt-in; endpoints that do not use it are unaffected, and `traceparent`/`tracestate` correlation works regardless. +- Version 10 and version 11 endpoints exchange messages during a rolling upgrade. + +To avoid the problem, upgrade message **receivers before senders** so that no version 10 endpoint receives baggage produced by a version 11 endpoint. + +#### Baggage + +##### Whitespace in values + +Contrary to version 10, version 11 of NServiceBus does not preserve leading or trailing whitespace in a baggage value. The W3C propagator treats such whitespace as insignificant optional whitespace and trims it when reading, whereas version 10 percent-encoded it. For example, a value of `" tenant"` is read back as `"tenant"`. This applies even when both endpoints run version 11. If exact leading or trailing whitespace must be retained, encode it into the value (for example, percent-encode it) before adding it to baggage and decode it after reading. + +##### Empty values are no longer propagated + +Version 10 preserved a baggage item that had an empty value: a header such as `key1=value1,key3=` was read back with `key3` present and set to an empty string. Version 11 discards baggage members that have an empty value when reading, so `key3` is not added to the activity at all. The propagator also stops parsing at the first empty-valued member, so members listed after it can be dropped as well. + +This is the behavior of the underlying .NET `DistributedContextPropagator`, which on this point is stricter than the [W3C Baggage](https://www.w3.org/TR/baggage/) specification (the specification permits empty values). Avoid relying on empty or null baggage values; if an item only needs to signal presence, give it a non-empty value such as `true` or `1`. Note that a `null` and an empty baggage value are indistinguishable on the wire - both serialize to `key=` - so neither survives. + +#### Trace state must conform to the W3C format + +Version 10 copied the `tracestate` value onto outgoing messages verbatim, without validation. Version 11 validates `tracestate` against the [W3C Trace Context](https://www.w3.org/TR/trace-context/#tracestate-header) format and drops any content that does not conform. As a result, a non-conformant trace state set on an ambient activity - for example, free-form text such as `my custom state`, or a member whose key contains uppercase letters - is no longer propagated to the message spans. + +To retain custom trace state, ensure it is a comma-separated list of `key=value` members with lowercase keys, for example `vendorkey=vendorvalue`. Values may contain mixed case; only keys are restricted to lowercase letters, digits, and `_`, `-`, `*`, `/`, `@`. + +### Metrics + +#### New performance metrics + +Version 11 adds six new histograms to the `NServiceBus.Core.Pipeline.Incoming` meter source: + +| Metric | Description | +|---|---| +| `nservicebus.messaging.deserialize_time` | Time to deserialize an incoming message | +| `nservicebus.messaging.serialize_time` | Time to serialize an outgoing message | +| `nservicebus.sagas.fetch_time` | Time to load saga data from the persister | +| `nservicebus.outbox.fetch_time` | Time to query outbox storage for deduplication | +| `nservicebus.outbox.store_time` | Time to store a message in outbox storage | +| `nservicebus.persistence.commit_time` | Time to complete the synchronized storage session | + +These metrics are emitted automatically when the meter source is subscribed. No additional configuration is required. See [OpenTelemetry metrics](/nservicebus/operations/opentelemetry.md#meters-emitted-meters) for the full tag reference. + +#### Meter source version + +The `NServiceBus.Core.Pipeline.Incoming` meter source version has been updated to `0.4.0`. If any monitoring configuration references this version string explicitly, update it accordingly. + +#### execution.result tag is now opt-out + +The `execution.result` tag, which carries `"success"` or `"failure"`, is now opt-out. It is emitted on: `nservicebus.messaging.successes`, `nservicebus.messaging.failures`, `nservicebus.messaging.processing_time`, `nservicebus.messaging.critical_time`, `nservicebus.messaging.handler_time`, `nservicebus.messaging.deserialize_time`, `nservicebus.messaging.serialize_time`, and `nservicebus.sagas.fetch_time`. It remains enabled by default for backward compatibility. To disable it: + +```csharp +var options = endpointConfiguration.Tracing(); +options.Meters.EmitExecutionResultTags = false; +``` + +Disabling the tag reduces metric cardinality and ingestion cost when the success/failure breakdown is not needed at the metric level, for example when that information is already available through spans or logs. \ No newline at end of file