diff --git a/Directory.Packages.props b/Directory.Packages.props index 1f08f661a..413ed032b 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -25,6 +25,7 @@ + diff --git a/samples/Directory.Packages.props b/samples/Directory.Packages.props index e612c1390..3d218cd80 100644 --- a/samples/Directory.Packages.props +++ b/samples/Directory.Packages.props @@ -41,7 +41,7 @@ - + diff --git a/samples/distributed-tracing/v2/DistributedTracingSample/README.md b/samples/distributed-tracing/v2/DistributedTracingSample/README.md index 855928e8d..dd41f1d53 100644 --- a/samples/distributed-tracing/v2/DistributedTracingSample/README.md +++ b/samples/distributed-tracing/v2/DistributedTracingSample/README.md @@ -28,6 +28,8 @@ You will also need to specify an Application Insights resource in the environmen Next, you'll need to copy the connection string or instrumentation key for that resource to add in the environment variables. We recommend adding the `APPLICATIONINSIGHTS_CONNECTION_STRING` environment variable, but you can also add `APPINSIGHTS_INSTRUMENTATIONKEY`. If you are running your app locally, then add this environment variable in `local.settings.json`. If you are running the app in Azure, then add it under `Configuration` as an application setting. +If the Application Insights resource has local authentication disabled, also configure `APPLICATIONINSIGHTS_AUTHENTICATION_STRING`. Use `Authorization=AAD` for the function app's system-assigned managed identity, or `Authorization=AAD;ClientId=` for a user-assigned managed identity. The selected identity needs the `Monitoring Metrics Publisher` role on the Application Insights resource. Microsoft Entra authentication for Application Insights isn't supported by the Functions host during local development. + #### Run the sample The sample shows how Distributed Tracing V2 improves the observability of some flagship Durable Functions patterns. diff --git a/src/WebJobs.Extensions.DurableTask/Correlation/HostForwardingTelemetryChannel.cs b/src/WebJobs.Extensions.DurableTask/Correlation/HostForwardingTelemetryChannel.cs new file mode 100644 index 000000000..3130de72a --- /dev/null +++ b/src/WebJobs.Extensions.DurableTask/Correlation/HostForwardingTelemetryChannel.cs @@ -0,0 +1,108 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.Extensibility; + +namespace Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation +{ + /// + /// A telemetry channel that forwards telemetry to a channel owned by the Functions host. + /// + /// + /// + /// Durable distributed tracing keeps its own so that its + /// telemetry initializers, processors and modules stay isolated from the host pipeline. That + /// private configuration has no Microsoft Entra credential of its own, so when the Application + /// Insights component has local authentication disabled its telemetry is rejected at ingestion. + /// + /// + /// The host's channel already carries the credential and the Entra ingestion endpoint, so + /// forwarding to it reuses the host's authentication without Durable ever creating, reading or + /// referencing an Azure.Core.TokenCredential. That matters because the Application + /// Insights SDK resolves that type in the host load context, and a credential created in the + /// function app load context is rejected. + /// + /// + /// This channel deliberately does not own the wrapped channel. It never disposes it, and it + /// keeps and local so that + /// configuring the Durable configuration cannot mutate host state. + /// + /// + internal sealed class HostForwardingTelemetryChannel : ITelemetryChannel, IAsyncFlushable + { + private readonly ITelemetryChannel hostChannel; + + /// + /// Initializes a new instance of the class. + /// + /// The host-owned channel to forward telemetry to. + public HostForwardingTelemetryChannel(ITelemetryChannel hostChannel) + { + this.hostChannel = hostChannel ?? throw new ArgumentNullException(nameof(hostChannel)); + } + + /// + /// Gets or sets a value indicating whether this channel is in developer mode. + /// + /// + /// Kept local. Forwarding this would let the Durable configuration change how the host + /// batches its own telemetry. + /// + public bool? DeveloperMode { get; set; } + + /// + /// Gets or sets the channel's endpoint. + /// + /// + /// Kept local. The host channel already points at the Entra ingestion endpoint, and the + /// Durable configuration has no credential of its own, so propagating this value would + /// downgrade the host to the non-Entra endpoint. + /// + public string EndpointAddress { get; set; } + + /// + /// Gets the host-owned channel that telemetry is forwarded to. + /// + internal ITelemetryChannel HostChannel => this.hostChannel; + + /// + /// Forwards a telemetry item to the host channel, which transmits it using the host's + /// credential and endpoint. + /// + /// The telemetry item to send. + public void Send(ITelemetry item) => this.hostChannel.Send(item); + + /// + /// Flushes the host channel. + /// + public void Flush() => this.hostChannel.Flush(); + + /// + /// Asynchronously flushes the host channel when supported. + /// + /// The cancellation token. + /// The host channel's asynchronous flush result, or false after a synchronous flush. + public Task FlushAsync(CancellationToken cancellationToken) + { + if (this.hostChannel is IAsyncFlushable asyncFlushable) + { + return asyncFlushable.FlushAsync(cancellationToken); + } + + this.hostChannel.Flush(); + return Task.FromResult(false); + } + + /// + /// Does nothing. The host owns the wrapped channel's lifetime. + /// + public void Dispose() + { + // Intentionally empty: disposing the host channel here would stop host telemetry. + } + } +} diff --git a/src/WebJobs.Extensions.DurableTask/Correlation/TelemetryActivator.cs b/src/WebJobs.Extensions.DurableTask/Correlation/TelemetryActivator.cs index 946c0f60f..d0f2b5722 100644 --- a/src/WebJobs.Extensions.DurableTask/Correlation/TelemetryActivator.cs +++ b/src/WebJobs.Extensions.DurableTask/Correlation/TelemetryActivator.cs @@ -1,16 +1,20 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See LICENSE in the project root for license information. using System; +using System.Security.Authentication; using System.Threading.Tasks; +using Azure.Identity; using DurableTask.ApplicationInsights; using DurableTask.Core; using DurableTask.Core.Settings; using Microsoft.ApplicationInsights; using Microsoft.ApplicationInsights.Channel; using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.Azure.WebJobs.Logging.ApplicationInsights; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using ApplicationInsightsTokenCredentialOptions = Microsoft.Azure.WebJobs.Logging.ApplicationInsights.TokenCredentialOptions; namespace Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation { @@ -21,6 +25,7 @@ public class TelemetryActivator : ITelemetryActivator, IAsyncDisposable, IDispos { private readonly DurableTaskOptions options; private readonly INameResolver nameResolver; + private readonly TelemetryConfiguration hostTelemetryConfiguration; private EndToEndTraceHelper endToEndTraceHelper; private TelemetryClient telemetryClient; @@ -30,9 +35,24 @@ public class TelemetryActivator : ITelemetryActivator, IAsyncDisposable, IDispos /// DurableTask options. /// Name resolver used for environment variables. public TelemetryActivator(IOptions options, INameResolver nameResolver) + : this(options, nameResolver, hostTelemetryConfiguration: null) + { + } + + /// + /// Constructor for initializing Distributed Tracing with the host telemetry configuration. + /// + /// DurableTask options. + /// Name resolver used for environment variables. + /// Application Insights configuration owned by the host. + public TelemetryActivator( + IOptions options, + INameResolver nameResolver, + TelemetryConfiguration hostTelemetryConfiguration) { this.options = options.Value; this.nameResolver = nameResolver; + this.hostTelemetryConfiguration = hostTelemetryConfiguration; } /// @@ -45,20 +65,24 @@ public TelemetryActivator(IOptions options, INameResolver na internal IAsyncDisposable WebJobsTelemetryModule { get; set; } + /// + /// Gets the private configuration Durable builds for its own telemetry. Exposed so tests can + /// assert which credential was applied without reaching into Application Insights internals. + /// + internal TelemetryConfiguration TelemetryConfiguration { get; private set; } + /// - public ValueTask DisposeAsync() + public async ValueTask DisposeAsync() { if (this.TelemetryModule != null) { - this.TelemetryModule.DisposeAsync(); + await this.TelemetryModule.DisposeAsync(); } if (this.WebJobsTelemetryModule != null) { - this.WebJobsTelemetryModule.DisposeAsync(); + await this.WebJobsTelemetryModule.DisposeAsync(); } - - return default; } /// @@ -90,6 +114,7 @@ public void Initialize(ILogger logger) } TelemetryConfiguration telemetryConfiguration = this.SetupTelemetryConfiguration(); + this.TelemetryConfiguration = telemetryConfiguration; if (this.options.Tracing.Version == Options.DurableDistributedTracingVersion.V2) { @@ -190,6 +215,7 @@ private TelemetryConfiguration SetupTelemetryConfiguration() string resolvedInstrumentationKey = this.nameResolver.Resolve("APPINSIGHTS_INSTRUMENTATIONKEY"); string resolvedConnectionString = this.nameResolver.Resolve("APPLICATIONINSIGHTS_CONNECTION_STRING"); + string resolvedAuthenticationString = this.nameResolver.Resolve("APPLICATIONINSIGHTS_AUTHENTICATION_STRING"); bool instrumentationKeyProvided = !string.IsNullOrEmpty(resolvedInstrumentationKey); bool connectionStringProvided = !string.IsNullOrEmpty(resolvedConnectionString); @@ -238,7 +264,101 @@ private TelemetryConfiguration SetupTelemetryConfiguration() config.ConnectionString = resolvedConnectionString; } + if (!string.IsNullOrWhiteSpace(resolvedAuthenticationString)) + { + try + { + ApplicationInsightsTokenCredentialOptions tokenCredentialOptions = + ApplicationInsightsTokenCredentialOptions.ParseAuthenticationString(resolvedAuthenticationString); + bool userAssignedIdentity = tokenCredentialOptions.ClientId != null; + bool reusedHostChannel = ApplyEntraAuthentication( + config, + this.hostTelemetryConfiguration, + tokenCredentialOptions, + preserveExistingChannel: this.OnSend != null); + if (this.hostTelemetryConfiguration != null && !reusedHostChannel && this.OnSend == null) + { + this.LogTracingWarning( + "The Application Insights telemetry channel owned by the Functions host could not be read, so Durable distributed tracing created its own managed identity credential. If Durable spans are missing, the host and the function app are likely loading different Azure.Core versions."); + } + + this.endToEndTraceHelper.ExtensionInformationalEvent( + hubName: this.options.HubName, + functionName: string.Empty, + instanceId: string.Empty, + message: + "Microsoft Entra authentication enabled for Durable distributed tracing using the " + + $"{(userAssignedIdentity ? "user-assigned" : "system-assigned")} managed identity.", + writeToUserLogs: true); + } + catch (AuthenticationException) + { + this.LogTracingWarning( + "APPLICATIONINSIGHTS_AUTHENTICATION_STRING is invalid and will not be used for Durable Functions distributed tracing."); + } + catch (FormatException) + { + this.LogTracingWarning( + "APPLICATIONINSIGHTS_AUTHENTICATION_STRING is invalid and will not be used for Durable Functions distributed tracing."); + } + catch (ArgumentException) + { + this.LogTracingWarning( + "APPLICATIONINSIGHTS_AUTHENTICATION_STRING could not be applied and will not be used for Durable Functions distributed tracing."); + } + } + return config; } + + /// + /// Enables Microsoft Entra authenticated ingestion for the private Durable telemetry + /// configuration. + /// + /// + /// When the Functions host exposes its telemetry configuration, Durable forwards telemetry + /// to the host's channel, which already holds the host's credential and Entra ingestion + /// endpoint. Durable therefore never creates an Azure.Core.TokenCredential, which is + /// important because the Application Insights SDK resolves that type in the host load + /// context and rejects a credential created in the function app load context. + /// Outside the Functions host there is no channel to reuse, so Durable falls back to + /// creating its own managed identity credential. + /// + /// The private Durable telemetry configuration. + /// The host telemetry configuration, if available. + /// The parsed authentication string. + /// + /// True when the configuration already has a channel that must not be replaced, such as the + /// test channel installed by . + /// + /// True when the host channel was reused. + internal static bool ApplyEntraAuthentication( + TelemetryConfiguration durableConfiguration, + TelemetryConfiguration hostConfiguration, + ApplicationInsightsTokenCredentialOptions tokenCredentialOptions, + bool preserveExistingChannel = false) + { + ITelemetryChannel hostChannel = hostConfiguration?.TelemetryChannel; + if (hostChannel != null && !preserveExistingChannel) + { + durableConfiguration.TelemetryChannel = new HostForwardingTelemetryChannel(hostChannel); + return true; + } + + ManagedIdentityId managedIdentityId = tokenCredentialOptions.ClientId != null + ? ManagedIdentityId.FromUserAssignedClientId(tokenCredentialOptions.ClientId) + : ManagedIdentityId.SystemAssigned; + durableConfiguration.SetAzureTokenCredential(new ManagedIdentityCredential(managedIdentityId)); + return false; + } + + private void LogTracingWarning(string message) + { + this.endToEndTraceHelper.ExtensionWarningEvent( + hubName: this.options.HubName, + functionName: string.Empty, + instanceId: string.Empty, + message: message); + } } } diff --git a/src/WebJobs.Extensions.DurableTask/WebJobs.Extensions.DurableTask.csproj b/src/WebJobs.Extensions.DurableTask/WebJobs.Extensions.DurableTask.csproj index 472958bf9..033b5b7e3 100644 --- a/src/WebJobs.Extensions.DurableTask/WebJobs.Extensions.DurableTask.csproj +++ b/src/WebJobs.Extensions.DurableTask/WebJobs.Extensions.DurableTask.csproj @@ -58,6 +58,7 @@ + diff --git a/test/Directory.Packages.props b/test/Directory.Packages.props index 221a7128b..2f8090771 100644 --- a/test/Directory.Packages.props +++ b/test/Directory.Packages.props @@ -9,7 +9,6 @@ - @@ -33,7 +32,7 @@ - + diff --git a/test/FunctionsV2/Helpers/PlatformSpecificHelpers.FunctionsV2.cs b/test/FunctionsV2/Helpers/PlatformSpecificHelpers.FunctionsV2.cs index a65b88c04..e365e3267 100644 --- a/test/FunctionsV2/Helpers/PlatformSpecificHelpers.FunctionsV2.cs +++ b/test/FunctionsV2/Helpers/PlatformSpecificHelpers.FunctionsV2.cs @@ -7,6 +7,7 @@ using System.Reflection; using System.Threading.Tasks; using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.Extensibility; using Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation; using Microsoft.Azure.WebJobs.Extensions.DurableTask.Options; using Microsoft.Azure.WebJobs.Extensions.DurableTask.Storage; @@ -40,7 +41,8 @@ public static ITestHost CreateJobHost( Action onSend, bool addDurableClientFactory, ITypeLocator typeLocator, - Action configureScaleOptions = null) + Action configureScaleOptions = null, + TelemetryConfiguration hostTelemetryConfiguration = null) { // Unless specified, use table partition management for tests as it makes the task hubs start up faster. // These tests run on a single task hub workers, so they don't test partition management anyways, and that is tested @@ -87,16 +89,22 @@ public static ITestHost CreateJobHost( serviceCollection.AddSingleton(serializerSettingsFactory); } + if (hostTelemetryConfiguration != null) + { + // Mirrors the Application Insights configuration the Functions host registers, + // so DI can select the host-aware TelemetryActivator constructor. + serviceCollection.AddSingleton(hostTelemetryConfiguration); + } + if (onSend != null) { serviceCollection.AddSingleton(serviceProvider => { - var durableTaskOptions = serviceProvider.GetService>(); - var nameResolver = serviceProvider.GetService(); - var telemetryActivator = new TelemetryActivator(durableTaskOptions, nameResolver) - { - OnSend = onSend, - }; + // Let the container pick the constructor exactly as the production + // AddSingleton() registration does. + var telemetryActivator = + ActivatorUtilities.CreateInstance(serviceProvider); + telemetryActivator.OnSend = onSend; return telemetryActivator; }); } diff --git a/test/FunctionsV2/Helpers/TelemetryActivatorTests.cs b/test/FunctionsV2/Helpers/TelemetryActivatorTests.cs new file mode 100644 index 000000000..7a1aaa04f --- /dev/null +++ b/test/FunctionsV2/Helpers/TelemetryActivatorTests.cs @@ -0,0 +1,372 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Azure.Identity; +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.DataContracts; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.Azure.WebJobs; +using Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Moq; +using Xunit; +using ApplicationInsightsTokenCredentialOptions = Microsoft.Azure.WebJobs.Logging.ApplicationInsights.TokenCredentialOptions; + +namespace Microsoft.Azure.WebJobs.Extensions.DurableTask.Tests +{ + public class TelemetryActivatorTests + { + private const string SystemAssigned = "Authorization=AAD"; + private const string UserAssigned = "Authorization=AAD;ClientId=00000000-0000-0000-0000-000000000001"; + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public void ApplyEntraAuthentication_ReusesHostChannel() + { + using TelemetryConfiguration hostConfiguration = TelemetryConfiguration.CreateDefault(); + using TelemetryConfiguration durableConfiguration = TelemetryConfiguration.CreateDefault(); + var hostChannel = new NoOpTelemetryChannel(); + hostConfiguration.TelemetryChannel = hostChannel; + + bool reusedHostChannel = Apply(durableConfiguration, hostConfiguration, SystemAssigned); + + Assert.True(reusedHostChannel); + var forwarding = Assert.IsType(durableConfiguration.TelemetryChannel); + Assert.Same(hostChannel, forwarding.HostChannel); + } + + /// + /// The whole point of the design: telemetry tracked on the private Durable configuration has + /// to physically leave through the host channel, because that is what holds the credential. + /// + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public void ApplyEntraAuthentication_SendsTelemetryThroughHostChannel() + { + var forwarded = new List(); + using TelemetryConfiguration hostConfiguration = TelemetryConfiguration.CreateDefault(); + using TelemetryConfiguration durableConfiguration = TelemetryConfiguration.CreateDefault(); + hostConfiguration.TelemetryChannel = new NoOpTelemetryChannel { OnSend = forwarded.Add }; + + Apply(durableConfiguration, hostConfiguration, SystemAssigned); + + new ApplicationInsights.TelemetryClient(durableConfiguration) + .TrackRequest(new RequestTelemetry("durable-span", DateTimeOffset.UtcNow, TimeSpan.Zero, "200", true)); + + ITelemetry item = Assert.Single(forwarded); + Assert.Equal("durable-span", Assert.IsType(item).Name); + } + + /// + /// Durable must never clear the credential the host installed on its own configuration. + /// + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public void ApplyEntraAuthentication_LeavesHostCredentialIntact() + { + using TelemetryConfiguration hostConfiguration = TelemetryConfiguration.CreateDefault(); + using TelemetryConfiguration durableConfiguration = TelemetryConfiguration.CreateDefault(); + var credential = new ManagedIdentityCredential(ManagedIdentityId.SystemAssigned); + hostConfiguration.TelemetryChannel = new NoOpTelemetryChannel(); + hostConfiguration.SetAzureTokenCredential(credential); + + Apply(durableConfiguration, hostConfiguration, SystemAssigned); + + Assert.Same(credential, GetCredential(hostConfiguration)); + Assert.Null(GetCredential(durableConfiguration)); + } + + /// + /// The Durable configuration has no credential, so any endpoint it computes is the non-Entra + /// one. Letting that reach the host channel would break host telemetry. + /// + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public void ApplyEntraAuthentication_DoesNotOverwriteHostChannelEndpoint() + { + using TelemetryConfiguration hostConfiguration = TelemetryConfiguration.CreateDefault(); + using TelemetryConfiguration durableConfiguration = TelemetryConfiguration.CreateDefault(); + hostConfiguration.TelemetryChannel = new NoOpTelemetryChannel + { + EndpointAddress = "https://host.example.com/v2.1/track", + }; + + Apply(durableConfiguration, hostConfiguration, SystemAssigned); + durableConfiguration.ConnectionString = + "InstrumentationKey=11111111-2222-3333-4444-555555555555;IngestionEndpoint=https://durable.example.com/"; + + Assert.Equal("https://host.example.com/v2.1/track", hostConfiguration.TelemetryChannel.EndpointAddress); + } + + /// + /// The host owns the channel's lifetime. Disposing the Durable configuration must not stop + /// host telemetry. + /// + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public void ApplyEntraAuthentication_DoesNotDisposeHostChannel() + { + int disposeCalls = 0; + var sent = new List(); + using TelemetryConfiguration hostConfiguration = TelemetryConfiguration.CreateDefault(); + TelemetryConfiguration durableConfiguration = TelemetryConfiguration.CreateDefault(); + hostConfiguration.TelemetryChannel = new NoOpTelemetryChannel + { + OnSend = sent.Add, + OnDispose = () => disposeCalls++, + }; + + Apply(durableConfiguration, hostConfiguration, SystemAssigned); + durableConfiguration.TelemetryChannel.Dispose(); + durableConfiguration.Dispose(); + + Assert.Equal(0, disposeCalls); + hostConfiguration.TelemetryChannel.Send(new RequestTelemetry()); + Assert.Single(sent); + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task HostForwardingTelemetryChannel_FlushAsyncForwardsToAsyncHostChannelExactlyOnce() + { + using var cancellationSource = new CancellationTokenSource(); + var hostChannel = new AsyncTelemetryChannel(flushResult: true); + using TelemetryConfiguration durableConfiguration = TelemetryConfiguration.CreateDefault(); + durableConfiguration.TelemetryChannel = new HostForwardingTelemetryChannel(hostChannel); + var telemetryClient = new TelemetryClient(durableConfiguration); + + bool result = await telemetryClient.FlushAsync(cancellationSource.Token); + + Assert.True(result); + Assert.Equal(1, hostChannel.AsyncFlushCalls); + Assert.Equal(0, hostChannel.FlushCalls); + Assert.Equal(cancellationSource.Token, hostChannel.LastCancellationToken); + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task HostForwardingTelemetryChannel_FlushAsyncFallsBackToSyncHostChannelExactlyOnce() + { + int flushCalls = 0; + var hostChannel = new NoOpTelemetryChannel { OnFlush = () => flushCalls++ }; + using TelemetryConfiguration durableConfiguration = TelemetryConfiguration.CreateDefault(); + durableConfiguration.TelemetryChannel = new HostForwardingTelemetryChannel(hostChannel); + var telemetryClient = new TelemetryClient(durableConfiguration); + + bool result = await telemetryClient.FlushAsync(CancellationToken.None); + + Assert.False(result); + Assert.Equal(1, flushCalls); + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task DisposeAsync_AwaitsTelemetryModulesSequentially() + { + var firstModule = new ControlledAsyncDisposable(); + var secondModule = new ControlledAsyncDisposable(); + var activator = new TelemetryActivator( + Microsoft.Extensions.Options.Options.Create(new DurableTaskOptions()), + Mock.Of()) + { + TelemetryModule = firstModule, + WebJobsTelemetryModule = secondModule, + }; + + ValueTask disposal = activator.DisposeAsync(); + + Assert.False(disposal.IsCompleted); + Assert.Equal(1, firstModule.DisposeCalls); + Assert.Equal(0, secondModule.DisposeCalls); + + firstModule.CompleteDisposal(); + await secondModule.DisposalStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.False(disposal.IsCompleted); + Assert.Equal(1, secondModule.DisposeCalls); + + secondModule.CompleteDisposal(); + await disposal; + } + + /// + /// The OnSend test hook installs its own channel. Forwarding would silently swallow it. + /// + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public void ApplyEntraAuthentication_PreservesExistingTestChannel() + { + using TelemetryConfiguration hostConfiguration = TelemetryConfiguration.CreateDefault(); + using TelemetryConfiguration durableConfiguration = TelemetryConfiguration.CreateDefault(); + var testChannel = new NoOpTelemetryChannel(); + durableConfiguration.TelemetryChannel = testChannel; + hostConfiguration.TelemetryChannel = new NoOpTelemetryChannel(); + + bool reusedHostChannel = TelemetryActivator.ApplyEntraAuthentication( + durableConfiguration, + hostConfiguration, + ApplicationInsightsTokenCredentialOptions.ParseAuthenticationString(SystemAssigned), + preserveExistingChannel: true); + + Assert.False(reusedHostChannel); + Assert.Same(testChannel, durableConfiguration.TelemetryChannel); + } + + /// + /// Outside the Functions host there is no channel to reuse, so Durable still has to + /// authenticate on its own. + /// + [Theory] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + [InlineData(SystemAssigned)] + [InlineData(UserAssigned)] + public void ApplyEntraAuthentication_WithoutHostConfiguration_UsesManagedIdentity(string authenticationString) + { + using TelemetryConfiguration durableConfiguration = TelemetryConfiguration.CreateDefault(); + + bool reusedHostChannel = Apply(durableConfiguration, hostConfiguration: null, authenticationString); + + Assert.False(reusedHostChannel); + Assert.IsType(GetCredential(durableConfiguration)); + Assert.IsNotType(durableConfiguration.TelemetryChannel); + } + + /// + /// A host configuration can exist without a channel. The caller relies on the return value + /// to warn instead of degrading silently. + /// + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public void ApplyEntraAuthentication_WithHostConfigurationLackingChannel_ReportsFallback() + { + using TelemetryConfiguration hostConfiguration = TelemetryConfiguration.CreateDefault(); + using TelemetryConfiguration durableConfiguration = TelemetryConfiguration.CreateDefault(); + hostConfiguration.TelemetryChannel = null; + + bool reusedHostChannel = Apply(durableConfiguration, hostConfiguration, SystemAssigned); + + Assert.False(reusedHostChannel); + Assert.IsType(GetCredential(durableConfiguration)); + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public void DependencyInjection_SelectsHostTelemetryConfigurationConstructor() + { + using TelemetryConfiguration hostConfiguration = TelemetryConfiguration.CreateDefault(); + var services = new ServiceCollection(); + services.AddSingleton>( + Microsoft.Extensions.Options.Options.Create(new DurableTaskOptions())); + services.AddSingleton(Mock.Of()); + services.AddSingleton(hostConfiguration); + services.AddSingleton(); + + using ServiceProvider provider = services.BuildServiceProvider(); + TelemetryActivator activator = + Assert.IsType(provider.GetRequiredService()); + FieldInfo field = typeof(TelemetryActivator).GetField( + "hostTelemetryConfiguration", + BindingFlags.Instance | BindingFlags.NonPublic); + + Assert.NotNull(field); + Assert.Same(hostConfiguration, field.GetValue(activator)); + } + + private static bool Apply( + TelemetryConfiguration durableConfiguration, + TelemetryConfiguration hostConfiguration, + string authenticationString) + { + return TelemetryActivator.ApplyEntraAuthentication( + durableConfiguration, + hostConfiguration, + ApplicationInsightsTokenCredentialOptions.ParseAuthenticationString(authenticationString)); + } + + /// + /// Reads the credential the Application Insights SDK keeps internally. Only the test needs + /// this; the production path no longer inspects it. + /// + private static object GetCredential(TelemetryConfiguration configuration) + { + PropertyInfo envelopeProperty = typeof(TelemetryConfiguration).GetProperty( + "CredentialEnvelope", + BindingFlags.Instance | BindingFlags.NonPublic); + object envelope = envelopeProperty?.GetValue(configuration); + PropertyInfo credentialProperty = envelope?.GetType().GetProperty( + "Credential", + BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + return credentialProperty?.GetValue(envelope); + } + + private sealed class AsyncTelemetryChannel : ITelemetryChannel, IAsyncFlushable + { + private readonly bool flushResult; + + public AsyncTelemetryChannel(bool flushResult) + { + this.flushResult = flushResult; + } + + public bool? DeveloperMode { get; set; } + + public string EndpointAddress { get; set; } + + public int AsyncFlushCalls { get; private set; } + + public int FlushCalls { get; private set; } + + public CancellationToken LastCancellationToken { get; private set; } + + public void Dispose() + { + } + + public void Flush() + { + this.FlushCalls++; + } + + public Task FlushAsync(CancellationToken cancellationToken) + { + this.AsyncFlushCalls++; + this.LastCancellationToken = cancellationToken; + return Task.FromResult(this.flushResult); + } + + public void Send(ITelemetry item) + { + } + } + + private sealed class ControlledAsyncDisposable : IAsyncDisposable + { + private readonly TaskCompletionSource disposalCompletion = + new (TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource DisposalStarted { get; } = + new (TaskCreationOptions.RunContinuationsAsynchronously); + + public int DisposeCalls { get; private set; } + + public ValueTask DisposeAsync() + { + this.DisposeCalls++; + this.DisposalStarted.TrySetResult(true); + return new ValueTask(this.disposalCompletion.Task); + } + + public void CompleteDisposal() + { + this.disposalCompletion.TrySetResult(true); + } + } + } +} diff --git a/test/FunctionsV2/Helpers/TestHelpers.cs b/test/FunctionsV2/Helpers/TestHelpers.cs index 36d993f35..6e56b2dfb 100644 --- a/test/FunctionsV2/Helpers/TestHelpers.cs +++ b/test/FunctionsV2/Helpers/TestHelpers.cs @@ -15,6 +15,7 @@ using Azure.Storage.Blobs.Specialized; using DurableTask.AzureStorage; using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.Extensibility; using Microsoft.Azure.WebJobs.Extensions.DurableTask.Storage; using Microsoft.Azure.WebJobs.Host.Scale; using Microsoft.Azure.WebJobs.Host.TestCommon; @@ -69,7 +70,8 @@ public static ITestHost GetJobHost( string exactTaskHubName = null, bool addDurableClientFactory = false, Action configureScaleOptions = null, - Type[] types = null) + Type[] types = null, + TelemetryConfiguration hostTelemetryConfiguration = null) { switch (storageProviderType) { @@ -169,7 +171,8 @@ public static ITestHost GetJobHost( addDurableClientFactory: addDurableClientFactory, types: types, configureScaleOptions: configureScaleOptions, - durabilityProviderFactoryType: durabilityProviderFactoryType); + durabilityProviderFactoryType: durabilityProviderFactoryType, + hostTelemetryConfiguration: hostTelemetryConfiguration); } public static ITestHost GetJobHostWithOptions( @@ -184,7 +187,8 @@ public static ITestHost GetJobHostWithOptions( Type durabilityProviderFactoryType = null, bool addDurableClientFactory = false, Action configureScaleOptions = null, - Type[] types = null) + Type[] types = null, + TelemetryConfiguration hostTelemetryConfiguration = null) { if (serializerSettings == null) { @@ -212,7 +216,8 @@ public static ITestHost GetJobHostWithOptions( durableHttpMessageHandler: durableHttpMessageHandler, lifeCycleNotificationHelper: lifeCycleNotificationHelper, serializerSettingsFactory: serializerSettings, - onSend: onSend); + onSend: onSend, + hostTelemetryConfiguration: hostTelemetryConfiguration); } public static IHost GetJobHostExternalEnvironment(IStorageServiceClientProviderFactory clientProviderFactory = null) diff --git a/test/FunctionsV2/Tests/E2E/CorrelationEndToEndTests.cs b/test/FunctionsV2/Tests/E2E/CorrelationEndToEndTests.cs index e637f6adb..01b67021a 100644 --- a/test/FunctionsV2/Tests/E2E/CorrelationEndToEndTests.cs +++ b/test/FunctionsV2/Tests/E2E/CorrelationEndToEndTests.cs @@ -5,14 +5,18 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; +using System.Reflection; using System.Threading.Tasks; +using Azure.Identity; using DurableTask.Core; using Microsoft.ApplicationInsights.Channel; using Microsoft.ApplicationInsights.DataContracts; +using Microsoft.ApplicationInsights.Extensibility; using Microsoft.ApplicationInsights.Extensibility.Implementation; using Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation; using Microsoft.Azure.WebJobs.Extensions.DurableTask.Options; using Microsoft.Azure.WebJobs.Host.TestCommon; +using Microsoft.Extensions.DependencyInjection; using Moq; using Xunit; using Xunit.Abstractions; @@ -124,6 +128,205 @@ public async Task DistributedTracingV2_DefaultOperationNames_ArePopulated() telemetry => Assert.Equal(telemetry.Name, telemetry.Context.Operation.Name)); } + [Theory] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + [InlineData("Authorization=AAD", "system-assigned")] + [InlineData("Authorization=AAD;ClientId=00000000-0000-0000-0000-000000000001", "user-assigned")] + [InlineData(" authoRization=AAD ;; ClIentId = 00000000-0000-0000-0000-000000000001 ", "user-assigned")] + public void TelemetryActivator_ValidAuthenticationString_EnablesEntraAuthentication( + string authenticationString, + string expectedIdentityType) + { + TraceOptions traceOptions = new TraceOptions() + { + DistributedTracingEnabled = true, + Version = DurableDistributedTracingVersion.V2, + }; + DurableTaskOptions options = new DurableTaskOptions + { + Tracing = traceOptions, + }; + var mockNameResolver = GetNameResolverMock( + new[] + { + ("APPLICATIONINSIGHTS_CONNECTION_STRING", "InstrumentationKey=00000000-0000-0000-0000-000000000000"), + ("APPLICATIONINSIGHTS_AUTHENTICATION_STRING", authenticationString), + }); + + using (var host = TestHelpers.GetJobHost( + this.loggerProvider, + nameof(this.TelemetryActivator_ValidAuthenticationString_EnablesEntraAuthentication), + enableExtendedSessions: false, + nameResolver: mockNameResolver.Object, + options: options)) + { + string expectedMessage = + $"Microsoft Entra authentication enabled for Durable distributed tracing using the {expectedIdentityType} managed identity."; + Assert.Contains( + this.loggerProvider.GetAllLogMessages(), + log => log.FormattedMessage?.StartsWith(expectedMessage, StringComparison.Ordinal) == true); + } + } + + [Theory] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + [InlineData("Authorization=AAD1")] + [InlineData("Auth123=AAD")] + [InlineData("Authorization=AAD;ClientId=123")] + public void TelemetryActivator_InvalidAuthenticationString_LogsWarning(string authenticationString) + { + TraceOptions traceOptions = new TraceOptions() + { + DistributedTracingEnabled = true, + Version = DurableDistributedTracingVersion.V2, + }; + DurableTaskOptions options = new DurableTaskOptions + { + Tracing = traceOptions, + }; + var mockNameResolver = GetNameResolverMock( + new[] + { + ("APPLICATIONINSIGHTS_CONNECTION_STRING", "InstrumentationKey=00000000-0000-0000-0000-000000000000"), + ("APPLICATIONINSIGHTS_AUTHENTICATION_STRING", authenticationString), + }); + + using (var host = TestHelpers.GetJobHost( + this.loggerProvider, + nameof(this.TelemetryActivator_InvalidAuthenticationString_LogsWarning), + enableExtendedSessions: false, + nameResolver: mockNameResolver.Object, + options: options)) + { + const string expectedMessage = + "APPLICATIONINSIGHTS_AUTHENTICATION_STRING is invalid and will not be used for Durable Functions distributed tracing."; + Assert.Contains( + this.loggerProvider.GetAllLogMessages(), + log => log.FormattedMessage?.StartsWith(expectedMessage, StringComparison.Ordinal) == true); + } + } + + /* + * Reproduces the root cause of #3497 end to end. Durable keeps a private + * TelemetryConfiguration, which has no Microsoft Entra credential of its own, so with + * DisableLocalAuth=true its spans are rejected at ingestion. The fix forwards them to the + * host channel, which already carries the host's credential. Capturing on that host channel + * proves the spans really traverse it, and leaving onSend unset means the TelemetryActivator + * comes purely from the production DI registration. + */ + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task DistributedTracingV2_EntraAuthentication_ForwardsModuleSpansThroughHostChannel() + { + string[] orchestrationFunctionNames = + { + nameof(TestOrchestrations.SayHelloWithActivity), + }; + + using TelemetryConfiguration hostTelemetryConfiguration = TelemetryConfiguration.CreateDefault(); + TelemetryActivator activator = null; + + var result = await this.ExecuteOrchestrationWithExceptionAsync( + orchestrationFunctionNames, + "DTV2EntraForward", + "world", + extendedSessions: false, + protocol: "W3CTraceContext", + version: DurableDistributedTracingVersion.V2, + authenticationString: "Authorization=AAD", + hostTelemetryConfiguration: hostTelemetryConfiguration, + inspectTelemetryActivator: telemetryActivator => activator = telemetryActivator, + captureThroughHostChannel: true); + + // The activator must come from production DI with the host-aware constructor selected, + // otherwise this test would silently degrade to the standalone credential path. + Assert.NotNull(activator); + FieldInfo hostConfigurationField = typeof(TelemetryActivator).GetField( + "hostTelemetryConfiguration", + BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(hostConfigurationField); + Assert.Same(hostTelemetryConfiguration, hostConfigurationField.GetValue(activator)); + + var forwardingChannel = Assert.IsType( + activator.TelemetryConfiguration.TelemetryChannel); + Assert.Same(hostTelemetryConfiguration.TelemetryChannel, forwardingChannel.HostChannel); + + const string expectedMessage = + "Microsoft Entra authentication enabled for Durable distributed tracing using the system-assigned managed identity."; + Assert.Contains( + this.loggerProvider.GetAllLogMessages(), + log => log.FormattedMessage?.StartsWith(expectedMessage, StringComparison.Ordinal) == true); + + const string fallbackWarning = + "The Application Insights telemetry channel owned by the Functions host could not be read"; + Assert.DoesNotContain( + this.loggerProvider.GetAllLogMessages(), + log => log.FormattedMessage?.Contains(fallbackWarning) == true); + + AssertSayHelloWithActivitySpans(result.Item1); + } + + /* + * Outside the Functions host there is no host configuration to forward to, so Durable still + * has to authenticate on its own. This keeps the pre-existing behaviour covered and guards + * the missing-spans regression that caused #3009 to be reverted by #3053. + */ + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task DistributedTracingV2_EntraAuthentication_WithoutHostConfiguration_PreservesModuleSpans() + { + string[] orchestrationFunctionNames = + { + nameof(TestOrchestrations.SayHelloWithActivity), + }; + + TelemetryActivator activator = null; + + var result = await this.ExecuteOrchestrationWithExceptionAsync( + orchestrationFunctionNames, + "DTV2EntraFallback", + "world", + extendedSessions: false, + protocol: "W3CTraceContext", + version: DurableDistributedTracingVersion.V2, + authenticationString: "Authorization=AAD", + inspectTelemetryActivator: telemetryActivator => activator = telemetryActivator); + + Assert.NotNull(activator); + Assert.IsNotType(activator.TelemetryConfiguration.TelemetryChannel); + + const string expectedMessage = + "Microsoft Entra authentication enabled for Durable distributed tracing using the system-assigned managed identity."; + Assert.Contains( + this.loggerProvider.GetAllLogMessages(), + log => log.FormattedMessage?.StartsWith(expectedMessage, StringComparison.Ordinal) == true); + + AssertSayHelloWithActivitySpans(result.Item1); + } + + private static void AssertSayHelloWithActivitySpans(List telemetry) + { + DependencyTelemetry createOrchestration = Assert.Single( + telemetry.OfType(), + item => item.Name.StartsWith($"{TraceActivityConstants.CreateOrchestration}:", StringComparison.Ordinal)); + RequestTelemetry orchestration = Assert.Single( + telemetry.OfType(), + item => item.Name.StartsWith($"{TraceActivityConstants.Orchestration}:", StringComparison.Ordinal)); + DependencyTelemetry scheduleActivity = Assert.Single( + telemetry.OfType(), + item => item.Name.StartsWith($"{TraceActivityConstants.Activity}:", StringComparison.Ordinal)); + RequestTelemetry activity = Assert.Single( + telemetry.OfType(), + item => item.Name.StartsWith($"{TraceActivityConstants.Activity}:", StringComparison.Ordinal)); + + Assert.Equal(createOrchestration.Id, orchestration.Context.Operation.ParentId); + Assert.Equal(orchestration.Id, scheduleActivity.Context.Operation.ParentId); + Assert.Equal(scheduleActivity.Id, activity.Context.Operation.ParentId); + Assert.All( + new OperationTelemetry[] { createOrchestration, orchestration, scheduleActivity, activity }, + item => Assert.Equal(orchestration.Context.Operation.Id, item.Context.Operation.Id)); + } + [Theory] [Trait("Category", PlatformSpecificHelpers.TestCategory)] [InlineData(false, "W3CTraceContext")] @@ -208,7 +411,11 @@ internal async Task, List>> object input, bool extendedSessions, string protocol, - DurableDistributedTracingVersion version = DurableDistributedTracingVersion.V1) + DurableDistributedTracingVersion version = DurableDistributedTracingVersion.V1, + string authenticationString = null, + TelemetryConfiguration hostTelemetryConfiguration = null, + Action inspectTelemetryActivator = null, + bool captureThroughHostChannel = false) { ConcurrentQueue sendItems = new ConcurrentQueue(); TraceOptions traceOptions = new TraceOptions() @@ -222,9 +429,23 @@ internal async Task, List>> var sendAction = new Action( delegate(ITelemetry telemetry) { sendItems.Enqueue(telemetry); }); + // Capturing on the host channel instead of the OnSend hook leaves the TelemetryActivator + // entirely to the production registration and proves telemetry really travels through + // the host channel, which is what carries the host's Entra credential. + if (captureThroughHostChannel) + { + hostTelemetryConfiguration.TelemetryChannel = new NoOpTelemetryChannel { OnSend = sendAction }; + } + string siteNameEnvironmentVarName = "WEBSITE_SITE_NAME"; string siteNameEnvironmentVarValue = TestSiteName; - var mockNameResolver = GetNameResolverMock(new[] { (siteNameEnvironmentVarName, siteNameEnvironmentVarValue) }); + var mockNameResolver = GetNameResolverMock( + new[] + { + (siteNameEnvironmentVarName, siteNameEnvironmentVarValue), + ("APPLICATIONINSIGHTS_CONNECTION_STRING", "InstrumentationKey=00000000-0000-0000-0000-000000000000"), + ("APPLICATIONINSIGHTS_AUTHENTICATION_STRING", authenticationString), + }); using (var host = TestHelpers.GetJobHost( this.loggerProvider, @@ -232,9 +453,19 @@ internal async Task, List>> extendedSessions, options: options, nameResolver: mockNameResolver.Object, - onSend: sendAction)) + onSend: captureThroughHostChannel ? null : sendAction, + hostTelemetryConfiguration: hostTelemetryConfiguration)) { await host.StartAsync(); + + if (inspectTelemetryActivator != null) + { + IServiceProvider services = + ((PlatformSpecificHelpers.FunctionsV2HostWrapper)host).InnerHost.Services; + inspectTelemetryActivator( + Assert.IsType(services.GetRequiredService())); + } + var client = await host.StartOrchestratorAsync(orchestratorFunctionNames[0], input, this.output); await client.WaitForCompletionAsync(this.output, timeout: TimeSpan.FromSeconds(90)); await host.StopAsync(); @@ -413,6 +644,7 @@ private IEnumerable FilterOperationTelemetry(IEnumerable - + diff --git a/test/e2e/Apps/BasicDotNetIsolated/Directory.Build.props b/test/e2e/Apps/BasicDotNetIsolated/Directory.Build.props index 99c84f62b..22997f00e 100644 --- a/test/e2e/Apps/BasicDotNetIsolated/Directory.Build.props +++ b/test/e2e/Apps/BasicDotNetIsolated/Directory.Build.props @@ -1,12 +1,14 @@ - + diff --git a/test/e2e/Apps/BasicDotNetIsolated/Directory.Build.targets b/test/e2e/Apps/BasicDotNetIsolated/Directory.Build.targets index 2cada6cc7..1d2f41d03 100644 --- a/test/e2e/Apps/BasicDotNetIsolated/Directory.Build.targets +++ b/test/e2e/Apps/BasicDotNetIsolated/Directory.Build.targets @@ -2,7 +2,7 @@ diff --git a/test/e2e/Apps/BasicDotNetIsolated/Directory.Packages.props b/test/e2e/Apps/BasicDotNetIsolated/Directory.Packages.props index 2078abe3f..195911aa7 100644 --- a/test/e2e/Apps/BasicDotNetIsolated/Directory.Packages.props +++ b/test/e2e/Apps/BasicDotNetIsolated/Directory.Packages.props @@ -7,7 +7,7 @@ NuGet walks upward from the project file looking for the nearest Directory.Packages.props. This file is found before test/Directory.Packages.props, giving us the chance to: - Disable CPM for WorkerExtensions (so its hardcoded versions + our System.Drawing.Common - 4.7.3 override from Directory.Build.props are accepted). + 6.0.0 override from Directory.Build.props are accepted). - Import the parent Directory.Packages.props for app.csproj and all other normal projects. -->