diff --git a/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableClient.cs b/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableClient.cs index 8a96143ff..4ff2b491c 100644 --- a/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableClient.cs +++ b/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableClient.cs @@ -181,6 +181,8 @@ async Task IDurableOrchestrationClient.StartNewAsync(string orchestra this.config?.ThrowIfFunctionDoesNotExist(orchestratorFunctionName, FunctionType.Orchestrator); } + this.ThrowIfOrchestratorFunctionIsDisabled(orchestratorFunctionName); + if (string.IsNullOrEmpty(instanceId)) { instanceId = Guid.NewGuid().ToString("N"); @@ -361,6 +363,14 @@ private bool ClientReferencesCurrentApp(DurableClient client) this.ConnectionNameMatchesCurrentApp(client); } + internal void ThrowIfOrchestratorFunctionIsDisabled(string name) + { + if (this.ClientReferencesCurrentApp(this)) + { + this.config?.ThrowIfOrchestratorFunctionIsDisabled(name); + } + } + private bool TaskHubMatchesCurrentApp(DurableClient client) { var taskHubName = this.durableTaskOptions.HubName; @@ -1263,6 +1273,8 @@ async Task IDurableOrchestrationClient.RestartAsync(string instanceId, b // GetOrchestrationInstanceStateAsync will throw ArgumentException if the provided instanceid is not found. OrchestrationState state = await this.GetOrchestrationInstanceStateAsync(instanceId); + this.ThrowIfOrchestratorFunctionIsDisabled(state.Name); + JToken input = ParseToJToken(state.Input); string newInstanceId = null; diff --git a/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs b/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs index 86b67ba39..6974b80ef 100644 --- a/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs +++ b/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs @@ -1525,6 +1525,18 @@ internal void ThrowIfFunctionDoesNotExist(string name, FunctionType functionType } } + internal void ThrowIfOrchestratorFunctionIsDisabled(string name) + { + // A null registration means the function was indexed but disabled, so its listener was + // never created. A missing registration and IsDeregistered are deliberately allowed: + // the target may exist in another app, or its listener may be restarting on this host. + if (this.knownOrchestrators.TryGetValue(new FunctionName(name), out RegisteredFunctionInfo info) && + info == null) + { + throw new OrchestratorFunctionUnavailableException(this.GetInvalidOrchestratorFunctionMessage(name)); + } + } + private static bool IsDurableHttpTask(string functionName) { return string.Equals(functionName, HttpOptions.HttpTaskActivityReservedName); diff --git a/src/WebJobs.Extensions.DurableTask/Exceptions/OrchestratorFunctionUnavailableException.cs b/src/WebJobs.Extensions.DurableTask/Exceptions/OrchestratorFunctionUnavailableException.cs new file mode 100644 index 000000000..81fd4fd4c --- /dev/null +++ b/src/WebJobs.Extensions.DurableTask/Exceptions/OrchestratorFunctionUnavailableException.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; + +namespace Microsoft.Azure.WebJobs.Extensions.DurableTask +{ + internal sealed class OrchestratorFunctionUnavailableException : ArgumentException + { + internal OrchestratorFunctionUnavailableException(string message) + : base(message) + { + } + } +} diff --git a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs index 94c9d4c8e..b7b9d9f24 100644 --- a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs +++ b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs @@ -958,6 +958,8 @@ private async Task HandleStartOrchestratorRequestAsync( if (client is DurableClient durableClient) { + durableClient.ThrowIfOrchestratorFunctionIsDisabled(functionName); + var instance = new OrchestrationInstance { InstanceId = id, @@ -1101,6 +1103,10 @@ private async Task HandleRestartInstanceRequestAsync( return client.CreateCheckStatusResponse(request, newInstanceId); } } + catch (OrchestratorFunctionUnavailableException e) + { + return request.CreateErrorResponse(HttpStatusCode.BadRequest, e.Message, e); + } catch (ArgumentException e) { return request.CreateErrorResponse(HttpStatusCode.BadRequest, "InstanceId does not match a valid orchestration instance.", e); diff --git a/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs b/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs index d1646519a..b3f3f5f3b 100644 --- a/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs +++ b/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs @@ -59,6 +59,11 @@ public override Task Hello(Empty request, ServerCallContext context) { try { + if (this.GetClient(context) is DurableClient durableClient) + { + durableClient.ThrowIfOrchestratorFunctionIsDisabled(request.Name); + } + List allStatuses = System.Enum .GetValues() .ToList(); @@ -521,6 +526,10 @@ await this.GetDurabilityProvider(context).CreateTaskOrchestrationAsync( string newInstanceId = await this.GetClient(context).RestartAsync(request.InstanceId, request.RestartWithNewInstanceId); return new P.RestartInstanceResponse { InstanceId = newInstanceId }; } + catch (OrchestratorFunctionUnavailableException ex) + { + throw new RpcException(new Status(StatusCode.InvalidArgument, ex.Message)); + } catch (ArgumentException ex) { // Thrown when th instanceId is not found. diff --git a/test/FunctionsV2/FunctionApp/TestOrchestrations.cs b/test/FunctionsV2/FunctionApp/TestOrchestrations.cs index d0791895b..c6f411d3c 100644 --- a/test/FunctionsV2/FunctionApp/TestOrchestrations.cs +++ b/test/FunctionsV2/FunctionApp/TestOrchestrations.cs @@ -20,6 +20,12 @@ internal static class TestOrchestrations public static bool SayHelloWithActivityForRewindShouldFail { get; set; } = true; + [Disable] + public static string DisabledOrchestrator([OrchestrationTrigger] IDurableOrchestrationContext ctx) + { + return nameof(DisabledOrchestrator); + } + public static string SayHelloInline([OrchestrationTrigger] IDurableOrchestrationContext ctx) { string input = ctx.GetInput(); diff --git a/test/FunctionsV2/Tests/E2E/StatusAndQueryTests.cs b/test/FunctionsV2/Tests/E2E/StatusAndQueryTests.cs index 4dfcee539..25b8879b5 100644 --- a/test/FunctionsV2/Tests/E2E/StatusAndQueryTests.cs +++ b/test/FunctionsV2/Tests/E2E/StatusAndQueryTests.cs @@ -57,6 +57,53 @@ public async Task StartOrchestration_OnUnregisteredOrchestrator(bool extendedSes } } + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task DisabledOrchestrator_IsRejectedWithoutPersistingInstance() + { + const string InstanceId = "disabled-orchestrator-instance"; + string taskHubName = TestHelpers.GetTaskHubNameFromTestName( + nameof(this.DisabledOrchestrator_IsRejectedWithoutPersistingInstance), + enableExtendedSessions: false); + var nameResolver = new SimpleNameResolver( + new Dictionary + { + { "TestTaskHub", taskHubName }, + }); + + using (ITestHost host = TestHelpers.GetJobHost( + this.loggerProvider, + nameof(this.DisabledOrchestrator_IsRejectedWithoutPersistingInstance), + enableExtendedSessions: false, + nameResolver: nameResolver, + storageProviderType: TestHelpers.AzureStorageProviderType, + exactTaskHubName: taskHubName)) + { + await host.StartAsync(); + + Exception exception = await Record.ExceptionAsync( + () => host.StartOrchestratorAsync( + nameof(TestOrchestrations.DisabledOrchestrator), + input: null, + this.output, + instanceId: InstanceId, + useTaskHubFromAppSettings: true)); + + IDurableOrchestrationClient defaultClient = + await host.GetOrchestrationClientBindingTest(this.output); + Assert.Null(await defaultClient.GetStatusAsync(InstanceId)); + + FunctionInvocationException invocationException = + Assert.IsType(exception); + Assert.NotNull(invocationException.InnerException); + Assert.Contains( + "doesn't exist, is disabled, or is not an orchestrator function", + invocationException.InnerException.ToString()); + + await host.StopAsync(); + } + } + /// /// End-to-end test which creates an external client that calls a non-existent orchestrator function. /// diff --git a/test/FunctionsV2/Tests/Unit/DisabledFunctionDispatchTests.cs b/test/FunctionsV2/Tests/Unit/DisabledFunctionDispatchTests.cs index 2fe6bedd5..7e50d6b40 100644 --- a/test/FunctionsV2/Tests/Unit/DisabledFunctionDispatchTests.cs +++ b/test/FunctionsV2/Tests/Unit/DisabledFunctionDispatchTests.cs @@ -91,6 +91,41 @@ public void GetObject_ActiveActivity_ReturnsRealActivityShim() Assert.IsType(shim); } + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public void ThrowIfOrchestratorFunctionIsDisabled_ActiveOutOfProcOrchestrator_DoesNotThrow() + { + var extension = CreateExtension(); + var mockExecutor = new Mock(); + extension.RegisterOrchestrator( + new FunctionName("ActiveOrchestrator"), + new RegisteredFunctionInfo(mockExecutor.Object, isOutOfProc: true)); + + extension.ThrowIfOrchestratorFunctionIsDisabled("ActiveOrchestrator"); + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public void ThrowIfOrchestratorFunctionIsDisabled_MissingOrchestrator_DoesNotThrow() + { + var extension = CreateExtension(); + + extension.ThrowIfOrchestratorFunctionIsDisabled("MissingOrchestrator"); + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public void ThrowIfFunctionDoesNotExist_DisabledOrchestrator_DoesNotBreakReplay() + { + // Deterministic orchestration calls use this helper while replaying history. A disabled + // orchestrator remains a known function, so availability checks belong only at new-start + // entry points and must not make existing sub-orchestration history fail to replay. + var extension = CreateExtension(); + extension.RegisterOrchestrator(new FunctionName("DisabledOrchestrator"), orchestratorInfo: null); + + extension.ThrowIfFunctionDoesNotExist("DisabledOrchestrator", FunctionType.Orchestrator); + } + [Fact] [Trait("Category", PlatformSpecificHelpers.TestCategory)] public void DisabledEntity_IsTreatedAsUnavailableByClassicDispatch() diff --git a/test/FunctionsV2/Tests/Unit/DurableClientBaseTests.cs b/test/FunctionsV2/Tests/Unit/DurableClientBaseTests.cs index e6774be6e..1ae8300b2 100644 --- a/test/FunctionsV2/Tests/Unit/DurableClientBaseTests.cs +++ b/test/FunctionsV2/Tests/Unit/DurableClientBaseTests.cs @@ -51,6 +51,305 @@ public async Task StartNewAsync_InvalidInstanceId_ThrowsException(string instanc await Assert.ThrowsAnyAsync(async () => await durableClient.StartNewAsync("anyOrchestratorFunction", instanceId, new { message = "any obj" })); } + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task StartNewAsync_MissingOrchestrator_ThrowsException() + { + var orchestrationServiceClientMock = new Mock(); + orchestrationServiceClientMock + .Setup(x => x.CreateTaskOrchestrationAsync(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + var storageProvider = new DurabilityProvider( + "test", + new Mock().Object, + orchestrationServiceClientMock.Object, + TestConstants.ConnectionName); + var durableExtension = GetDurableTaskConfig(); + var durableClient = (IDurableOrchestrationClient)new DurableClient( + storageProvider, + durableExtension, + durableExtension.HttpApiHandler, + new DurableClientAttribute()); + + ArgumentException exception = await Assert.ThrowsAnyAsync( + () => durableClient.StartNewAsync("MissingOrchestrator")); + + Assert.Contains("doesn't exist, is disabled, or is not an orchestrator function", exception.Message); + orchestrationServiceClientMock.Verify( + x => x.CreateTaskOrchestrationAsync(It.IsAny(), It.IsAny()), + Times.Never()); + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task StartNewAsync_DifferentConnection_MissingOrchestrator_ThrowsException() + { + (DurableClient durableClient, Mock serviceClient, _) = + CreateClient(); + + ArgumentException exception = await Assert.ThrowsAsync( + () => ((IDurableOrchestrationClient)durableClient).StartNewAsync("MissingOrchestrator")); + + Assert.Contains("doesn't exist, is disabled, or is not an orchestrator function", exception.Message); + serviceClient.Verify( + x => x.CreateTaskOrchestrationAsync(It.IsAny(), It.IsAny()), + Times.Never()); + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task StartNewAsync_CaseVariantTaskHub_MissingOrchestrator_SchedulesInstance() + { + (DurableClient durableClient, Mock serviceClient, _) = + CreateClient(taskHub: "durabletaskhub", connectionName: TestConstants.ConnectionName); + + await ((IDurableOrchestrationClient)durableClient).StartNewAsync("MissingOrchestrator"); + + serviceClient.Verify( + x => x.CreateTaskOrchestrationAsync(It.IsAny(), It.IsAny()), + Times.Once()); + } + + [Theory] + [InlineData(null, true)] + [InlineData("DurableTaskHub", true)] + [InlineData("durabletaskhub", false)] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task StartNewAsync_DisabledOrchestrator_UsesLegacyTaskHubMatching( + string taskHub, + bool rejectStart) + { + var orchestrationServiceClientMock = new Mock(); + orchestrationServiceClientMock + .Setup(x => x.CreateTaskOrchestrationAsync(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + var storageProvider = new DurabilityProvider( + "clientProvider", + new Mock().Object, + orchestrationServiceClientMock.Object, + TestConstants.ConnectionName); + var durableExtension = GetDurableTaskConfig(); + durableExtension.RegisterOrchestrator(new FunctionName("DisabledOrchestrator"), orchestratorInfo: null); + var durableClient = (IDurableOrchestrationClient)new DurableClient( + storageProvider, + durableExtension, + durableExtension.HttpApiHandler, + new DurableClientAttribute { TaskHub = taskHub }); + + if (rejectStart) + { + ArgumentException exception = await Assert.ThrowsAnyAsync( + () => durableClient.StartNewAsync("DisabledOrchestrator")); + Assert.Contains("doesn't exist, is disabled, or is not an orchestrator function", exception.Message); + } + else + { + await durableClient.StartNewAsync("DisabledOrchestrator"); + } + + orchestrationServiceClientMock.Verify( + x => x.CreateTaskOrchestrationAsync(It.IsAny(), It.IsAny()), + rejectStart ? Times.Never() : Times.Once()); + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task StartNewAsync_DifferentConnection_DisabledOrchestrator_ThrowsException() + { + const string FunctionName = "DisabledOrchestrator"; + (DurableClient durableClient, Mock serviceClient, DurableTaskExtension extension) = + CreateClient(); + extension.RegisterOrchestrator(new FunctionName(FunctionName), orchestratorInfo: null); + + await Assert.ThrowsAsync( + () => ((IDurableOrchestrationClient)durableClient).StartNewAsync(FunctionName)); + + serviceClient.Verify( + x => x.CreateTaskOrchestrationAsync(It.IsAny(), It.IsAny()), + Times.Never()); + } + + [Theory] + [InlineData(null, true)] + [InlineData("DurableTaskHub", true)] + [InlineData("durabletaskhub", false)] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task RestartAsync_DisabledOrchestrator_UsesLegacyTaskHubMatching( + string taskHub, + bool rejectRestart) + { + const string InstanceId = "completed-instance"; + const string FunctionName = "DisabledOrchestrator"; + var orchestrationServiceClientMock = new Mock(); + orchestrationServiceClientMock + .Setup(x => x.GetOrchestrationStateAsync(InstanceId, false)) + .ReturnsAsync( + new List + { + new OrchestrationState + { + Name = FunctionName, + Input = "null", + OrchestrationInstance = new OrchestrationInstance { InstanceId = InstanceId }, + OrchestrationStatus = OrchestrationStatus.Completed, + }, + }); + orchestrationServiceClientMock + .Setup(x => x.CreateTaskOrchestrationAsync(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + var storageProvider = new DurabilityProvider( + "test", + new Mock().Object, + orchestrationServiceClientMock.Object, + TestConstants.ConnectionName); + var durableExtension = GetDurableTaskConfig(); + durableExtension.RegisterOrchestrator(new FunctionName(FunctionName), orchestratorInfo: null); + var durableClient = (IDurableOrchestrationClient)new DurableClient( + storageProvider, + durableExtension, + durableExtension.HttpApiHandler, + new DurableClientAttribute { TaskHub = taskHub }); + + if (rejectRestart) + { + ArgumentException exception = await Assert.ThrowsAnyAsync( + () => durableClient.RestartAsync(InstanceId, restartWithNewInstanceId: false)); + Assert.Contains("doesn't exist, is disabled, or is not an orchestrator function", exception.Message); + } + else + { + await durableClient.RestartAsync(InstanceId, restartWithNewInstanceId: false); + } + + orchestrationServiceClientMock.Verify( + x => x.CreateTaskOrchestrationAsync(It.IsAny(), It.IsAny()), + rejectRestart ? Times.Never() : Times.Once()); + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task RestartAsync_DifferentConnection_DisabledOrchestrator_ThrowsException() + { + const string InstanceId = "completed-instance"; + const string FunctionName = "DisabledOrchestrator"; + (DurableClient durableClient, Mock serviceClient, DurableTaskExtension extension) = + CreateClient(); + serviceClient + .Setup(x => x.GetOrchestrationStateAsync(InstanceId, false)) + .ReturnsAsync( + new List + { + new OrchestrationState + { + Name = FunctionName, + Input = "null", + OrchestrationInstance = new OrchestrationInstance { InstanceId = InstanceId }, + OrchestrationStatus = OrchestrationStatus.Completed, + }, + }); + extension.RegisterOrchestrator(new FunctionName(FunctionName), orchestratorInfo: null); + + await Assert.ThrowsAsync( + () => ((IDurableOrchestrationClient)durableClient).RestartAsync( + InstanceId, + restartWithNewInstanceId: false)); + + serviceClient.Verify( + x => x.CreateTaskOrchestrationAsync(It.IsAny(), It.IsAny()), + Times.Never()); + } + + [Theory] + [InlineData("MissingOrchestrator", false)] + [InlineData("DisabledOrchestrator", true)] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task StartNewAsync_ExternalClient_BypassesLocalValidation( + string functionName, + bool registerDisabled) + { + (DurableClient durableClient, Mock serviceClient, DurableTaskExtension extension) = + CreateClient(externalClient: true); + if (registerDisabled) + { + extension.RegisterOrchestrator(new FunctionName(functionName), orchestratorInfo: null); + } + + await ((IDurableOrchestrationClient)durableClient).StartNewAsync(functionName); + + serviceClient.Verify( + x => x.CreateTaskOrchestrationAsync(It.IsAny(), It.IsAny()), + Times.Once()); + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task RestartAsync_ExternalClient_BypassesDisabledValidation() + { + const string InstanceId = "completed-instance"; + const string FunctionName = "DisabledOrchestrator"; + (DurableClient durableClient, Mock serviceClient, DurableTaskExtension extension) = + CreateClient(externalClient: true); + serviceClient + .Setup(x => x.GetOrchestrationStateAsync(InstanceId, false)) + .ReturnsAsync( + new List + { + new OrchestrationState + { + Name = FunctionName, + Input = "null", + OrchestrationInstance = new OrchestrationInstance { InstanceId = InstanceId }, + OrchestrationStatus = OrchestrationStatus.Completed, + }, + }); + extension.RegisterOrchestrator(new FunctionName(FunctionName), orchestratorInfo: null); + + await ((IDurableOrchestrationClient)durableClient).RestartAsync( + InstanceId, + restartWithNewInstanceId: false); + + serviceClient.Verify( + x => x.CreateTaskOrchestrationAsync(It.IsAny(), It.IsAny()), + Times.Once()); + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task StartNewAsync_DeregisteredOrchestrator_SchedulesInstance() + { + const string FunctionName = "DrainingOrchestrator"; + const string InstanceId = "new-instance"; + var orchestrationServiceClientMock = new Mock(); + orchestrationServiceClientMock + .Setup(x => x.CreateTaskOrchestrationAsync(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + var storageProvider = new DurabilityProvider( + "test", + new Mock().Object, + orchestrationServiceClientMock.Object, + TestConstants.ConnectionName); + var durableExtension = GetDurableTaskConfig(); + var functionName = new FunctionName(FunctionName); + durableExtension.RegisterOrchestrator( + functionName, + new RegisteredFunctionInfo(executor: null, isOutOfProc: true)); + durableExtension.DeregisterOrchestrator(functionName); + var durableClient = (IDurableOrchestrationClient)new DurableClient( + storageProvider, + durableExtension, + durableExtension.HttpApiHandler, + new DurableClientAttribute { }); + + string actualInstanceId = await durableClient.StartNewAsync( + FunctionName, + InstanceId); + + Assert.Equal(InstanceId, actualInstanceId); + orchestrationServiceClientMock.Verify( + x => x.CreateTaskOrchestrationAsync(It.IsAny(), It.IsAny()), + Times.Once()); + } + [Theory] [Trait("Category", PlatformSpecificHelpers.TestCategory)] [InlineData("@invalid")] @@ -64,7 +363,11 @@ public async Task SignalEntity_InvalidEntityKey_ThrowsException(string entityKey { var orchestrationServiceClientMock = new Mock(); orchestrationServiceClientMock.Setup(x => x.GetOrchestrationStateAsync(It.IsAny(), It.IsAny())).ReturnsAsync(GetInvalidInstanceState()); - var storageProvider = new DurabilityProvider("test", new Mock().Object, orchestrationServiceClientMock.Object, "test"); + var storageProvider = new DurabilityProvider( + "test", + new Mock().Object, + orchestrationServiceClientMock.Object, + TestConstants.ConnectionName); var durableExtension = GetDurableTaskConfig(); var durableClient = (IDurableEntityClient)new DurableClient(storageProvider, durableExtension, durableExtension.HttpApiHandler, new DurableClientAttribute { }); @@ -72,6 +375,42 @@ public async Task SignalEntity_InvalidEntityKey_ThrowsException(string entityKey await Assert.ThrowsAnyAsync(async () => await durableClient.SignalEntityAsync(entityId, "test")); } + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task SignalEntityAsync_DifferentConnection_MissingEntity_ThrowsException() + { + (DurableClient durableClient, Mock serviceClient, _) = + CreateClient(); + + ArgumentException exception = await Assert.ThrowsAsync( + () => ((IDurableEntityClient)durableClient).SignalEntityAsync( + new EntityId("MissingEntity", "entity-key"), + "operation", + new { Value = 1 })); + + Assert.Contains("doesn't exist, is disabled, or is not an entity function", exception.Message); + serviceClient.Verify( + x => x.SendTaskOrchestrationMessageAsync(It.IsAny()), + Times.Never()); + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task SignalEntityAsync_CaseVariantTaskHub_MissingEntity_SendsMessage() + { + (DurableClient durableClient, Mock serviceClient, _) = + CreateClient(taskHub: "durabletaskhub", connectionName: TestConstants.ConnectionName); + + await ((IDurableEntityClient)durableClient).SignalEntityAsync( + new EntityId("MissingEntity", "entity-key"), + "operation", + new { Value = 1 }); + + serviceClient.Verify( + x => x.SendTaskOrchestrationMessageAsync(It.IsAny()), + Times.Once()); + } + [Theory] [Trait("Category", PlatformSpecificHelpers.TestCategory)] [InlineData(false)] @@ -336,6 +675,41 @@ private IDurableOrchestrationClient GetDurableClient(IOrchestrationServiceClient return durableOrchestrationClient; } + private static ( + DurableClient Client, + Mock ServiceClient, + DurableTaskExtension Extension) CreateClient( + string taskHub = "DurableTaskHub", + string connectionName = "TargetStorage", + bool externalClient = false) + { + var serviceClient = new Mock(); + serviceClient + .Setup(x => x.CreateTaskOrchestrationAsync(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + serviceClient + .Setup(x => x.SendTaskOrchestrationMessageAsync(It.IsAny())) + .Returns(Task.CompletedTask); + var storageProvider = new DurabilityProvider( + "clientProvider", + new Mock().Object, + serviceClient.Object, + connectionName); + var durableExtension = GetDurableTaskConfig(); + var attribute = new DurableClientAttribute + { + TaskHub = taskHub, + ConnectionName = connectionName, + ExternalClient = externalClient, + }; + var client = new DurableClient( + storageProvider, + durableExtension, + durableExtension.HttpApiHandler, + attribute); + return (client, serviceClient, durableExtension); + } + [Fact] [Trait("Category", PlatformSpecificHelpers.TestCategory)] public async Task HttpRequest_HttpRequestMessage_ClientMethods_Identical() diff --git a/test/FunctionsV2/Tests/Unit/HttpApiHandlerTests.cs b/test/FunctionsV2/Tests/Unit/HttpApiHandlerTests.cs index 965922473..e4a31e767 100644 --- a/test/FunctionsV2/Tests/Unit/HttpApiHandlerTests.cs +++ b/test/FunctionsV2/Tests/Unit/HttpApiHandlerTests.cs @@ -1362,6 +1362,63 @@ public async Task RestartInstance_Returns_HTTP_400_On_Invalid_InstanceId() Assert.Equal("InstanceId does not match a valid orchestration instance.", error["Message"].ToString()); } + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task RestartInstance_Returns_HTTP_400_On_Disabled_Function() + { + const string InstanceId = "completed-instance"; + const string FunctionName = "DisabledOrchestrator"; + var requestUri = new Uri( + $"http://localhost/runtime/webhooks/durabletask/instances/{InstanceId}/restart"); + var orchestrationServiceMock = new Mock(MockBehavior.Strict); + var orchestrationServiceClientMock = new Mock(); + orchestrationServiceClientMock + .Setup(client => client.GetOrchestrationStateAsync(InstanceId, false)) + .ReturnsAsync( + new List + { + new OrchestrationState + { + Name = FunctionName, + Input = "null", + OrchestrationInstance = new OrchestrationInstance { InstanceId = InstanceId }, + OrchestrationStatus = OrchestrationStatus.Completed, + }, + }); + orchestrationServiceClientMock + .Setup(client => client.CreateTaskOrchestrationAsync( + It.IsAny(), + It.IsAny())) + .Returns(Task.CompletedTask); + var durabilityProvider = new DurabilityProvider( + "storageProviderName", + orchestrationServiceMock.Object, + orchestrationServiceClientMock.Object, + TestConstants.ConnectionName); + var options = new DurableTaskOptions + { + WebhookUriProviderOverride = () => new Uri("http://localhost/runtime/webhooks/durabletask"), + HubName = TestConstants.TaskHub, + }; + var extension = TestDurableTaskExtension.CreateWithProvider(options, durabilityProvider); + extension.RegisterOrchestrator(new FunctionName(FunctionName), orchestratorInfo: null); + var httpApiHandler = new HttpApiHandler(extension, NullLogger.Instance); + + HttpResponseMessage response = await httpApiHandler.HandleRequestAsync( + new HttpRequestMessage(HttpMethod.Post, requestUri), + CancellationToken.None); + + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + string content = await response.Content.ReadAsStringAsync(); + var error = JsonConvert.DeserializeObject(content); + Assert.Contains("doesn't exist, is disabled, or is not an orchestrator function", error["Message"].ToString()); + orchestrationServiceClientMock.Verify( + client => client.CreateTaskOrchestrationAsync( + It.IsAny(), + It.IsAny()), + Times.Never); + } + [Fact] [Trait("Category", PlatformSpecificHelpers.TestCategory)] public async Task RestartInstance_Returns_HTTP_409_On_Invalid_Existing_Instance() @@ -1995,7 +2052,7 @@ public async Task StartNewInstance_Calls_CreateTaskOrchestrationAsync_With_Corre "storageProviderName", orchestrationServiceMock.Object, orchestrationServiceClientMock.Object, - "connectionName"); + TestConstants.ConnectionName); var options = new DurableTaskOptions { @@ -2005,6 +2062,9 @@ public async Task StartNewInstance_Calls_CreateTaskOrchestrationAsync_With_Corre }; var customExtension = TestDurableTaskExtension.CreateWithProvider(options, durabilityProvider); + customExtension.RegisterOrchestrator( + new FunctionName(functionName), + new RegisteredFunctionInfo(executor: null, isOutOfProc: true)); var handler = new HttpApiHandler(customExtension, NullLogger.Instance); var request = new HttpRequestMessage(HttpMethod.Post, requestUri); @@ -2020,6 +2080,88 @@ public async Task StartNewInstance_Calls_CreateTaskOrchestrationAsync_With_Corre Assert.Equal(expectedVersion, capturedEvent.Version); } + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task StartNewInstance_Returns_HTTP_400_On_Disabled_Function() + { + const string FunctionName = "DisabledOrchestrator"; + var requestUri = new Uri( + $"http://localhost/runtime/webhooks/durabletask/orchestrators/{FunctionName}"); + var orchestrationServiceMock = new Mock(MockBehavior.Strict); + var orchestrationServiceClientMock = new Mock(); + orchestrationServiceClientMock + .Setup(p => p.CreateTaskOrchestrationAsync( + It.IsAny(), + It.IsAny())) + .Returns(Task.CompletedTask); + var durabilityProvider = new DurabilityProvider( + "storageProviderName", + orchestrationServiceMock.Object, + orchestrationServiceClientMock.Object, + TestConstants.ConnectionName); + var options = new DurableTaskOptions + { + WebhookUriProviderOverride = () => new Uri("http://localhost/runtime/webhooks/durabletask"), + HubName = TestConstants.TaskHub, + }; + var extension = TestDurableTaskExtension.CreateWithProvider(options, durabilityProvider); + extension.RegisterOrchestrator(new FunctionName(FunctionName), orchestratorInfo: null); + var handler = new HttpApiHandler(extension, NullLogger.Instance); + + HttpResponseMessage response = await handler.HandleRequestAsync( + new HttpRequestMessage(HttpMethod.Post, requestUri), + CancellationToken.None); + + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + string content = await response.Content.ReadAsStringAsync(); + Assert.Contains("doesn't exist, is disabled, or is not an orchestrator function", content); + orchestrationServiceClientMock.Verify( + p => p.CreateTaskOrchestrationAsync( + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task StartNewInstance_CaseVariantTaskHub_DoesNotRejectLocallyDisabledFunction() + { + const string FunctionName = "DisabledOrchestrator"; + var requestUri = new Uri( + $"http://localhost/runtime/webhooks/durabletask/orchestrators/{FunctionName}?taskHub=testhubname"); + var orchestrationServiceMock = new Mock(MockBehavior.Strict); + var orchestrationServiceClientMock = new Mock(); + orchestrationServiceClientMock + .Setup(p => p.CreateTaskOrchestrationAsync( + It.IsAny(), + It.IsAny())) + .Returns(Task.CompletedTask); + var durabilityProvider = new DurabilityProvider( + "storageProviderName", + orchestrationServiceMock.Object, + orchestrationServiceClientMock.Object, + TestConstants.ConnectionName); + var options = new DurableTaskOptions + { + WebhookUriProviderOverride = () => new Uri("http://localhost/runtime/webhooks/durabletask"), + HubName = TestConstants.TaskHub, + }; + var extension = TestDurableTaskExtension.CreateWithProvider(options, durabilityProvider); + extension.RegisterOrchestrator(new FunctionName(FunctionName), orchestratorInfo: null); + var handler = new HttpApiHandler(extension, NullLogger.Instance); + + HttpResponseMessage response = await handler.HandleRequestAsync( + new HttpRequestMessage(HttpMethod.Post, requestUri), + CancellationToken.None); + + Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); + orchestrationServiceClientMock.Verify( + p => p.CreateTaskOrchestrationAsync( + It.IsAny(), + It.IsAny()), + Times.Once); + } + private static DurableTaskExtension GetTestExtension() { var options = new DurableTaskOptions(); diff --git a/test/FunctionsV2/Tests/Unit/LocalGrpcListenerTests.cs b/test/FunctionsV2/Tests/Unit/LocalGrpcListenerTests.cs index ba1edcc63..4d4bf8ce9 100644 --- a/test/FunctionsV2/Tests/Unit/LocalGrpcListenerTests.cs +++ b/test/FunctionsV2/Tests/Unit/LocalGrpcListenerTests.cs @@ -267,7 +267,10 @@ public async Task TestGrpcListener_DoesNotLogStartInstanceClientCancellation() await providerCalled.Task.WaitAsync(TimeSpan.FromSeconds(5)); cancellation.Cancel(); await call.ResponseAsync; - }); + }, + extension => extension.RegisterOrchestrator( + new FunctionName("TestOrchestration"), + new RegisteredFunctionInfo(executor: null, isOutOfProc: true))); Assert.Equal(StatusCode.Cancelled, rpcException.StatusCode); Assert.DoesNotContain( @@ -276,6 +279,160 @@ public async Task TestGrpcListener_DoesNotLogStartInstanceClientCancellation() message.FormattedMessage.Contains("StartInstance")); } + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task TestGrpcListener_RejectsDisabledOrchestrator() + { + const string FunctionName = "DisabledOrchestrator"; + Mock durabilityProvider = CreateDurabilityProviderMock( + new Mock().Object, + new Mock().Object); + durabilityProvider + .Setup(provider => provider.CreateTaskOrchestrationAsync( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(Task.CompletedTask); + + RpcException rpcException = await this.InvokeFailingRpcAsync( + "DisabledOrchestratorStart", + durabilityProvider.Object, + async client => await client.StartInstanceAsync( + new P.CreateInstanceRequest { Name = FunctionName }).ResponseAsync, + extension => extension.RegisterOrchestrator( + new FunctionName(FunctionName), + orchestratorInfo: null)); + + Assert.Equal(StatusCode.InvalidArgument, rpcException.StatusCode); + Assert.Contains("doesn't exist, is disabled, or is not an orchestrator function", rpcException.Status.Detail); + durabilityProvider.Verify( + provider => provider.CreateTaskOrchestrationAsync( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task TestGrpcListener_CaseVariantTaskHub_DoesNotRejectLocallyDisabledFunction() + { + const string FunctionName = "RemoteOrchestrator"; + Mock durabilityProvider = CreateDurabilityProviderMock( + new Mock().Object, + new Mock().Object); + durabilityProvider + .Setup(provider => provider.CreateTaskOrchestrationAsync( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(Task.CompletedTask); + using DurableTaskExtension extension = this.CreateExtension( + "CurrentHub", + durabilityProvider.Object, + durabilityProvider.Object); + extension.RegisterOrchestrator(new FunctionName(FunctionName), orchestratorInfo: null); + ILocalGrpcListener listener = LocalGrpcListener.Create( + extension, + LocalGrpcListenerMode.AspNetCore); + + try + { + await listener.StartAsync(default); + using GrpcChannel channel = GrpcChannel.ForAddress(listener.ListenAddress); + var client = new P.TaskHubSidecarService.TaskHubSidecarServiceClient(channel); + var headers = new Metadata { { TaskHubMetadataKey, "currenthub" } }; + + await client.StartInstanceAsync( + new P.CreateInstanceRequest { Name = FunctionName }, + headers).ResponseAsync; + } + finally + { + await listener.StopAsync(default); + } + + durabilityProvider.Verify( + provider => provider.CreateTaskOrchestrationAsync( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Once); + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task TestGrpcListener_UnknownOrchestrator_SchedulesInstance() + { + Mock durabilityProvider = CreateDurabilityProviderMock( + new Mock().Object, + new Mock().Object); + durabilityProvider + .Setup(provider => provider.CreateTaskOrchestrationAsync( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(Task.CompletedTask); + + await this.InvokeRpcAsync( + durabilityProvider.Object, + async client => await client.StartInstanceAsync( + new P.CreateInstanceRequest { Name = "UnknownOrchestrator" }).ResponseAsync); + + durabilityProvider.Verify( + provider => provider.CreateTaskOrchestrationAsync( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Once); + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task TestGrpcListener_RestartRejectsDisabledOrchestrator() + { + const string InstanceId = "completed-instance"; + const string FunctionName = "DisabledOrchestrator"; + var orchestrationServiceClient = new Mock(); + orchestrationServiceClient + .Setup(client => client.GetOrchestrationStateAsync(InstanceId, false)) + .ReturnsAsync( + new List + { + new OrchestrationState + { + Name = FunctionName, + Input = "null", + OrchestrationInstance = new OrchestrationInstance { InstanceId = InstanceId }, + OrchestrationStatus = OrchestrationStatus.Completed, + }, + }); + Mock durabilityProvider = CreateDurabilityProviderMock( + new Mock().Object, + orchestrationServiceClient.Object); + + RpcException rpcException = await this.InvokeFailingRpcAsync( + "DisabledOrchestratorRestart", + durabilityProvider.Object, + async client => await client.RestartInstanceAsync( + new P.RestartInstanceRequest + { + InstanceId = InstanceId, + RestartWithNewInstanceId = false, + }).ResponseAsync, + extension => extension.RegisterOrchestrator( + new FunctionName(FunctionName), + orchestratorInfo: null)); + + Assert.Equal(StatusCode.InvalidArgument, rpcException.StatusCode); + Assert.Contains("doesn't exist, is disabled, or is not an orchestrator function", rpcException.Status.Detail); + orchestrationServiceClient.Verify( + client => client.CreateTaskOrchestrationAsync( + It.IsAny(), + It.IsAny()), + Times.Never); + } + [Theory] [InlineData(false, nameof(ApplicationException))] [InlineData(true, nameof(OperationCanceledException))] @@ -978,9 +1135,11 @@ private DurableTaskExtension CreateExtension(string hubName) private async Task InvokeFailingRpcAsync( string hubName, DurabilityProvider durabilityProvider, - Func invoke) + Func invoke, + Action configureExtension = null) { using DurableTaskExtension extension = this.CreateExtension(hubName, durabilityProvider); + configureExtension?.Invoke(extension); ILocalGrpcListener listener = LocalGrpcListener.Create(extension, LocalGrpcListenerMode.AspNetCore); try @@ -996,6 +1155,26 @@ private async Task InvokeFailingRpcAsync( } } + private async Task InvokeRpcAsync( + DurabilityProvider durabilityProvider, + Func invoke) + { + using DurableTaskExtension extension = this.CreateExtension("TestHub", durabilityProvider); + ILocalGrpcListener listener = LocalGrpcListener.Create(extension, LocalGrpcListenerMode.AspNetCore); + + try + { + await listener.StartAsync(default); + using GrpcChannel channel = GrpcChannel.ForAddress(listener.ListenAddress); + var client = new P.TaskHubSidecarService.TaskHubSidecarServiceClient(channel); + await invoke(client); + } + finally + { + await listener.StopAsync(default); + } + } + private LogMessage AssertWarning(string errorMessage, string methodName, string exceptionType) { LogMessage warning = Assert.Single( @@ -1038,13 +1217,18 @@ private DurableTaskExtension CreateExtension(DurableTaskOptions options, WorkerR platformInformationService: TestHelpers.GetMockPlatformInformationService(language: runtimeType)); } - private DurableTaskExtension CreateExtension(string hubName, DurabilityProvider durabilityProvider) + private DurableTaskExtension CreateExtension( + string hubName, + DurabilityProvider durabilityProvider, + DurabilityProvider defaultDurabilityProvider = null) { var options = new DurableTaskOptions { HubName = hubName }; var wrappedOptions = new OptionsWrapper(options); var serviceFactory = new Mock(); serviceFactory.SetupGet(factory => factory.Name).Returns(AzureStorageDurabilityProviderFactory.ProviderName); - serviceFactory.Setup(factory => factory.GetDurabilityProvider()).Returns(durabilityProvider); + serviceFactory + .Setup(factory => factory.GetDurabilityProvider()) + .Returns(defaultDurabilityProvider ?? durabilityProvider); serviceFactory .Setup(factory => factory.GetDurabilityProvider(It.IsAny())) .Returns(durabilityProvider); @@ -1187,13 +1371,14 @@ private static DurabilityProvider CreateDurabilityProvider( private static Mock CreateDurabilityProviderMock( IOrchestrationService orchestrationService, - IOrchestrationServiceClient orchestrationServiceClient) + IOrchestrationServiceClient orchestrationServiceClient, + string connectionName = "TestConnection") { var durabilityProvider = new Mock( "Test", orchestrationService, orchestrationServiceClient, - "TestConnection") + connectionName) { CallBase = true, };