Skip to content
21 changes: 18 additions & 3 deletions src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -901,10 +901,12 @@ await this.AbandonAndReleaseSessionAsync(
session?.Instance.ExecutionId ?? string.Empty,
e.ToString());

if (orchestrationWorkItem != null)
if (session != null)
{
// The work-item needs to be released so that it can be retried later.
await this.ReleaseTaskOrchestrationWorkItemAsync(orchestrationWorkItem);
// No work item was handed off. Release the session and its messages.
await this.AbandonAndReleaseSessionAsync(
session,
"Failed to prepare the orchestration work item.");
}

throw;
Expand Down Expand Up @@ -1144,6 +1146,19 @@ await this.trackingStore.UpdateInstanceStatusForCompletedOrchestrationAsync(

async Task AbandonAndReleaseSessionAsync(OrchestrationSession session, string details)
{
// An earlier cleanup may already have released this session.
if (!this.orchestrationSessionManager.TryGetExistingSession(session.Instance.InstanceId, out OrchestrationSession currentSession) ||
!ReferenceEquals(session, currentSession))
{
return;
Comment thread
sophiatev marked this conversation as resolved.
Resolved
}

// Deferred messages stay buffered for the next session.
foreach (MessageData message in session.DeferredMessages)
{
session.CurrentMessageBatch.Remove(message);
}

try
{
await this.AbandonSessionAsync(session, details);
Expand Down
18 changes: 16 additions & 2 deletions src/DurableTask.AzureStorage/Messaging/ControlQueue.cs
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,9 @@ public ControlQueue(

protected override TimeSpan MessageVisibilityTimeout => this.settings.ControlQueueVisibilityTimeout;

public async Task<IReadOnlyList<MessageData>> GetMessagesAsync(CancellationToken cancellationToken)
public async Task<IReadOnlyList<MessageData>> GetMessagesAsync(
CancellationToken cancellationToken,
List<string>? newlyReservedMessageIds = null)
{
using (var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(this.releaseCancellationToken, cancellationToken))
{
Expand Down Expand Up @@ -75,6 +77,7 @@ public async Task<IReadOnlyList<MessageData>> GetMessagesAsync(CancellationToken
}

pendingOrchestratorMessageLimitReached = false;
var addedMessageIds = new ConcurrentBag<string>();

try
{
Expand Down Expand Up @@ -134,7 +137,11 @@ await batch.ParallelForEachAsync(async delegate (QueueMessage queueMessage)
}

// Check to see whether we've already dequeued this message.
if (!this.stats.PendingOrchestratorMessages.TryAdd(queueMessage.MessageId, 1))
if (this.stats.PendingOrchestratorMessages.TryAdd(queueMessage.MessageId, 1))
{
addedMessageIds.Add(queueMessage.MessageId);
}
else
{
// This message is already loaded in memory and is therefore a duplicate.
// We will continue to process it because we need the updated pop receipt.
Expand Down Expand Up @@ -166,10 +173,17 @@ await batch.ParallelForEachAsync(async delegate (QueueMessage queueMessage)
this.storageAccountName);
}

newlyReservedMessageIds?.AddRange(addedMessageIds);
return sortedMessages;
}
catch (Exception e)
{
// This batch was not handed off. Keep entries from earlier batches.
foreach (string messageId in addedMessageIds)
{
this.stats.PendingOrchestratorMessages.TryRemove(messageId, out _);
}

if (!linkedCts.IsCancellationRequested)
{
this.settings.Logger.MessageFailure(
Expand Down
38 changes: 30 additions & 8 deletions src/DurableTask.AzureStorage/OrchestrationSessionManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -147,15 +147,30 @@ async Task DequeueLoop(string partitionId, ControlQueue controlQueue, Cancellati
Guid traceActivityId = AzureStorageOrchestrationService.StartNewLogicalTraceScope(useExisting: false);

// This will block until either new messages arrive or the queue is released.
IReadOnlyList<MessageData> messages = await controlQueue.GetMessagesAsync(cancellationToken);
var newlyReservedMessageIds = new List<string>();
IReadOnlyList<MessageData> messages = await controlQueue.GetMessagesAsync(cancellationToken, newlyReservedMessageIds);
if (messages.Count > 0)
{
// De-dupe any execution started messages
IEnumerable<MessageData> filteredMessages = await this.DedupeExecutionStartedMessagesAsync(
controlQueue,
messages,
traceActivityId,
cancellationToken);
IEnumerable<MessageData> filteredMessages;
try
{
filteredMessages = await this.DedupeExecutionStartedMessagesAsync(
controlQueue,
messages,
traceActivityId,
cancellationToken);
}
catch
{
// Keep reservations owned by earlier copies of duplicate messages.
foreach (string messageId in newlyReservedMessageIds)
{
this.stats.PendingOrchestratorMessages.TryRemove(messageId, out _);
}

throw;
}

this.AddMessageToPendingOrchestration(controlQueue, filteredMessages, traceActivityId, cancellationToken);
}
Expand Down Expand Up @@ -566,7 +581,7 @@ async Task ScheduleOrchestrationStatePrefetch(
this.orchestrationsReadyForProcessingQueue.Enqueue(node);
}
}
catch (OperationCanceledException)
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
// shutting down
}
Expand Down Expand Up @@ -705,10 +720,17 @@ public bool TryReleaseSession(string instanceId, CancellationToken cancellationT
if (this.activeOrchestrationSessions.TryGetValue(instanceId, out session) &&
this.activeOrchestrationSessions.Remove(instanceId))
{
IEnumerable<MessageData> retainedMessages = session.PendingMessages.Concat(session.DeferredMessages);
// Abandoning a current message can remove a reservation still needed by a pending duplicate.
foreach (MessageData message in retainedMessages)
{
this.stats.PendingOrchestratorMessages.TryAdd(message.OriginalQueueMessage.MessageId, 1);
Comment thread
sophiatev marked this conversation as resolved.
Resolved
}

// Put any unprocessed messages back into the pending buffer.
this.AddMessageToPendingOrchestration(
session.ControlQueue,
session.PendingMessages.Concat(session.DeferredMessages),
retainedMessages,
session.TraceActivityId,
cancellationToken);
return true;
Expand Down
Loading
Loading