Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 33 additions & 5 deletions src/DurableTask.AzureStorage/MessageManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ namespace DurableTask.AzureStorage
using Azure.Storage.Blobs.Models;
using Azure.Storage.Queues.Models;
using DurableTask.AzureStorage.Storage;
using DurableTask.AzureStorage.Tracking;
using Newtonsoft.Json;
#if NETSTANDARD2_0
using Newtonsoft.Json.Serialization;
Expand All @@ -43,6 +44,8 @@ class MessageManager
// increasing the size by a factor of 4/3.
const int MaxStorageQueuePayloadSizeInBytes = 45 * 1024; // 45KB
const int DefaultBufferSize = 64 * 2014; // 64KB
internal const string MessageBlobKind = "message-";
internal const string HistoryBlobKind = "history-";

readonly AzureStorageOrchestrationServiceSettings settings;
readonly AzureStorageClient azureStorageClient;
Expand Down Expand Up @@ -357,22 +360,40 @@ public string GetNewLargeMessageBlobName(MessageData message)
string eventType = message.TaskMessage.Event.EventType.ToString();
string activityId = message.ActivityId.ToString("N");

return $"{instanceId}/message-{activityId}-{eventType}.json.gz";
return $"{instanceId}/{MessageBlobKind}{activityId}-{eventType}.json.gz";
}

public async Task<int> DeleteLargeMessageBlobs(string sanitizedInstanceId, CancellationToken cancellationToken = default)
public async Task<int> DeleteLargeMessageBlobs(string instanceId, CancellationToken cancellationToken = default)
{
// Message blobs are named "{instanceId}/message-..." (GetNewLargeMessageBlobName) and history blobs
// "{escaped instanceId}/history-..." (AzureTableTrackingStore.GetBlobName). Only blobs directly under
// each prefix are deleted, so instances whose id starts with this one (e.g. "abc_other" or "abc/other"
// when purging "abc") keep their blobs.
string escapedInstanceId = KeySanitation.EscapePartitionKey(instanceId);
if (escapedInstanceId == instanceId)
{
return await this.DeleteInstanceBlobs(instanceId + "/", new[] { MessageBlobKind, HistoryBlobKind }, cancellationToken);
}

int messageBlobOperations = await this.DeleteInstanceBlobs(instanceId + "/", new[] { MessageBlobKind }, cancellationToken);
int historyBlobOperations = await this.DeleteInstanceBlobs(escapedInstanceId + "/", new[] { HistoryBlobKind }, cancellationToken);
return messageBlobOperations + historyBlobOperations;
}

async Task<int> DeleteInstanceBlobs(string blobPrefix, string[] blobKinds, CancellationToken cancellationToken)
{
int storageOperationCount = 0;

try
{
await foreach (Page<Blob> page in this.blobContainer.ListBlobsAsync(sanitizedInstanceId, cancellationToken).AsPages())
await foreach (Page<Blob> page in this.blobContainer.ListBlobsAsync(blobPrefix, cancellationToken).AsPages())
{
storageOperationCount++;

await Task.WhenAll(page.Values.Select(b => b.DeleteIfExistsAsync(cancellationToken)));
Blob[] instanceBlobs = page.Values.Where(b => IsInstanceBlob(b.Name, blobPrefix, blobKinds)).ToArray();
await Task.WhenAll(instanceBlobs.Select(b => b.DeleteIfExistsAsync(cancellationToken)));

storageOperationCount += page.Values.Count;
storageOperationCount += instanceBlobs.Length;
}

// Count the list operation even if no blobs found (the initial list request still happened)
Expand All @@ -395,6 +416,13 @@ public async Task<int> DeleteLargeMessageBlobs(string sanitizedInstanceId, Cance
return storageOperationCount;
}

static bool IsInstanceBlob(string blobName, string blobPrefix, string[] blobKinds)
{
string relativeName = blobName.Substring(blobPrefix.Length);
return relativeName.IndexOf('/') < 0
&& blobKinds.Any(kind => relativeName.StartsWith(kind, StringComparison.Ordinal));
}

/// Escapes a blob name for safe inclusion in a URI while preserving path structure.
static string EscapeBlobNamePreservingSlashes(string blobName)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1483,7 +1483,7 @@ static string GetBlobName(TableEntity entity, string property)
// randomize the blob name to prevent accidental races in split-brain situations (#890)
uint random = (uint)(new Random()).Next();

return $"{sanitizedInstanceId}/history-{sequenceNumber}-{eventType}-{random:X8}-{property}.json.gz";
return $"{sanitizedInstanceId}/{MessageManager.HistoryBlobKind}{sequenceNumber}-{eventType}-{random:X8}-{property}.json.gz";
}

async Task<ETag?> GetHistoryETagAsync(string instanceId, string executionId, CancellationToken cancellationToken)
Expand Down
89 changes: 89 additions & 0 deletions test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -788,6 +788,95 @@ public async Task PurgeSingleInstance_WithLargeBlobs_CleansUpBlobs()
}
}

[DataTestMethod]
[DataRow("_other")]
[DataRow(":12")]
[DataRow("/other")]
public async Task PurgeSingleInstance_DoesNotDeleteBlobsOfInstanceWithPrefixedId(string otherIdSuffix)
{
using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false))
{
await host.StartAsync();

string instanceId = Guid.NewGuid().ToString("N");
string otherInstanceId = instanceId + otherIdSuffix;
string largeMessage = new string('x', 70 * 1024);

TestOrchestrationClient client = await host.StartOrchestrationAsync(
typeof(Orchestrations.Echo), largeMessage, instanceId);
OrchestrationState status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2));
Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus);

TestOrchestrationClient otherClient = await host.StartOrchestrationAsync(
typeof(Orchestrations.Echo), largeMessage, otherInstanceId);
status = await otherClient.WaitForCompletionAsync(TimeSpan.FromMinutes(2));
Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus);

(int otherMessageBlobs, int otherHistoryBlobs) = await this.GetInstanceBlobCounts(otherInstanceId);
Assert.IsTrue(otherMessageBlobs > 0, "Should have large message blobs before purge");
Assert.IsTrue(otherHistoryBlobs > 0, "Should have large history blobs before purge");

await client.PurgeInstanceHistory();

Assert.AreEqual((0, 0), await this.GetInstanceBlobCounts(instanceId));
Assert.AreEqual((otherMessageBlobs, otherHistoryBlobs), await this.GetInstanceBlobCounts(otherInstanceId));

// Loading the history downloads the large-property blobs, as a replay does.
List<HistoryStateEvent> otherHistoryEvents = await otherClient.GetOrchestrationHistoryAsync(otherInstanceId);
Assert.IsTrue(otherHistoryEvents.Count > 0);

await host.StopAsync();
}
}

[DataTestMethod]
[DataRow("^")]
[DataRow("/nested")]
public async Task PurgeSingleInstance_WithEscapedInstanceId_DeletesAllBlobs(string idSuffix)
{
using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false))
{
await host.StartAsync();

string instanceId = Guid.NewGuid().ToString("N") + idSuffix;
string largeMessage = new string('x', 70 * 1024);

TestOrchestrationClient client = await host.StartOrchestrationAsync(
typeof(Orchestrations.Echo), largeMessage, instanceId);
OrchestrationState status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2));
Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus);

(int messageBlobs, int historyBlobs) = await this.GetInstanceBlobCounts(instanceId);
Assert.IsTrue(messageBlobs > 0, "Should have large message blobs before purge");
Assert.IsTrue(historyBlobs > 0, "Should have large history blobs before purge");

await client.PurgeInstanceHistory();

Assert.AreEqual((0, 0), await this.GetInstanceBlobCounts(instanceId));

await host.StopAsync();
}
}

// Counts the blobs that belong to exactly this instance: message blobs are named after the instance id,
// history blobs after the escaped instance id, and blobs of longer ids nested below them are not counted.
async Task<(int MessageBlobs, int HistoryBlobs)> GetInstanceBlobCounts(string instanceId)
{
var containerClient = new BlobServiceClient(TestHelpers.GetTestStorageAccountConnectionString())
.GetBlobContainerClient("test-largemessages");
await containerClient.CreateIfNotExistsAsync();

async Task<int> CountAsync(string prefix)
{
return await containerClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix, CancellationToken.None)
.CountAsync(b => b.Name.IndexOf('/', prefix.Length) < 0);
}

return (
await CountAsync(instanceId + "/message-"),
await CountAsync(KeySanitation.EscapePartitionKey(instanceId) + "/history-"));
}

[TestMethod]
public async Task PurgeInstance_WithManyHistoryRows_DeletesAll()
{
Expand Down