Redesign airlock storage account architecture

.github/actions/devcontainer_run_command/action.yml

@@ -100,6 +100,10 @@ inputs:
     description: "If False, Airlock requests will skip the malware scanning stage."
     required: false
     default: "false"
+  ENABLE_LEGACY_AIRLOCK:
+    description: "If True, deploys legacy per-stage airlock storage accounts alongside the consolidated accounts."
+    required: false
+    default: "true"
   CI_CACHE_ACR_NAME:
     description: "A secondary ACR used for caching in CI environments"
     required: true
@@ -297,6 +301,7 @@ runs:
           -e TF_VAR_stateful_resources_locked=${{ inputs.STATEFUL_RESOURCES_LOCKED }} \
           -e TF_VAR_kv_purge_protection_enabled="${{ inputs.KV_PURGE_PROTECTION_ENABLED }}" \
           -e TF_VAR_enable_airlock_malware_scanning=${{ inputs.ENABLE_AIRLOCK_MALWARE_SCANNING }} \
+          -e TF_VAR_enable_legacy_airlock=${{ inputs.ENABLE_LEGACY_AIRLOCK }} \
           -e CI_CACHE_ACR_NAME="${{ inputs.CI_CACHE_ACR_NAME }}" \
           -e TF_VAR_core_app_service_plan_sku="${{ (inputs.CORE_APP_SERVICE_PLAN_SKU != ''
             && inputs.CORE_APP_SERVICE_PLAN_SKU) || 'P1v2' }}" \

.github/dependabot.yml

@@ -56,8 +56,7 @@ updates:
         update-types: ["version-update:semver-patch"]
     open-pull-requests-limit: 0
 
-  # Dependabot's Docker manager detects Dockerfile and Containerfile substrings,
-  # so Dockerfile.tmpl base images are included by this repository-wide glob.
+  # The Docker manager includes Dockerfile.tmpl files in this repository-wide glob.
   - package-ecosystem: "docker"
     directories:
       - "**/*"

.github/scripts/build.js

@@ -123,6 +123,15 @@ async function getCommandFromComment({ core, context, github }) {
           break;
         }
 
+      case "/test-airlock":
+        {
+          const runTests = await handleTestCommand({ core, github }, parts, "airlock tests", runId, { number: prNumber, authorUsername: prAuthorUsername, repoOwner, repoName, headSha: prHeadSha, refId: prRefId, details: pr }, { username: commentUsername, link: commentLink });
+          if (runTests) {
+            command = "run-tests-airlock";
+          }
+          break;
+        }
+
       case "/test-force-approve":
         {
           command = "test-force-approve";

.github/scripts/build.test.js

@@ -438,6 +438,17 @@ describe('getCommandFromComment', () => {
         });
       });
 
+      describe(`for '/test-airlock'`, () => {
+        test(`should set command to 'run-tests-airlock'`, async () => {
+          const context = createCommentContext({
+            username: 'admin',
+            body: '/test-airlock',
+          });
+          await getCommandFromComment({ core, context, github });
+          expect(outputFor(mockCoreSetOutput, 'command')).toBe('run-tests-airlock');
+        });
+      });
+
       describe(`for '/test-backups'`, () => {
         test(`should set command to 'run-tests-backups'`, async () => {
           const context = createCommentContext({

.github/workflows/deploy_tre_reusable.yml

@@ -423,6 +423,7 @@ jobs:
           AUTO_GRANT_WORKSPACE_CONSENT: ${{ vars.AUTO_GRANT_WORKSPACE_CONSENT }}
           ENABLE_DNS_POLICY: ${{ vars.ENABLE_DNS_POLICY }}
           ALLOWED_DNS: ${{ vars.ALLOWED_DNS }}
+          ENABLE_LEGACY_AIRLOCK: ${{ vars.ENABLE_LEGACY_AIRLOCK || true }}
 
       - name: API Healthcheck
         uses: ./.github/actions/devcontainer_run_command

.github/workflows/pr_comment_bot.yml

@@ -159,7 +159,8 @@ jobs:
       needs.pr_comment.outputs.command == 'run-tests-extended' ||
       needs.pr_comment.outputs.command == 'run-tests-extended-aad' ||
       needs.pr_comment.outputs.command == 'run-tests-shared-services' ||
-      needs.pr_comment.outputs.command == 'run-tests-backups'
+      needs.pr_comment.outputs.command == 'run-tests-backups' ||
+      needs.pr_comment.outputs.command == 'run-tests-airlock'
     name: Deploy PR
     uses: ./.github/workflows/deploy_tre_reusable.yml
     permissions:
@@ -176,6 +177,7 @@ jobs:
         (needs.pr_comment.outputs.command == 'run-tests-extended-aad' && 'extended_aad') ||
         (needs.pr_comment.outputs.command == 'run-tests-shared-services' && 'shared_services') ||
         (needs.pr_comment.outputs.command == 'run-tests-backups' && 'backups') ||
+        (needs.pr_comment.outputs.command == 'run-tests-airlock' && 'airlock') ||
         (needs.pr_comment.outputs.command == 'run-tests' && '') }}
       environmentName: CICD
       E2E_TESTS_NUMBER_PROCESSES: 1

.gitignore

@@ -214,3 +214,4 @@ validation.txt
 
 /index.html
 .DS_Store
+*_old.tf

CHANGELOG.md

@@ -1,8 +1,17 @@
 <!-- markdownlint-disable MD041 -->
 ## (Unreleased)
 **BREAKING CHANGES**
+* Set `enable_legacy_airlock` explicitly to `true` in your `config.yaml`. It currently defaults to `true` but will default to `false` in a future release;
+Setting to `false` will delete existing airlock storage accounts and must only be done once all workspaces use the v2 airlock. ([#5048](https://github.com/microsoft/AzureTRE/pull/5048))
 
 ENHANCEMENTS:
+* Redesign Airlock storage to consolidated metadata-based accounts (v2), now the default for new workspaces. Legacy per-stage storage is retained behind `enable_legacy_airlock` (default `true`; sample config sets `false`). Run `POST /migrations` after upgrading to stamp pre-v2 workspaces with `airlock_version=1` so they keep routing to legacy storage.
+See [Legacy Airlock & migration](docs/azure-tre-overview/airlock.md#legacy-airlock) ([#5048](https://github.com/microsoft/AzureTRE/pull/5048))
+* Add E2E airlock coverage for the draft container seal, file count validation, rejected/cancelled lifecycles and cross-workspace access, runnable via `make test-e2e-airlock` or the `/test-airlock` PR comment ([#5048](https://github.com/microsoft/AzureTRE/pull/5048))
+
+BUG FIXES:
+* Emit airlock malware scan verdicts without reading the scanned blob, so a verdict arriving after the draft container is sealed no longer strands the request in `Submitted` ([#5048](https://github.com/microsoft/AzureTRE/pull/5048))
+* Use the cloud-specific workload identity token exchange audience so v2 airlock SAS signing works in sovereign clouds ([#5048](https://github.com/microsoft/AzureTRE/pull/5048))
 
 ## (0.29.0) (August 14, 2026)
 **BREAKING CHANGES**

Makefile

@@ -526,6 +526,14 @@ test-e2e-backups: ## 🧪 Run E2E backup tests
 	$(call target_title, "Running E2E backup tests") && \
 	$(MAKE) test-e2e-custom SELECTOR=backups
 
+# Description: Run E2E airlock tests
+# # The E2E airlock tests include:
+# # - tests marked with the `airlock` selector that verify airlock import/export flows and their access controls
+# Example: make test-e2e-airlock
+test-e2e-airlock: ## 🧪 Run E2E airlock tests
+	$(call target_title, "Running E2E airlock tests") && \
+	$(MAKE) test-e2e-custom SELECTOR=airlock
+
 # Description: Run E2E tests with custom selector
 # Arguments: SELECTOR - the selector to run the tests with
 # Example: make test-e2e-custom SELECTOR=smoke

airlock_processor/BlobCreatedTrigger/__init__.py

@@ -11,6 +11,13 @@
 from shared_code.blob_operations import get_blob_info_from_topic_and_subject, get_blob_client_from_blob_info
 
 
+# Only cross-account approval copies complete through BlobCreated events.
+V2_STAGE_COMPLETION_MAP = {
+    constants.STAGE_IMPORT_APPROVED: (constants.STAGE_APPROVAL_INPROGRESS, constants.STAGE_APPROVED),
+    constants.STAGE_EXPORT_APPROVED: (constants.STAGE_APPROVAL_INPROGRESS, constants.STAGE_APPROVED),
+}
+
+
 def main(msg: func.ServiceBusMessage,
          stepResultEvent: func.Out[func.EventGridOutputEvent],
          dataDeletionEvent: func.Out[func.EventGridOutputEvent]):
@@ -23,6 +30,10 @@ def main(msg: func.ServiceBusMessage,
     topic = json_body["topic"]
     request_id = re.search(r'/blobServices/default/containers/(.*?)/blobs', json_body["subject"]).group(1)
 
+    if constants.STORAGE_ACCOUNT_NAME_AIRLOCK_CORE in topic or constants.STORAGE_ACCOUNT_NAME_AIRLOCK_WORKSPACE_GLOBAL in topic:
+        _handle_v2_blob_created(json_body, topic, request_id, stepResultEvent, dataDeletionEvent)
+        return
+
     # message originated from in-progress blob creation
     if constants.STORAGE_ACCOUNT_NAME_IMPORT_INPROGRESS in topic or constants.STORAGE_ACCOUNT_NAME_EXPORT_INPROGRESS in topic:
         try:
@@ -55,6 +66,9 @@ def main(msg: func.ServiceBusMessage,
     elif constants.STORAGE_ACCOUNT_NAME_IMPORT_BLOCKED in topic or constants.STORAGE_ACCOUNT_NAME_EXPORT_BLOCKED in topic:
         completed_step = constants.STAGE_BLOCKING_INPROGRESS
         new_status = constants.STAGE_BLOCKED_BY_SCAN
+    else:
+        logging.warning(f"Unknown storage account in topic: {topic}")
+        return
 
     # reply with a step completed event
     stepResultEvent.set(
@@ -74,6 +88,9 @@ def send_delete_event(dataDeletionEvent: func.Out[func.EventGridOutputEvent], js
     blob_client = get_blob_client_from_blob_info(
         *get_blob_info_from_topic_and_subject(topic=json_body["topic"], subject=json_body["subject"]))
     blob_metadata = blob_client.get_blob_properties()["metadata"]
+    if "copied_from" not in blob_metadata:
+        logging.info(f"Blob for request {request_id} has no copied_from metadata; skipping data deletion event.")
+        return
     copied_from = json.loads(blob_metadata["copied_from"])
     logging.info(f"copied from history: {copied_from}")
 
@@ -88,3 +105,36 @@ def send_delete_event(dataDeletionEvent: func.Out[func.EventGridOutputEvent], js
             data_version=constants.DATA_DELETION_EVENT_DATA_VERSION
         )
     )
+
+
+def _handle_v2_blob_created(json_body, topic, request_id, stepResultEvent, dataDeletionEvent):
+    storage_account_name, _, _ = get_blob_info_from_topic_and_subject(
+        topic=json_body["topic"], subject=json_body["subject"])
+
+    from shared_code.blob_operations_metadata import get_container_metadata
+    try:
+        metadata = get_container_metadata(storage_account_name, request_id)
+    except Exception:
+        # Retry rather than acknowledge without a StepResult.
+        logging.exception(f"Could not read container metadata for request {request_id} on {storage_account_name}")
+        raise
+
+    stage = metadata.get('stage', '')
+    logging.info(f"V2 BlobCreated for request {request_id}: stage={stage}, account={storage_account_name}")
+
+    if stage in V2_STAGE_COMPLETION_MAP:
+        completed_step, new_status = V2_STAGE_COMPLETION_MAP[stage]
+        logging.info(f"V2 copy completed for request {request_id}: {completed_step} -> {new_status}")
+
+        stepResultEvent.set(
+            func.EventGridOutputEvent(
+                id=str(uuid.uuid4()),
+                data={"completed_step": completed_step, "new_status": new_status, "request_id": request_id},
+                subject=request_id,
+                event_type="Airlock.StepResult",
+                event_time=datetime.datetime.now(datetime.UTC),
+                data_version=constants.STEP_RESULT_EVENT_DATA_VERSION))
+
+        send_delete_event(dataDeletionEvent, json_body, request_id)
+    else:
+        logging.info(f"V2 BlobCreated for non-terminal stage '{stage}' on request {request_id}, no action needed")

airlock_processor/ScanResultTrigger/__init__.py

@@ -5,7 +5,7 @@
 import uuid
 import json
 import os
-from shared_code import constants, blob_operations, parsers
+from shared_code import constants, blob_operations, parsers, airlock_storage_helper
 
 
 def main(msg: func.ServiceBusMessage,
@@ -14,7 +14,6 @@ def main(msg: func.ServiceBusMessage,
     logging.info("Python ServiceBus queue trigger processed message - Malware scan result arrived!")
     body = msg.get_body().decode('utf-8')
     logging.info(f'Python ServiceBus queue trigger processed message: {body}')
-    status_message = None
 
     try:
         enable_malware_scanning = parsers.parse_bool(os.environ["ENABLE_MALWARE_SCANNING"])
@@ -39,24 +38,22 @@ def main(msg: func.ServiceBusMessage,
         raise e
 
     # Extract request id
-    _, request_id, _ = blob_operations.get_blob_info_from_blob_url(blob_url=blob_uri)
-
-    # If clean, we can continue and move the request to the review stage
-    # Otherwise, move the request to the blocked stage
-    completed_step = constants.STAGE_SUBMITTED
-    if verdict == constants.NO_THREATS:
-        logging.info(f'No malware were found in request id {request_id}, moving to {constants.STAGE_IN_REVIEW} stage')
-        new_status = constants.STAGE_IN_REVIEW
-    else:
-        logging.info(f'Malware was found in request id {request_id}, moving to {constants.STAGE_BLOCKING_INPROGRESS} stage')
-        new_status = constants.STAGE_BLOCKING_INPROGRESS
-        status_message = verdict
-
-    # Send the event to indicate this step is done (and to request a new status change)
+    account_name, container_name, blob_name = blob_operations.get_blob_info_from_blob_url(blob_url=blob_uri)
+    request_id = airlock_storage_helper.get_request_id_from_container_name(container_name)
+
+    # The draft container stays writable until submission, so its verdict may describe content that
+    # was later replaced. Only the sealed copy's verdict describes the data actually under review.
+    # Only v2 creates -draft containers, so the suffix identifies them without depending on account names.
+    if container_name.endswith(constants.DRAFT_CONTAINER_SUFFIX):
+        logging.info(f'Scan result for draft blob in request {request_id} ignored; the submitted copy gates review.')
+        return
+
+    # The verdict is reported as a fact; the API decides the status once submission is validated.
     outputEvent.set(
         func.EventGridOutputEvent(
             id=str(uuid.uuid4()),
-            data={"completed_step": completed_step, "new_status": new_status, "request_id": request_id, "status_message": status_message},
+            data={"completed_step": constants.STAGE_SUBMITTED, "request_id": request_id,
+                  "scan_result": {"clean": verdict == constants.NO_THREATS, "message": None if verdict == constants.NO_THREATS else verdict}},
             subject=request_id,
             event_type="Airlock.StepResult",
             event_time=datetime.datetime.now(datetime.UTC),