diff --git a/.github/actions/devcontainer_run_command/action.yml b/.github/actions/devcontainer_run_command/action.yml index c6673f8aae..3f86e90a5b 100644 --- a/.github/actions/devcontainer_run_command/action.yml +++ b/.github/actions/devcontainer_run_command/action.yml @@ -100,6 +100,10 @@ inputs: description: "If False, Airlock requests will skip the malware scanning stage." required: false default: "false" + ENABLE_LEGACY_AIRLOCK: + description: "If True, deploys legacy per-stage airlock storage accounts alongside the consolidated accounts." + required: false + default: "true" CI_CACHE_ACR_NAME: description: "A secondary ACR used for caching in CI environments" required: true @@ -297,6 +301,7 @@ runs: -e TF_VAR_stateful_resources_locked=${{ inputs.STATEFUL_RESOURCES_LOCKED }} \ -e TF_VAR_kv_purge_protection_enabled="${{ inputs.KV_PURGE_PROTECTION_ENABLED }}" \ -e TF_VAR_enable_airlock_malware_scanning=${{ inputs.ENABLE_AIRLOCK_MALWARE_SCANNING }} \ + -e TF_VAR_enable_legacy_airlock=${{ inputs.ENABLE_LEGACY_AIRLOCK }} \ -e CI_CACHE_ACR_NAME="${{ inputs.CI_CACHE_ACR_NAME }}" \ -e TF_VAR_core_app_service_plan_sku="${{ (inputs.CORE_APP_SERVICE_PLAN_SKU != '' && inputs.CORE_APP_SERVICE_PLAN_SKU) || 'P1v2' }}" \ diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1a7dba260f..2420da5659 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -56,8 +56,7 @@ updates: update-types: ["version-update:semver-patch"] open-pull-requests-limit: 0 - # Dependabot's Docker manager detects Dockerfile and Containerfile substrings, - # so Dockerfile.tmpl base images are included by this repository-wide glob. + # The Docker manager includes Dockerfile.tmpl files in this repository-wide glob. - package-ecosystem: "docker" directories: - "**/*" diff --git a/.github/scripts/build.js b/.github/scripts/build.js index 3e791dbc57..b2c247755d 100644 --- a/.github/scripts/build.js +++ b/.github/scripts/build.js @@ -123,6 +123,15 @@ async function getCommandFromComment({ core, context, github }) { break; } + case "/test-airlock": + { + const runTests = await handleTestCommand({ core, github }, parts, "airlock tests", runId, { number: prNumber, authorUsername: prAuthorUsername, repoOwner, repoName, headSha: prHeadSha, refId: prRefId, details: pr }, { username: commentUsername, link: commentLink }); + if (runTests) { + command = "run-tests-airlock"; + } + break; + } + case "/test-force-approve": { command = "test-force-approve"; diff --git a/.github/scripts/build.test.js b/.github/scripts/build.test.js index 95e6b9b970..92eb20364e 100644 --- a/.github/scripts/build.test.js +++ b/.github/scripts/build.test.js @@ -438,6 +438,17 @@ describe('getCommandFromComment', () => { }); }); + describe(`for '/test-airlock'`, () => { + test(`should set command to 'run-tests-airlock'`, async () => { + const context = createCommentContext({ + username: 'admin', + body: '/test-airlock', + }); + await getCommandFromComment({ core, context, github }); + expect(outputFor(mockCoreSetOutput, 'command')).toBe('run-tests-airlock'); + }); + }); + describe(`for '/test-backups'`, () => { test(`should set command to 'run-tests-backups'`, async () => { const context = createCommentContext({ diff --git a/.github/workflows/deploy_tre_reusable.yml b/.github/workflows/deploy_tre_reusable.yml index 2141aea5f5..e725d7805d 100644 --- a/.github/workflows/deploy_tre_reusable.yml +++ b/.github/workflows/deploy_tre_reusable.yml @@ -423,6 +423,7 @@ jobs: AUTO_GRANT_WORKSPACE_CONSENT: ${{ vars.AUTO_GRANT_WORKSPACE_CONSENT }} ENABLE_DNS_POLICY: ${{ vars.ENABLE_DNS_POLICY }} ALLOWED_DNS: ${{ vars.ALLOWED_DNS }} + ENABLE_LEGACY_AIRLOCK: ${{ vars.ENABLE_LEGACY_AIRLOCK || true }} - name: API Healthcheck uses: ./.github/actions/devcontainer_run_command diff --git a/.github/workflows/pr_comment_bot.yml b/.github/workflows/pr_comment_bot.yml index a2d5e140c7..7d4989a02d 100644 --- a/.github/workflows/pr_comment_bot.yml +++ b/.github/workflows/pr_comment_bot.yml @@ -159,7 +159,8 @@ jobs: needs.pr_comment.outputs.command == 'run-tests-extended' || needs.pr_comment.outputs.command == 'run-tests-extended-aad' || needs.pr_comment.outputs.command == 'run-tests-shared-services' || - needs.pr_comment.outputs.command == 'run-tests-backups' + needs.pr_comment.outputs.command == 'run-tests-backups' || + needs.pr_comment.outputs.command == 'run-tests-airlock' name: Deploy PR uses: ./.github/workflows/deploy_tre_reusable.yml permissions: @@ -176,6 +177,7 @@ jobs: (needs.pr_comment.outputs.command == 'run-tests-extended-aad' && 'extended_aad') || (needs.pr_comment.outputs.command == 'run-tests-shared-services' && 'shared_services') || (needs.pr_comment.outputs.command == 'run-tests-backups' && 'backups') || + (needs.pr_comment.outputs.command == 'run-tests-airlock' && 'airlock') || (needs.pr_comment.outputs.command == 'run-tests' && '') }} environmentName: CICD E2E_TESTS_NUMBER_PROCESSES: 1 diff --git a/.gitignore b/.gitignore index 57359aa4ca..ce7f21d2bd 100644 --- a/.gitignore +++ b/.gitignore @@ -214,3 +214,4 @@ validation.txt /index.html .DS_Store +*_old.tf diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a57a4c1ca..e41e2e0b65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,24 @@ ## (Unreleased) **BREAKING CHANGES** +* Set `enable_legacy_airlock` explicitly to `true` in your `config.yaml`. It currently defaults to `true` but will default to `false` in a future release; +Setting to `false` will delete existing airlock storage accounts and must only be done once all workspaces use the v2 airlock. ([#5048](https://github.com/microsoft/AzureTRE/pull/5048)) ENHANCEMENTS: +* Redesign Airlock storage to consolidated metadata-based accounts (v2), now the default for new workspaces. Legacy per-stage storage is retained behind `enable_legacy_airlock` (default `true`; sample config sets `false`). Existing workspaces upgrade in place and stay on `airlock_version=1` (a minor, non-destructive `tre-workspace-base` + upgrade to `2.11.0`); run `POST /migrations` after upgrading to stamp pre-v2 workspaces with `airlock_version=1`, then opt into v2 per workspace by patching `airlock_version=2`. +See [Legacy Airlock & migration](docs/azure-tre-overview/airlock.md#legacy-airlock) ([#5048](https://github.com/microsoft/AzureTRE/pull/5048)) +* Add E2E airlock coverage for the draft container seal, file count validation, rejected/cancelled lifecycles and cross-workspace access, runnable via `make test-e2e-airlock` or the `/test-airlock` PR comment ([#5048](https://github.com/microsoft/AzureTRE/pull/5048)) + +BUG FIXES: +* Derive a new workspace's `airlock_version` from its template rather than always defaulting to 2, so a workspace created from a legacy (pre-v2) template is correctly stamped `airlock_version=1` instead of being marked v2 while deploying v1 storage. The version guard now validates the resolved value, and the "missing means legacy (1)" default is applied consistently ([#5048](https://github.com/microsoft/AzureTRE/pull/5048)) +* Remove incorrect Terraform `moved` blocks for the legacy airlock role assignments and Defender action, which already used `count` on `main` (moving an unindexed address to `[0]` was a no-op at best and misleading) ([#5048](https://github.com/microsoft/AzureTRE/pull/5048)) +* Raise a clear error when an Event Grid topic/subject or blob URL can't be parsed (including the BlobCreated trigger), and fix a malformed log statement, so unexpected airlock scan/blob events are diagnosable instead of failing with an opaque `NoneType` error (`airlock-processor` 0.8.31) ([#5048](https://github.com/microsoft/AzureTRE/pull/5048)) +* Fail an airlock submission cleanly with `NoDataInRequestException` when neither the draft nor the sealed container exists, instead of raising `ResourceNotFoundError` that leaves the request retrying/dead-lettered while stuck in `Submitted` (`airlock-processor` 0.8.31) ([#5048](https://github.com/microsoft/AzureTRE/pull/5048)) +* Stop logging the full blob URL (including the SAS query string) in the airlock E2E upload/delete helpers, so short-lived read/write/delete credentials aren't exposed in test logs ([#5048](https://github.com/microsoft/AzureTRE/pull/5048)) +* Reject creating an airlock request on a legacy (`airlock_version=1`) workspace when `enable_legacy_airlock=false`, instead of letting it silently stall in `Submitted` because the legacy storage no longer exists ([#5048](https://github.com/microsoft/AzureTRE/pull/5048)) +* Emit airlock malware scan verdicts without reading the scanned blob, so a verdict arriving after the draft container is sealed no longer strands the request in `Submitted` ([#5048](https://github.com/microsoft/AzureTRE/pull/5048)) +* Use the cloud-specific workload identity token exchange audience so v2 airlock SAS signing works in sovereign clouds ([#5048](https://github.com/microsoft/AzureTRE/pull/5048)) ## (0.29.0) (August 14, 2026) **BREAKING CHANGES** diff --git a/Makefile b/Makefile index c315a8795b..4aa5a12d07 100644 --- a/Makefile +++ b/Makefile @@ -526,6 +526,14 @@ test-e2e-backups: ## 🧪 Run E2E backup tests $(call target_title, "Running E2E backup tests") && \ $(MAKE) test-e2e-custom SELECTOR=backups +# Description: Run E2E airlock tests +# # The E2E airlock tests include: +# # - tests marked with the `airlock` selector that verify airlock import/export flows and their access controls +# Example: make test-e2e-airlock +test-e2e-airlock: ## 🧪 Run E2E airlock tests + $(call target_title, "Running E2E airlock tests") && \ + $(MAKE) test-e2e-custom SELECTOR=airlock + # Description: Run E2E tests with custom selector # Arguments: SELECTOR - the selector to run the tests with # Example: make test-e2e-custom SELECTOR=smoke diff --git a/airlock_processor/BlobCreatedTrigger/__init__.py b/airlock_processor/BlobCreatedTrigger/__init__.py index f119ad3eda..9f813f7193 100644 --- a/airlock_processor/BlobCreatedTrigger/__init__.py +++ b/airlock_processor/BlobCreatedTrigger/__init__.py @@ -2,7 +2,6 @@ import datetime import uuid import json -import re import os import azure.functions as func @@ -11,6 +10,13 @@ from shared_code.blob_operations import get_blob_info_from_topic_and_subject, get_blob_client_from_blob_info +# Only cross-account approval copies complete through BlobCreated events. +V2_STAGE_COMPLETION_MAP = { + constants.STAGE_IMPORT_APPROVED: (constants.STAGE_APPROVAL_INPROGRESS, constants.STAGE_APPROVED), + constants.STAGE_EXPORT_APPROVED: (constants.STAGE_APPROVAL_INPROGRESS, constants.STAGE_APPROVED), +} + + def main(msg: func.ServiceBusMessage, stepResultEvent: func.Out[func.EventGridOutputEvent], dataDeletionEvent: func.Out[func.EventGridOutputEvent]): @@ -21,7 +27,12 @@ def main(msg: func.ServiceBusMessage, json_body = json.loads(body) topic = json_body["topic"] - request_id = re.search(r'/blobServices/default/containers/(.*?)/blobs', json_body["subject"]).group(1) + # Parse through the shared helper so a malformed topic/subject raises a clear ValueError. + _, request_id, _ = get_blob_info_from_topic_and_subject(topic=topic, subject=json_body["subject"]) + + if constants.STORAGE_ACCOUNT_NAME_AIRLOCK_CORE in topic or constants.STORAGE_ACCOUNT_NAME_AIRLOCK_WORKSPACE_GLOBAL in topic: + _handle_v2_blob_created(json_body, topic, request_id, stepResultEvent, dataDeletionEvent) + return # message originated from in-progress blob creation if constants.STORAGE_ACCOUNT_NAME_IMPORT_INPROGRESS in topic or constants.STORAGE_ACCOUNT_NAME_EXPORT_INPROGRESS in topic: @@ -55,6 +66,9 @@ def main(msg: func.ServiceBusMessage, elif constants.STORAGE_ACCOUNT_NAME_IMPORT_BLOCKED in topic or constants.STORAGE_ACCOUNT_NAME_EXPORT_BLOCKED in topic: completed_step = constants.STAGE_BLOCKING_INPROGRESS new_status = constants.STAGE_BLOCKED_BY_SCAN + else: + logging.warning(f"Unknown storage account in topic: {topic}") + return # reply with a step completed event stepResultEvent.set( @@ -74,6 +88,9 @@ def send_delete_event(dataDeletionEvent: func.Out[func.EventGridOutputEvent], js blob_client = get_blob_client_from_blob_info( *get_blob_info_from_topic_and_subject(topic=json_body["topic"], subject=json_body["subject"])) blob_metadata = blob_client.get_blob_properties()["metadata"] + if "copied_from" not in blob_metadata: + logging.info(f"Blob for request {request_id} has no copied_from metadata; skipping data deletion event.") + return copied_from = json.loads(blob_metadata["copied_from"]) logging.info(f"copied from history: {copied_from}") @@ -88,3 +105,36 @@ def send_delete_event(dataDeletionEvent: func.Out[func.EventGridOutputEvent], js data_version=constants.DATA_DELETION_EVENT_DATA_VERSION ) ) + + +def _handle_v2_blob_created(json_body, topic, request_id, stepResultEvent, dataDeletionEvent): + storage_account_name, _, _ = get_blob_info_from_topic_and_subject( + topic=json_body["topic"], subject=json_body["subject"]) + + from shared_code.blob_operations_metadata import get_container_metadata + try: + metadata = get_container_metadata(storage_account_name, request_id) + except Exception: + # Retry rather than acknowledge without a StepResult. + logging.exception(f"Could not read container metadata for request {request_id} on {storage_account_name}") + raise + + stage = metadata.get('stage', '') + logging.info(f"V2 BlobCreated for request {request_id}: stage={stage}, account={storage_account_name}") + + if stage in V2_STAGE_COMPLETION_MAP: + completed_step, new_status = V2_STAGE_COMPLETION_MAP[stage] + logging.info(f"V2 copy completed for request {request_id}: {completed_step} -> {new_status}") + + stepResultEvent.set( + func.EventGridOutputEvent( + id=str(uuid.uuid4()), + data={"completed_step": completed_step, "new_status": new_status, "request_id": request_id}, + subject=request_id, + event_type="Airlock.StepResult", + event_time=datetime.datetime.now(datetime.UTC), + data_version=constants.STEP_RESULT_EVENT_DATA_VERSION)) + + send_delete_event(dataDeletionEvent, json_body, request_id) + else: + logging.info(f"V2 BlobCreated for non-terminal stage '{stage}' on request {request_id}, no action needed") diff --git a/airlock_processor/ScanResultTrigger/__init__.py b/airlock_processor/ScanResultTrigger/__init__.py index 1e4ffa4274..8c06d27dc0 100644 --- a/airlock_processor/ScanResultTrigger/__init__.py +++ b/airlock_processor/ScanResultTrigger/__init__.py @@ -5,7 +5,7 @@ import uuid import json import os -from shared_code import constants, blob_operations, parsers +from shared_code import constants, blob_operations, parsers, airlock_storage_helper def main(msg: func.ServiceBusMessage, @@ -14,7 +14,6 @@ def main(msg: func.ServiceBusMessage, logging.info("Python ServiceBus queue trigger processed message - Malware scan result arrived!") body = msg.get_body().decode('utf-8') logging.info(f'Python ServiceBus queue trigger processed message: {body}') - status_message = None try: enable_malware_scanning = parsers.parse_bool(os.environ["ENABLE_MALWARE_SCANNING"]) @@ -35,28 +34,26 @@ def main(msg: func.ServiceBusMessage, blob_uri = json_body["data"]["blobUri"] verdict = json_body["data"]["scanResultType"] except KeyError as e: - logging.error("body was not as expected {}", e) + logging.error("body was not as expected: %s", e) raise e # Extract request id - _, request_id, _ = blob_operations.get_blob_info_from_blob_url(blob_url=blob_uri) - - # If clean, we can continue and move the request to the review stage - # Otherwise, move the request to the blocked stage - completed_step = constants.STAGE_SUBMITTED - if verdict == constants.NO_THREATS: - logging.info(f'No malware were found in request id {request_id}, moving to {constants.STAGE_IN_REVIEW} stage') - new_status = constants.STAGE_IN_REVIEW - else: - logging.info(f'Malware was found in request id {request_id}, moving to {constants.STAGE_BLOCKING_INPROGRESS} stage') - new_status = constants.STAGE_BLOCKING_INPROGRESS - status_message = verdict - - # Send the event to indicate this step is done (and to request a new status change) + account_name, container_name, blob_name = blob_operations.get_blob_info_from_blob_url(blob_url=blob_uri) + request_id = airlock_storage_helper.get_request_id_from_container_name(container_name) + + # The draft container stays writable until submission, so its verdict may describe content that + # was later replaced. Only the sealed copy's verdict describes the data actually under review. + # Only v2 creates -draft containers, so the suffix identifies them without depending on account names. + if container_name.endswith(constants.DRAFT_CONTAINER_SUFFIX): + logging.info(f'Scan result for draft blob in request {request_id} ignored; the submitted copy gates review.') + return + + # The verdict is reported as a fact; the API decides the status once submission is validated. outputEvent.set( func.EventGridOutputEvent( id=str(uuid.uuid4()), - data={"completed_step": completed_step, "new_status": new_status, "request_id": request_id, "status_message": status_message}, + data={"completed_step": constants.STAGE_SUBMITTED, "request_id": request_id, + "scan_result": {"clean": verdict == constants.NO_THREATS, "message": None if verdict == constants.NO_THREATS else verdict}}, subject=request_id, event_type="Airlock.StepResult", event_time=datetime.datetime.now(datetime.UTC), diff --git a/airlock_processor/StatusChangedQueueTrigger/__init__.py b/airlock_processor/StatusChangedQueueTrigger/__init__.py index 9dc6ef3aa9..480e7f5894 100644 --- a/airlock_processor/StatusChangedQueueTrigger/__init__.py +++ b/airlock_processor/StatusChangedQueueTrigger/__init__.py @@ -7,9 +7,9 @@ import uuid import json -from exceptions import NoFilesInRequestException, TooManyFilesInRequestException +from exceptions import NoFilesInRequestException, TooManyFilesInRequestException, NoDataInRequestException -from shared_code import blob_operations, constants +from shared_code import blob_operations, constants, airlock_storage_helper, parsers from pydantic import BaseModel, TypeAdapter @@ -19,6 +19,9 @@ class RequestProperties(BaseModel): previous_status: Optional[str] = None type: str workspace_id: str + review_workspace_id: Optional[str] = None + # Versionless events are pre-v2 and therefore legacy. + airlock_version: int = 1 class ContainersCopyMetadata: @@ -43,8 +46,14 @@ def main(msg: func.ServiceBusMessage, stepResultEvent: func.Out[func.EventGridOu set_output_event_to_report_failure(stepResultEvent, request_properties, failure_reason=constants.NO_FILES_IN_REQUEST_MESSAGE, request_files=request_files) except TooManyFilesInRequestException: set_output_event_to_report_failure(stepResultEvent, request_properties, failure_reason=constants.TOO_MANY_FILES_IN_REQUEST_MESSAGE, request_files=request_files) + except NoDataInRequestException: + set_output_event_to_report_failure(stepResultEvent, request_properties, failure_reason=constants.NO_DATA_IN_REQUEST_MESSAGE, request_files=request_files) except Exception: - set_output_event_to_report_failure(stepResultEvent, request_properties, failure_reason=constants.UNKNOWN_REASON_MESSAGE, request_files=request_files) + # Only the deterministic validation failures above should fail the request. Anything else may be + # transient (throttling, identity propagation, DNS, copy polling), so let it escape to be retried + # by Service Bus and dead-lettered after maxDeliveryCount, rather than silently failing the request. + logging.exception("Unexpected error processing airlock request; leaving message for Service Bus retry") + raise def handle_status_changed(request_properties: RequestProperties, stepResultEvent: func.Out[func.EventGridOutputEvent], dataDeletionEvent: func.Out[func.EventGridOutputEvent], request_files): @@ -56,26 +65,112 @@ def handle_status_changed(request_properties: RequestProperties, stepResultEvent logging.info('Processing request with id %s. new status is "%s", type is "%s"', req_id, new_status, request_type) + use_metadata = request_properties.airlock_version >= 2 + if new_status == constants.STAGE_DRAFT: - account_name = get_storage_account(status=constants.STAGE_DRAFT, request_type=request_type, short_workspace_id=ws_id) - blob_operations.create_container(account_name, req_id) + if use_metadata: + from shared_code.blob_operations_metadata import create_container_with_metadata + account_name = airlock_storage_helper.get_storage_account_name_for_request(request_type, new_status) + stage = airlock_storage_helper.get_stage_from_status(request_type, new_status) + draft_container = airlock_storage_helper.get_container_name_for_request(req_id, new_status) + create_container_with_metadata(account_name, draft_container, stage, workspace_id=ws_id, request_type=request_type) + else: + account_name = get_storage_account(status=constants.STAGE_DRAFT, request_type=request_type, short_workspace_id=ws_id) + blob_operations.create_container(account_name, req_id) return if new_status == constants.STAGE_CANCELLED: - storage_account_name = get_storage_account(previous_status, request_type, ws_id) - container_to_delete_url = blob_operations.get_blob_url(account_name=storage_account_name, container_name=req_id) + if use_metadata: + storage_account_name = airlock_storage_helper.get_storage_account_name_for_request(request_type, previous_status) + container_name = airlock_storage_helper.get_container_name_for_request(req_id, previous_status) + else: + storage_account_name = get_storage_account(previous_status, request_type, ws_id) + container_name = req_id + container_to_delete_url = blob_operations.get_blob_url(account_name=storage_account_name, container_name=container_name) set_output_event_to_trigger_container_deletion(dataDeletionEvent, request_properties, container_url=container_to_delete_url) return if new_status == constants.STAGE_SUBMITTED: - set_output_event_to_report_request_files(stepResultEvent, request_properties, request_files) + # v2 submit does not copy, so enforce the single-file rule explicitly. + if not request_files: + raise NoFilesInRequestException(constants.NO_FILES_IN_REQUEST_MESSAGE) + if len(request_files) > 1: + raise TooManyFilesInRequestException(constants.TOO_MANY_FILES_IN_REQUEST_MESSAGE) if (is_require_data_copy(new_status)): - logging.info('Request with id %s. requires data copy between storage accounts', req_id) - containers_metadata = get_source_dest_for_copy(new_status=new_status, previous_status=previous_status, request_type=request_type, short_workspace_id=ws_id) - blob_operations.create_container(containers_metadata.dest_account_name, req_id) - blob_operations.copy_data(containers_metadata.source_account_name, - containers_metadata.dest_account_name, req_id) + if use_metadata: + from shared_code.blob_operations_metadata import update_container_stage, create_container_with_metadata + + source_account = airlock_storage_helper.get_storage_account_name_for_request(request_type, previous_status) + dest_account = airlock_storage_helper.get_storage_account_name_for_request(request_type, new_status) + new_stage = airlock_storage_helper.get_stage_from_status(request_type, new_status) + + if source_account == dest_account: + if new_status == constants.STAGE_SUBMITTED: + # Copy out of the draft container and delete it, so any SAS already issued + # is revoked structurally rather than by an eventually-consistent condition. + draft_container = airlock_storage_helper.get_container_name_for_request(req_id, previous_status) + if blob_operations.container_exists(source_account, draft_container): + logging.info(f'Request {req_id}: Sealing submission - copying {draft_container} to {req_id}') + create_container_with_metadata(dest_account, req_id, new_stage, workspace_id=ws_id, request_type=request_type) + blob_operations.copy_data(source_account, dest_account, req_id, + source_container=draft_container, destination_container=req_id) + blob_operations.delete_container(source_account, draft_container) + elif blob_operations.container_exists(dest_account, req_id): + # A redelivery after the draft was deleted but before the result was published: + # the data is already sealed, so resume by re-emitting the completion event. + logging.info(f'Request {req_id}: already sealed, re-emitting the submission result') + else: + raise NoDataInRequestException(f'Request {req_id}: neither the draft nor the sealed container exists, cannot complete submission') + + try: + enable_malware_scanning = parsers.parse_bool(os.environ["ENABLE_MALWARE_SCANNING"]) + except KeyError: + logging.error("environment variable 'ENABLE_MALWARE_SCANNING' does not exist. Cannot continue.") + raise + if not enable_malware_scanning: + logging.info(f'Request {req_id}: Malware scanning disabled, skipping to in_review') + stepResultEvent.set( + func.EventGridOutputEvent( + id=str(uuid.uuid4()), + data={"completed_step": constants.STAGE_SUBMITTED, "new_status": constants.STAGE_IN_REVIEW, "request_id": req_id, "request_files": request_files}, + subject=req_id, + event_type="Airlock.StepResult", + event_time=datetime.datetime.now(datetime.UTC), + data_version=constants.STEP_RESULT_EVENT_DATA_VERSION)) + else: + logging.info(f'Request {req_id}: Malware scanning enabled, scan result gates the move to in_review') + set_output_event_to_report_request_files(stepResultEvent, request_properties, request_files) + return + + logging.info(f'Request {req_id}: Updating container stage to {new_stage} (no copy needed)') + update_container_stage(source_account, req_id, new_stage, changed_by='system') + + if new_status in [constants.STAGE_REJECTION_INPROGRESS, constants.STAGE_BLOCKING_INPROGRESS]: + final_status = constants.STAGE_REJECTED if new_status == constants.STAGE_REJECTION_INPROGRESS else constants.STAGE_BLOCKED_BY_SCAN + logging.info(f'Request {req_id}: Emitting StepResult for terminal transition {new_status} -> {final_status}') + stepResultEvent.set( + func.EventGridOutputEvent( + id=str(uuid.uuid4()), + data={"completed_step": new_status, "new_status": final_status, "request_id": req_id}, + subject=req_id, + event_type="Airlock.StepResult", + event_time=datetime.datetime.now(datetime.UTC), + data_version=constants.STEP_RESULT_EVENT_DATA_VERSION)) + else: + # BlobCreatedTrigger reports cross-account copy completion. + logging.info(f'Request {req_id}: Copying from {source_account} to {dest_account}') + create_container_with_metadata(dest_account, req_id, new_stage, workspace_id=ws_id, request_type=request_type) + blob_operations.copy_data(source_account, dest_account, req_id) + else: + logging.info('Request with id %s. requires data copy between storage accounts', req_id) + review_ws_id = request_properties.review_workspace_id + containers_metadata = get_source_dest_for_copy(new_status=new_status, previous_status=previous_status, request_type=request_type, short_workspace_id=ws_id, review_workspace_id=review_ws_id) + blob_operations.create_container(containers_metadata.dest_account_name, req_id) + blob_operations.copy_data(containers_metadata.source_account_name, + containers_metadata.dest_account_name, req_id) + if new_status == constants.STAGE_SUBMITTED: + set_output_event_to_report_request_files(stepResultEvent, request_properties, request_files) return # Other statuses which do not require data copy are dismissed as we don't need to do anything... @@ -105,7 +200,7 @@ def is_require_data_copy(new_status: str): return False -def get_source_dest_for_copy(new_status: str, previous_status: str, request_type: str, short_workspace_id: str) -> ContainersCopyMetadata: +def get_source_dest_for_copy(new_status: str, previous_status: str, request_type: str, short_workspace_id: str, review_workspace_id: str = None) -> ContainersCopyMetadata: # sanity if is_require_data_copy(new_status) is False: raise Exception("Given new status is not supported") @@ -118,7 +213,7 @@ def get_source_dest_for_copy(new_status: str, previous_status: str, request_type raise Exception(msg) source_account_name = get_storage_account(previous_status, request_type, short_workspace_id) - dest_account_name = get_storage_account_destination_for_copy(new_status, request_type, short_workspace_id) + dest_account_name = get_storage_account_destination_for_copy(new_status, request_type, short_workspace_id, review_workspace_id=review_workspace_id) return ContainersCopyMetadata(source_account_name, dest_account_name) @@ -154,11 +249,12 @@ def get_storage_account(status: str, request_type: str, short_workspace_id: str) raise Exception(error_message) -def get_storage_account_destination_for_copy(new_status: str, request_type: str, short_workspace_id: str) -> str: +def get_storage_account_destination_for_copy(new_status: str, request_type: str, short_workspace_id: str, review_workspace_id: str = None) -> str: tre_id = _get_tre_id() if request_type == constants.IMPORT_TYPE: if new_status == constants.STAGE_SUBMITTED: + # review_workspace_id must not affect the v1 account. return constants.STORAGE_ACCOUNT_NAME_IMPORT_INPROGRESS + tre_id elif new_status == constants.STAGE_APPROVAL_INPROGRESS: return constants.STORAGE_ACCOUNT_NAME_IMPORT_APPROVED + short_workspace_id @@ -228,8 +324,21 @@ def set_output_event_to_trigger_container_deletion(dataDeletionEvent, request_pr def get_request_files(request_properties: RequestProperties): - storage_account_name = get_storage_account(request_properties.previous_status, request_properties.type, request_properties.workspace_id) - return blob_operations.get_request_files(account_name=storage_account_name, request_id=request_properties.request_id) + use_metadata = request_properties.airlock_version >= 2 + container_name = None + if use_metadata: + storage_account_name = airlock_storage_helper.get_storage_account_name_for_request(request_properties.type, request_properties.previous_status) + container_name = airlock_storage_helper.get_container_name_for_request(request_properties.request_id, request_properties.previous_status) + # On a redelivery the draft is already sealed away, so enumerate the submitted copy instead. + if not blob_operations.container_exists(storage_account_name, container_name): + container_name = request_properties.request_id + # Neither container present means there is no data to submit; fail cleanly rather than + # letting a ResourceNotFoundError escape to Service Bus retry/dead-letter (stuck in Submitted). + if not blob_operations.container_exists(storage_account_name, container_name): + raise NoDataInRequestException(f'Request {request_properties.request_id}: neither the draft nor the sealed container exists, cannot enumerate request files') + else: + storage_account_name = get_storage_account(request_properties.previous_status, request_properties.type, request_properties.workspace_id) + return blob_operations.get_request_files(account_name=storage_account_name, request_id=request_properties.request_id, container_name=container_name) def _get_tre_id(): diff --git a/airlock_processor/_version.py b/airlock_processor/_version.py index cb4382b891..9e4d45324c 100644 --- a/airlock_processor/_version.py +++ b/airlock_processor/_version.py @@ -1 +1 @@ -__version__ = "0.8.12" +__version__ = "0.8.31" diff --git a/airlock_processor/exceptions/__init__.py b/airlock_processor/exceptions/__init__.py index dcaede73c1..bbf7738df6 100644 --- a/airlock_processor/exceptions/__init__.py +++ b/airlock_processor/exceptions/__init__.py @@ -4,3 +4,7 @@ class NoFilesInRequestException(Exception): class TooManyFilesInRequestException(Exception): pass + + +class NoDataInRequestException(Exception): + pass diff --git a/airlock_processor/shared_code/airlock_storage_helper.py b/airlock_processor/shared_code/airlock_storage_helper.py new file mode 100644 index 0000000000..16bd89a037 --- /dev/null +++ b/airlock_processor/shared_code/airlock_storage_helper.py @@ -0,0 +1,61 @@ +import os +from shared_code import constants + + +def get_container_name_for_request(request_id: str, status: str) -> str: + if status == constants.STAGE_DRAFT: + return f"{request_id}{constants.DRAFT_CONTAINER_SUFFIX}" + return request_id + + +def get_request_id_from_container_name(container_name: str) -> str: + if container_name.endswith(constants.DRAFT_CONTAINER_SUFFIX): + return container_name[:-len(constants.DRAFT_CONTAINER_SUFFIX)] + return container_name + + +def get_storage_account_name_for_request(request_type: str, status: str) -> str: + # v1 routing lives in StatusChangedQueueTrigger.get_storage_account. + tre_id = os.environ.get("TRE_ID", "") + + if request_type not in (constants.IMPORT_TYPE, constants.EXPORT_TYPE): + # Falling through to the export layout would silently place data in the wrong account. + raise ValueError(f"Unknown airlock request type '{request_type}'") + + if request_type == constants.IMPORT_TYPE: + if status in [constants.STAGE_DRAFT, constants.STAGE_SUBMITTED, constants.STAGE_IN_REVIEW, + constants.STAGE_REJECTED, constants.STAGE_REJECTION_INPROGRESS, + constants.STAGE_BLOCKED_BY_SCAN, constants.STAGE_BLOCKING_INPROGRESS]: + return constants.STORAGE_ACCOUNT_NAME_AIRLOCK_CORE + tre_id + return constants.STORAGE_ACCOUNT_NAME_AIRLOCK_WORKSPACE_GLOBAL + tre_id + + if status in [constants.STAGE_APPROVED, constants.STAGE_APPROVAL_INPROGRESS]: + return constants.STORAGE_ACCOUNT_NAME_AIRLOCK_CORE + tre_id + return constants.STORAGE_ACCOUNT_NAME_AIRLOCK_WORKSPACE_GLOBAL + tre_id + + +def get_stage_from_status(request_type: str, status: str) -> str: + if request_type == constants.IMPORT_TYPE: + if status == constants.STAGE_DRAFT: + return constants.STAGE_IMPORT_EXTERNAL + elif status in [constants.STAGE_SUBMITTED, constants.STAGE_IN_REVIEW]: + return constants.STAGE_IMPORT_IN_PROGRESS + elif status in [constants.STAGE_APPROVED, constants.STAGE_APPROVAL_INPROGRESS]: + return constants.STAGE_IMPORT_APPROVED + elif status in [constants.STAGE_REJECTED, constants.STAGE_REJECTION_INPROGRESS]: + return constants.STAGE_IMPORT_REJECTED + elif status in [constants.STAGE_BLOCKED_BY_SCAN, constants.STAGE_BLOCKING_INPROGRESS]: + return constants.STAGE_IMPORT_BLOCKED + else: + if status == constants.STAGE_DRAFT: + return constants.STAGE_EXPORT_INTERNAL + elif status in [constants.STAGE_SUBMITTED, constants.STAGE_IN_REVIEW]: + return constants.STAGE_EXPORT_IN_PROGRESS + elif status in [constants.STAGE_APPROVED, constants.STAGE_APPROVAL_INPROGRESS]: + return constants.STAGE_EXPORT_APPROVED + elif status in [constants.STAGE_REJECTED, constants.STAGE_REJECTION_INPROGRESS]: + return constants.STAGE_EXPORT_REJECTED + elif status in [constants.STAGE_BLOCKED_BY_SCAN, constants.STAGE_BLOCKING_INPROGRESS]: + return constants.STAGE_EXPORT_BLOCKED + + return "unknown" diff --git a/airlock_processor/shared_code/blob_operations.py b/airlock_processor/shared_code/blob_operations.py index 211b1925aa..c6f7b079f5 100644 --- a/airlock_processor/shared_code/blob_operations.py +++ b/airlock_processor/shared_code/blob_operations.py @@ -2,15 +2,19 @@ import logging import json import re +import time from datetime import datetime, timedelta, UTC from typing import Tuple -from azure.core.exceptions import ResourceExistsError +from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError from azure.identity import DefaultAzureCredential from azure.storage.blob import ContainerSasPermissions, generate_container_sas, BlobServiceClient from exceptions import NoFilesInRequestException, TooManyFilesInRequestException +COPY_TIMEOUT_SECONDS = 300 +COPY_POLL_INTERVAL_SECONDS = 2 + def get_account_url(account_name: str) -> str: return f"https://{account_name}.blob.{get_storage_endpoint_suffix()}/" @@ -34,10 +38,26 @@ def create_container(account_name: str, request_id: str): logging.info(f'Did not create a new container. Container already exists for request id: {request_id}.') -def get_request_files(account_name: str, request_id: str) -> list: +def container_exists(account_name: str, container_name: str) -> bool: + blob_service_client = BlobServiceClient(account_url=get_account_url(account_name), + credential=get_credential()) + return blob_service_client.get_container_client(container_name).exists() + + +def delete_container(account_name: str, container_name: str): + blob_service_client = BlobServiceClient(account_url=get_account_url(account_name), + credential=get_credential()) + try: + blob_service_client.delete_container(container_name) + logging.info(f'Deleted container {container_name} from {account_name}.') + except ResourceNotFoundError: + logging.info(f'Container {container_name} already absent from {account_name}.') + + +def get_request_files(account_name: str, request_id: str, container_name: str = None) -> list: files = [] blob_service_client = BlobServiceClient(account_url=get_account_url(account_name), credential=get_credential()) - container_client = blob_service_client.get_container_client(container=request_id) + container_client = blob_service_client.get_container_client(container=container_name or request_id) for blob in container_client.list_blobs(): files.append({"name": blob.name, "size": blob.size}) @@ -45,9 +65,11 @@ def get_request_files(account_name: str, request_id: str) -> list: return files -def copy_data(source_account_name: str, destination_account_name: str, request_id: str): +def copy_data(source_account_name: str, destination_account_name: str, request_id: str, + source_container: str = None, destination_container: str = None): credential = get_credential() - container_name = request_id + container_name = source_container or request_id + dest_container_name = destination_container or request_id source_blob_service_client = BlobServiceClient(account_url=get_account_url(source_account_name), credential=credential) @@ -93,7 +115,7 @@ def copy_data(source_account_name: str, destination_account_name: str, request_i # Copy files dest_blob_service_client = BlobServiceClient(account_url=get_account_url(destination_account_name), credential=credential) - copied_blob = dest_blob_service_client.get_blob_client(container_name, source_blob.blob_name) + copied_blob = dest_blob_service_client.get_blob_client(dest_container_name, source_blob.blob_name) copy = copied_blob.start_copy_from_url(source_url, metadata=metadata) try: @@ -102,6 +124,25 @@ def copy_data(source_account_name: str, destination_account_name: str, request_i except KeyError as e: logging.error(f"Failed getting operation id and status {e}") + # An async copy still reads from the source, so the caller must not delete it until this settles. + copy_status = copy.get("copy_status") + waited_seconds = 0 + while copy_status == "pending" and waited_seconds < COPY_TIMEOUT_SECONDS: + time.sleep(COPY_POLL_INTERVAL_SECONDS) + waited_seconds += COPY_POLL_INTERVAL_SECONDS + copy_status = copied_blob.get_blob_properties().copy.status + + if copy_status != "success": + if copy_status == "pending": + # Abort the copy so a late completion cannot recreate the destination after we fail, + # which would otherwise leave orphaned data once the source is deleted. + try: + copied_blob.abort_copy(copy["copy_id"]) + logging.warning(f"Aborted still-pending copy of '{source_blob.blob_name}' after {waited_seconds}s") + except Exception as abort_error: + logging.error(f"Failed aborting pending copy of '{source_blob.blob_name}': {abort_error}") + raise Exception(f"Copy of '{source_blob.blob_name}' did not complete: status '{copy_status}' after {waited_seconds}s") + def get_credential() -> DefaultAzureCredential: managed_identity = os.environ.get("MANAGED_IDENTITY_CLIENT_ID") @@ -113,16 +154,25 @@ def get_credential() -> DefaultAzureCredential: def get_blob_info_from_topic_and_subject(topic: str, subject: str): # Example of a topic: "/subscriptions//resourceGroups//providers/Microsoft.Storage/storageAccounts/" - storage_account_name = re.search(r'providers/Microsoft.Storage/storageAccounts/(.*?)$', topic).group(1) + account_match = re.search(r'providers/Microsoft.Storage/storageAccounts/(.*?)$', topic) + if account_match is None: + raise ValueError(f"Could not parse storage account name from Event Grid topic: '{topic}'") + storage_account_name = account_match.group(1) # Example of a subject: "/blobServices/default/containers//blobs/" - container_name, blob_name = re.search(r'/blobServices/default/containers/(.*?)/blobs/(.*?)$', subject).groups() + subject_match = re.search(r'/blobServices/default/containers/(.*?)/blobs/(.*?)$', subject) + if subject_match is None: + raise ValueError(f"Could not parse container and blob name from Event Grid subject: '{subject}'") + container_name, blob_name = subject_match.groups() return storage_account_name, container_name, blob_name def get_blob_info_from_blob_url(blob_url: str) -> Tuple[str, str, str]: # Example of blob url: https://stalimappws663d.blob.core.windows.net/50866a82-d13a-4fd5-936f-deafdf1022ce/test_blob.txt - return re.search(rf'https://(.*?).blob.{get_storage_endpoint_suffix()}/(.*?)/(.*?)$', blob_url).groups() + url_match = re.search(rf'https://(.*?).blob.{get_storage_endpoint_suffix()}/(.*?)/(.*?)$', blob_url) + if url_match is None: + raise ValueError(f"Could not parse account, container and blob name from blob URL: '{blob_url}'") + return url_match.groups() def get_blob_url(account_name: str, container_name: str, blob_name='') -> str: diff --git a/airlock_processor/shared_code/blob_operations_metadata.py b/airlock_processor/shared_code/blob_operations_metadata.py new file mode 100644 index 0000000000..2b9127e8f6 --- /dev/null +++ b/airlock_processor/shared_code/blob_operations_metadata.py @@ -0,0 +1,131 @@ +import os +import logging +from datetime import datetime, UTC +from typing import Dict + +from azure.core import MatchConditions +from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError, ResourceModifiedError +from azure.identity import DefaultAzureCredential +from azure.storage.blob import BlobServiceClient +from azure.core.exceptions import HttpResponseError + + +def get_account_url(account_name: str) -> str: + return f"https://{account_name}.blob.{get_storage_endpoint_suffix()}/" + + +def get_storage_endpoint_suffix() -> str: + return os.environ.get("STORAGE_ENDPOINT_SUFFIX", "core.windows.net") + + +def get_credential(): + managed_identity = os.environ.get("MANAGED_IDENTITY_CLIENT_ID") + if managed_identity: + logging.info("using the Airlock processor's managed identity to get credentials.") + return DefaultAzureCredential(managed_identity_client_id=managed_identity, + exclude_shared_token_cache_credential=True) + return DefaultAzureCredential() + + +def create_container_with_metadata(account_name: str, request_id: str, stage: str, + workspace_id: str = None, request_type: str = None, + created_by: str = None) -> None: + try: + container_name = request_id + blob_service_client = BlobServiceClient( + account_url=get_account_url(account_name), + credential=get_credential() + ) + + metadata = { + "stage": stage, + "stage_history": stage, + "created_at": datetime.now(UTC).isoformat(), + "last_stage_change": datetime.now(UTC).isoformat(), + } + + if workspace_id: + metadata["workspace_id"] = workspace_id + if request_type: + metadata["request_type"] = request_type + if created_by: + metadata["created_by"] = created_by + + container_client = blob_service_client.get_container_client(container_name) + container_client.create_container(metadata=metadata) + + logging.info(f'Container created for request id: {request_id} with stage: {stage}') + + except ResourceExistsError: + logging.info(f'Did not create a new container. Container already exists for request id: {request_id}.') + + +def update_container_stage(account_name: str, request_id: str, new_stage: str, + changed_by: str = None, additional_metadata: Dict[str, str] = None, + max_attempts: int = 5) -> bool: + """Update stage metadata with optimistic concurrency.""" + container_name = request_id + blob_service_client = BlobServiceClient( + account_url=get_account_url(account_name), + credential=get_credential() + ) + container_client = blob_service_client.get_container_client(container_name) + + for attempt in range(1, max_attempts + 1): + try: + properties = container_client.get_container_properties() + except ResourceNotFoundError: + logging.error(f"Container {request_id} not found in account {account_name}") + raise + + metadata = properties.metadata.copy() + old_stage = metadata.get('stage', 'unknown') + + metadata['stage'] = new_stage + stage_history = metadata.get('stage_history', old_stage) + metadata['stage_history'] = f"{stage_history},{new_stage}" + metadata['last_stage_change'] = datetime.now(UTC).isoformat() + if changed_by: + metadata['last_changed_by'] = changed_by + if additional_metadata: + metadata.update(additional_metadata) + + try: + container_client.set_container_metadata( + metadata, + etag=properties.etag, + match_condition=MatchConditions.IfNotModified + ) + except ResourceModifiedError: + logging.warning( + f"Container {request_id} metadata changed concurrently (attempt {attempt}/{max_attempts}), retrying" + ) + continue + except HttpResponseError as e: + logging.error(f"Failed to update container metadata: {str(e)}") + raise + + logging.info( + f"Updated container {request_id} from stage '{old_stage}' to '{new_stage}' in account {account_name}" + ) + return True + + raise HttpResponseError( + message=f"Could not update stage for container {request_id} after {max_attempts} attempts due to concurrent updates" + ) + + +def get_container_metadata(account_name: str, request_id: str) -> Dict[str, str]: + container_name = request_id + blob_service_client = BlobServiceClient( + account_url=get_account_url(account_name), + credential=get_credential() + ) + container_client = blob_service_client.get_container_client(container_name) + + try: + properties = container_client.get_container_properties() + return properties.metadata + except ResourceNotFoundError: + logging.error(f"Container {request_id} not found in account {account_name}") + raise diff --git a/airlock_processor/shared_code/constants.py b/airlock_processor/shared_code/constants.py index 277312d1cb..5dab9008e9 100644 --- a/airlock_processor/shared_code/constants.py +++ b/airlock_processor/shared_code/constants.py @@ -4,6 +4,22 @@ IMPORT_TYPE = "import" EXPORT_TYPE = "export" + +STORAGE_ACCOUNT_NAME_AIRLOCK_CORE = "stalairlock" +STORAGE_ACCOUNT_NAME_AIRLOCK_WORKSPACE_GLOBAL = "stalairlockg" + +STAGE_IMPORT_EXTERNAL = "import-external" +STAGE_IMPORT_IN_PROGRESS = "import-in-progress" +STAGE_IMPORT_APPROVED = "import-approved" +STAGE_IMPORT_REJECTED = "import-rejected" +STAGE_IMPORT_BLOCKED = "import-blocked" +STAGE_EXPORT_INTERNAL = "export-internal" +STAGE_EXPORT_IN_PROGRESS = "export-in-progress" +STAGE_EXPORT_APPROVED = "export-approved" +STAGE_EXPORT_REJECTED = "export-rejected" +STAGE_EXPORT_BLOCKED = "export-blocked" + +# Retained for legacy workspace compatibility. # Import STORAGE_ACCOUNT_NAME_IMPORT_EXTERNAL = "stalimex" STORAGE_ACCOUNT_NAME_IMPORT_INPROGRESS = "stalimip" @@ -35,9 +51,13 @@ NO_FILES_IN_REQUEST_MESSAGE = "Request did not contain any files." TOO_MANY_FILES_IN_REQUEST_MESSAGE = "Request contained more than 1 file." UNKNOWN_REASON_MESSAGE = "Request failed due to an unknown reason." +NO_DATA_IN_REQUEST_MESSAGE = "Request data could not be found in storage." # Event Grid STEP_RESULT_EVENT_DATA_VERSION = "1.0" DATA_DELETION_EVENT_DATA_VERSION = "1.0" NO_THREATS = "No threats found" + +# Draft data lives in its own container so submission can delete it, revoking any SAS already issued. +DRAFT_CONTAINER_SUFFIX = "-draft" diff --git a/airlock_processor/tests/shared_code/test_airlock_storage_helper.py b/airlock_processor/tests/shared_code/test_airlock_storage_helper.py new file mode 100644 index 0000000000..4826f5a94b --- /dev/null +++ b/airlock_processor/tests/shared_code/test_airlock_storage_helper.py @@ -0,0 +1,164 @@ +import os +from unittest.mock import patch + +import pytest + +from shared_code.airlock_storage_helper import ( + get_storage_account_name_for_request, + get_stage_from_status +) +from shared_code import constants + + +class TestGetStageFromStatus: + + def test_import_draft_maps_to_import_external(self): + stage = get_stage_from_status(constants.IMPORT_TYPE, constants.STAGE_DRAFT) + assert stage == constants.STAGE_IMPORT_EXTERNAL + + def test_import_submitted_maps_to_import_in_progress(self): + stage = get_stage_from_status(constants.IMPORT_TYPE, constants.STAGE_SUBMITTED) + assert stage == constants.STAGE_IMPORT_IN_PROGRESS + + def test_import_in_review_maps_to_import_in_progress(self): + stage = get_stage_from_status(constants.IMPORT_TYPE, constants.STAGE_IN_REVIEW) + assert stage == constants.STAGE_IMPORT_IN_PROGRESS + + def test_import_approved_maps_to_import_approved(self): + stage = get_stage_from_status(constants.IMPORT_TYPE, constants.STAGE_APPROVED) + assert stage == constants.STAGE_IMPORT_APPROVED + + def test_import_approval_in_progress_maps_to_import_approved(self): + stage = get_stage_from_status(constants.IMPORT_TYPE, constants.STAGE_APPROVAL_INPROGRESS) + assert stage == constants.STAGE_IMPORT_APPROVED + + def test_import_rejected_maps_to_import_rejected(self): + stage = get_stage_from_status(constants.IMPORT_TYPE, constants.STAGE_REJECTED) + assert stage == constants.STAGE_IMPORT_REJECTED + + def test_import_rejection_in_progress_maps_to_import_rejected(self): + stage = get_stage_from_status(constants.IMPORT_TYPE, constants.STAGE_REJECTION_INPROGRESS) + assert stage == constants.STAGE_IMPORT_REJECTED + + def test_import_blocked_maps_to_import_blocked(self): + stage = get_stage_from_status(constants.IMPORT_TYPE, constants.STAGE_BLOCKED_BY_SCAN) + assert stage == constants.STAGE_IMPORT_BLOCKED + + def test_import_blocking_in_progress_maps_to_import_blocked(self): + stage = get_stage_from_status(constants.IMPORT_TYPE, constants.STAGE_BLOCKING_INPROGRESS) + assert stage == constants.STAGE_IMPORT_BLOCKED + + def test_export_draft_maps_to_export_internal(self): + stage = get_stage_from_status(constants.EXPORT_TYPE, constants.STAGE_DRAFT) + assert stage == constants.STAGE_EXPORT_INTERNAL + + def test_export_submitted_maps_to_export_in_progress(self): + stage = get_stage_from_status(constants.EXPORT_TYPE, constants.STAGE_SUBMITTED) + assert stage == constants.STAGE_EXPORT_IN_PROGRESS + + def test_export_in_review_maps_to_export_in_progress(self): + stage = get_stage_from_status(constants.EXPORT_TYPE, constants.STAGE_IN_REVIEW) + assert stage == constants.STAGE_EXPORT_IN_PROGRESS + + def test_export_approved_maps_to_export_approved(self): + stage = get_stage_from_status(constants.EXPORT_TYPE, constants.STAGE_APPROVED) + assert stage == constants.STAGE_EXPORT_APPROVED + + def test_export_approval_in_progress_maps_to_export_approved(self): + stage = get_stage_from_status(constants.EXPORT_TYPE, constants.STAGE_APPROVAL_INPROGRESS) + assert stage == constants.STAGE_EXPORT_APPROVED + + def test_export_rejected_maps_to_export_rejected(self): + stage = get_stage_from_status(constants.EXPORT_TYPE, constants.STAGE_REJECTED) + assert stage == constants.STAGE_EXPORT_REJECTED + + def test_export_rejection_in_progress_maps_to_export_rejected(self): + stage = get_stage_from_status(constants.EXPORT_TYPE, constants.STAGE_REJECTION_INPROGRESS) + assert stage == constants.STAGE_EXPORT_REJECTED + + def test_export_blocked_maps_to_export_blocked(self): + stage = get_stage_from_status(constants.EXPORT_TYPE, constants.STAGE_BLOCKED_BY_SCAN) + assert stage == constants.STAGE_EXPORT_BLOCKED + + def test_export_blocking_in_progress_maps_to_export_blocked(self): + stage = get_stage_from_status(constants.EXPORT_TYPE, constants.STAGE_BLOCKING_INPROGRESS) + assert stage == constants.STAGE_EXPORT_BLOCKED + + def test_unknown_status_returns_unknown(self): + stage = get_stage_from_status(constants.IMPORT_TYPE, "nonexistent_status") + assert stage == "unknown" + + +class TestGetStorageAccountNameForRequestConsolidated: + + @patch.dict(os.environ, {"TRE_ID": "tre123"}, clear=True) + class TestImportRequests: + + def test_import_draft_uses_core_storage(self): + account = get_storage_account_name_for_request(constants.IMPORT_TYPE, constants.STAGE_DRAFT) + assert account == "stalairlocktre123" + + def test_import_submitted_uses_core_storage(self): + account = get_storage_account_name_for_request(constants.IMPORT_TYPE, constants.STAGE_SUBMITTED) + assert account == "stalairlocktre123" + + def test_import_in_review_uses_core_storage(self): + account = get_storage_account_name_for_request(constants.IMPORT_TYPE, constants.STAGE_IN_REVIEW) + assert account == "stalairlocktre123" + + def test_import_approved_uses_workspace_global_storage(self): + account = get_storage_account_name_for_request(constants.IMPORT_TYPE, constants.STAGE_APPROVED) + assert account == "stalairlockgtre123" + + def test_import_approval_in_progress_uses_workspace_global_storage(self): + account = get_storage_account_name_for_request(constants.IMPORT_TYPE, constants.STAGE_APPROVAL_INPROGRESS) + assert account == "stalairlockgtre123" + + def test_import_rejected_uses_core_storage(self): + account = get_storage_account_name_for_request(constants.IMPORT_TYPE, constants.STAGE_REJECTED) + assert account == "stalairlocktre123" + + def test_import_rejection_in_progress_uses_core_storage(self): + account = get_storage_account_name_for_request(constants.IMPORT_TYPE, constants.STAGE_REJECTION_INPROGRESS) + assert account == "stalairlocktre123" + + def test_import_blocked_uses_core_storage(self): + account = get_storage_account_name_for_request(constants.IMPORT_TYPE, constants.STAGE_BLOCKED_BY_SCAN) + assert account == "stalairlocktre123" + + def test_import_blocking_in_progress_uses_core_storage(self): + account = get_storage_account_name_for_request(constants.IMPORT_TYPE, constants.STAGE_BLOCKING_INPROGRESS) + assert account == "stalairlocktre123" + + @patch.dict(os.environ, {"TRE_ID": "tre123"}, clear=True) + class TestExportRequests: + + def test_export_draft_uses_workspace_global_storage(self): + account = get_storage_account_name_for_request(constants.EXPORT_TYPE, constants.STAGE_DRAFT) + assert account == "stalairlockgtre123" + + def test_export_submitted_uses_workspace_global_storage(self): + account = get_storage_account_name_for_request(constants.EXPORT_TYPE, constants.STAGE_SUBMITTED) + assert account == "stalairlockgtre123" + + def test_export_approved_uses_core_storage(self): + account = get_storage_account_name_for_request(constants.EXPORT_TYPE, constants.STAGE_APPROVED) + assert account == "stalairlocktre123" + + def test_export_approval_in_progress_uses_core_storage(self): + account = get_storage_account_name_for_request(constants.EXPORT_TYPE, constants.STAGE_APPROVAL_INPROGRESS) + assert account == "stalairlocktre123" + + def test_export_rejected_uses_workspace_global_storage(self): + account = get_storage_account_name_for_request(constants.EXPORT_TYPE, constants.STAGE_REJECTED) + assert account == "stalairlockgtre123" + + def test_export_blocked_uses_workspace_global_storage(self): + account = get_storage_account_name_for_request(constants.EXPORT_TYPE, constants.STAGE_BLOCKED_BY_SCAN) + assert account == "stalairlockgtre123" + + +def test_unknown_request_type_raises_rather_than_defaulting_to_export(): + # A typo must fail closed instead of quietly selecting the workspace-global account. + with pytest.raises(ValueError, match="Unknown airlock request type"): + get_storage_account_name_for_request("imprt", constants.STAGE_DRAFT) diff --git a/airlock_processor/tests/shared_code/test_blob_operations.py b/airlock_processor/tests/shared_code/test_blob_operations.py index c3a921f0b7..95db60b14e 100644 --- a/airlock_processor/tests/shared_code/test_blob_operations.py +++ b/airlock_processor/tests/shared_code/test_blob_operations.py @@ -62,7 +62,7 @@ def test_copy_data_adds_copied_from_metadata(self, _, mock_blob_service_client): dest_blob_client_mock = MagicMock() dest_blob_client_mock.bla = "bla" - dest_blob_client_mock.start_copy_from_url = MagicMock(return_value={"copy_id": "123", "copy_status": "status"}) + dest_blob_client_mock.start_copy_from_url = MagicMock(return_value={"copy_id": "123", "copy_status": "success"}) # Set source blob mock mock_blob_service_client().get_container_client().get_blob_client = MagicMock(return_value=source_blob_client_mock) @@ -78,7 +78,75 @@ def test_copy_data_adds_copied_from_metadata(self, _, mock_blob_service_client): # Check that copied_from field was set correctly in the metadata dest_blob_client_mock.start_copy_from_url.assert_called_with(f"{source_url}?sas", metadata=dest_metadata) - def test_get_blob_url_should_return_blob_url(self): + @patch("shared_code.blob_operations.time.sleep") + @patch("shared_code.blob_operations.BlobServiceClient") + @patch("shared_code.blob_operations.generate_container_sas", return_value="sas") + def test_copy_data_waits_for_pending_copy(self, _, mock_blob_service_client, __): + # The caller deletes the source immediately afterwards, so returning while the copy is + # still pending would abort it and lose the data. + source_url = f"http://storageacct.blob.{get_storage_endpoint_suffix()}/container/blob" + source_blob_client_mock = MagicMock() + source_blob_client_mock.url = source_url + source_blob_client_mock.get_blob_properties = MagicMock(return_value={"metadata": {}}) + + dest_blob_client_mock = MagicMock() + dest_blob_client_mock.start_copy_from_url = MagicMock(return_value={"copy_id": "123", "copy_status": "pending"}) + dest_blob_client_mock.get_blob_properties = MagicMock(side_effect=[ + MagicMock(copy=MagicMock(status="pending")), + MagicMock(copy=MagicMock(status="success"))]) + + mock_blob_service_client().get_container_client().get_blob_client = MagicMock(return_value=source_blob_client_mock) + mock_blob_service_client().get_blob_client = MagicMock(return_value=dest_blob_client_mock) + mock_blob_service_client().get_user_delegation_key = MagicMock(return_value="key") + mock_blob_service_client().get_container_client().list_blobs = MagicMock(return_value=[get_test_blob()("a")]) + + copy_data("source_acc", "dest_acc", "req_id") + + assert dest_blob_client_mock.get_blob_properties.call_count == 2 + + @patch("shared_code.blob_operations.time.sleep") + @patch("shared_code.blob_operations.BlobServiceClient") + @patch("shared_code.blob_operations.generate_container_sas", return_value="sas") + def test_copy_data_raises_if_copy_never_succeeds(self, _, mock_blob_service_client, __): + source_url = f"http://storageacct.blob.{get_storage_endpoint_suffix()}/container/blob" + source_blob_client_mock = MagicMock() + source_blob_client_mock.url = source_url + source_blob_client_mock.get_blob_properties = MagicMock(return_value={"metadata": {}}) + + dest_blob_client_mock = MagicMock() + dest_blob_client_mock.start_copy_from_url = MagicMock(return_value={"copy_id": "123", "copy_status": "failed"}) + + mock_blob_service_client().get_container_client().get_blob_client = MagicMock(return_value=source_blob_client_mock) + mock_blob_service_client().get_blob_client = MagicMock(return_value=dest_blob_client_mock) + mock_blob_service_client().get_user_delegation_key = MagicMock(return_value="key") + mock_blob_service_client().get_container_client().list_blobs = MagicMock(return_value=[get_test_blob()("a")]) + + with pytest.raises(Exception, match="did not complete"): + copy_data("source_acc", "dest_acc", "req_id") + + @patch("shared_code.blob_operations.time.sleep") + @patch("shared_code.blob_operations.BlobServiceClient") + @patch("shared_code.blob_operations.generate_container_sas", return_value="sas") + def test_copy_data_aborts_a_copy_still_pending_at_timeout(self, _, mock_blob_service_client, __): + # A late completion after we have failed the request would recreate the destination and orphan data. + source_url = f"http://storageacct.blob.{get_storage_endpoint_suffix()}/container/blob" + source_blob_client_mock = MagicMock() + source_blob_client_mock.url = source_url + source_blob_client_mock.get_blob_properties = MagicMock(return_value={"metadata": {}}) + + dest_blob_client_mock = MagicMock() + dest_blob_client_mock.start_copy_from_url = MagicMock(return_value={"copy_id": "123", "copy_status": "pending"}) + dest_blob_client_mock.get_blob_properties = MagicMock(return_value=MagicMock(copy=MagicMock(status="pending"))) + + mock_blob_service_client().get_container_client().get_blob_client = MagicMock(return_value=source_blob_client_mock) + mock_blob_service_client().get_blob_client = MagicMock(return_value=dest_blob_client_mock) + mock_blob_service_client().get_user_delegation_key = MagicMock(return_value="key") + mock_blob_service_client().get_container_client().list_blobs = MagicMock(return_value=[get_test_blob()("a")]) + + with pytest.raises(Exception, match="did not complete"): + copy_data("source_acc", "dest_acc", "req_id") + dest_blob_client_mock.abort_copy.assert_called_once_with("123") + account_name = "account" container_name = "container" blob_name = "blob" diff --git a/airlock_processor/tests/shared_code/test_blob_operations_metadata.py b/airlock_processor/tests/shared_code/test_blob_operations_metadata.py new file mode 100644 index 0000000000..eff668e720 --- /dev/null +++ b/airlock_processor/tests/shared_code/test_blob_operations_metadata.py @@ -0,0 +1,347 @@ +import pytest +from unittest.mock import MagicMock, patch + +from azure.core import MatchConditions +from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError, HttpResponseError + +from shared_code.blob_operations_metadata import ( + get_account_url, + get_storage_endpoint_suffix, + create_container_with_metadata, + update_container_stage, + get_container_metadata +) + + +class TestGetAccountUrl: + + @patch.dict('os.environ', {"STORAGE_ENDPOINT_SUFFIX": "core.windows.net"}, clear=True) + def test_returns_correct_url_format(self): + url = get_account_url("mystorageaccount") + assert url == "https://mystorageaccount.blob.core.windows.net/" + + @patch.dict('os.environ', {"STORAGE_ENDPOINT_SUFFIX": "core.chinacloudapi.cn"}, clear=True) + def test_uses_custom_endpoint_suffix(self): + url = get_account_url("mystorageaccount") + assert url == "https://mystorageaccount.blob.core.chinacloudapi.cn/" + + @patch.dict('os.environ', {}, clear=True) + def test_uses_default_endpoint_when_not_set(self): + url = get_account_url("mystorageaccount") + assert url == "https://mystorageaccount.blob.core.windows.net/" + + +class TestGetStorageEndpointSuffix: + + @patch.dict('os.environ', {"STORAGE_ENDPOINT_SUFFIX": "core.usgovcloudapi.net"}, clear=True) + def test_returns_configured_suffix(self): + suffix = get_storage_endpoint_suffix() + assert suffix == "core.usgovcloudapi.net" + + @patch.dict('os.environ', {}, clear=True) + def test_returns_default_when_not_configured(self): + suffix = get_storage_endpoint_suffix() + assert suffix == "core.windows.net" + + +class TestCreateContainerWithMetadata: + + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch("shared_code.blob_operations_metadata.get_credential") + def test_creates_container_with_stage_metadata(self, mock_get_credential, mock_blob_service_client): + mock_container_client = MagicMock() + mock_blob_service_client.return_value.get_container_client.return_value = mock_container_client + + create_container_with_metadata( + account_name="storageaccount", + request_id="request-123", + stage="import-external" + ) + + mock_container_client.create_container.assert_called_once() + call_args = mock_container_client.create_container.call_args + metadata = call_args.kwargs['metadata'] + + assert metadata['stage'] == "import-external" + assert 'created_at' in metadata + assert 'last_stage_change' in metadata + assert metadata['stage_history'] == "import-external" + + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch("shared_code.blob_operations_metadata.get_credential") + def test_creates_container_with_all_optional_metadata(self, mock_get_credential, mock_blob_service_client): + mock_container_client = MagicMock() + mock_blob_service_client.return_value.get_container_client.return_value = mock_container_client + + create_container_with_metadata( + account_name="storageaccount", + request_id="request-123", + stage="export-internal", + workspace_id="ws-456", + request_type="export", + created_by="user@example.com" + ) + + call_args = mock_container_client.create_container.call_args + metadata = call_args.kwargs['metadata'] + + assert metadata['stage'] == "export-internal" + assert metadata['workspace_id'] == "ws-456" + assert metadata['request_type'] == "export" + assert metadata['created_by'] == "user@example.com" + + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch("shared_code.blob_operations_metadata.get_credential") + def test_handles_container_already_exists(self, mock_get_credential, mock_blob_service_client): + mock_container_client = MagicMock() + mock_container_client.create_container.side_effect = ResourceExistsError("Container already exists") + mock_blob_service_client.return_value.get_container_client.return_value = mock_container_client + + create_container_with_metadata( + account_name="storageaccount", + request_id="request-123", + stage="import-external" + ) + + +class TestUpdateContainerStage: + + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch("shared_code.blob_operations_metadata.get_credential") + def test_updates_stage_metadata(self, mock_get_credential, mock_blob_service_client): + mock_container_client = MagicMock() + mock_properties = MagicMock() + mock_properties.metadata = { + 'stage': 'import-external', + 'stage_history': 'import-external', + 'created_at': '2024-01-01T00:00:00' + } + mock_container_client.get_container_properties.return_value = mock_properties + mock_blob_service_client.return_value.get_container_client.return_value = mock_container_client + + update_container_stage( + account_name="storageaccount", + request_id="request-123", + new_stage="import-in-progress" + ) + + mock_container_client.set_container_metadata.assert_called_once() + call_args = mock_container_client.set_container_metadata.call_args + updated_metadata = call_args.args[0] + + assert updated_metadata['stage'] == "import-in-progress" + assert "import-in-progress" in updated_metadata['stage_history'] + assert 'last_stage_change' in updated_metadata + + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch("shared_code.blob_operations_metadata.get_credential") + def test_writes_conditionally_on_the_etag_it_read(self, mock_get_credential, mock_blob_service_client): + mock_container_client = MagicMock() + mock_properties = MagicMock() + mock_properties.metadata = {'stage': 'import-external'} + mock_properties.etag = '"etag-1"' + mock_container_client.get_container_properties.return_value = mock_properties + mock_blob_service_client.return_value.get_container_client.return_value = mock_container_client + + assert update_container_stage("storageaccount", "request-123", "import-in-progress") is True + + kwargs = mock_container_client.set_container_metadata.call_args.kwargs + assert kwargs['etag'] == '"etag-1"' + assert kwargs['match_condition'] == MatchConditions.IfNotModified + + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch("shared_code.blob_operations_metadata.get_credential") + def test_appends_to_stage_history(self, mock_get_credential, mock_blob_service_client): + mock_container_client = MagicMock() + mock_properties = MagicMock() + mock_properties.metadata = { + 'stage': 'import-external', + 'stage_history': 'import-external', + } + mock_container_client.get_container_properties.return_value = mock_properties + mock_blob_service_client.return_value.get_container_client.return_value = mock_container_client + + update_container_stage( + account_name="storageaccount", + request_id="request-123", + new_stage="import-in-progress" + ) + + call_args = mock_container_client.set_container_metadata.call_args + updated_metadata = call_args.args[0] + + assert updated_metadata['stage_history'] == "import-external,import-in-progress" + + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch("shared_code.blob_operations_metadata.get_credential") + def test_adds_changed_by_when_provided(self, mock_get_credential, mock_blob_service_client): + mock_container_client = MagicMock() + mock_properties = MagicMock() + mock_properties.metadata = {'stage': 'import-external', 'stage_history': 'import-external'} + mock_container_client.get_container_properties.return_value = mock_properties + mock_blob_service_client.return_value.get_container_client.return_value = mock_container_client + + update_container_stage( + account_name="storageaccount", + request_id="request-123", + new_stage="import-in-progress", + changed_by="processor" + ) + + call_args = mock_container_client.set_container_metadata.call_args + updated_metadata = call_args.args[0] + + assert updated_metadata['last_changed_by'] == "processor" + + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch("shared_code.blob_operations_metadata.get_credential") + def test_adds_additional_metadata(self, mock_get_credential, mock_blob_service_client): + mock_container_client = MagicMock() + mock_properties = MagicMock() + mock_properties.metadata = {'stage': 'import-in-progress', 'stage_history': 'import-external,import-in-progress'} + mock_container_client.get_container_properties.return_value = mock_properties + mock_blob_service_client.return_value.get_container_client.return_value = mock_container_client + + update_container_stage( + account_name="storageaccount", + request_id="request-123", + new_stage="import-approved", + additional_metadata={"scan_result": "clean", "scan_time": "2024-01-01T12:00:00"} + ) + + call_args = mock_container_client.set_container_metadata.call_args + updated_metadata = call_args.args[0] + + assert updated_metadata['scan_result'] == "clean" + assert updated_metadata['scan_time'] == "2024-01-01T12:00:00" + + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch("shared_code.blob_operations_metadata.get_credential") + def test_raises_when_container_not_found(self, mock_get_credential, mock_blob_service_client): + mock_container_client = MagicMock() + mock_container_client.get_container_properties.side_effect = ResourceNotFoundError("Container not found") + mock_blob_service_client.return_value.get_container_client.return_value = mock_container_client + + with pytest.raises(ResourceNotFoundError): + update_container_stage( + account_name="storageaccount", + request_id="nonexistent-request", + new_stage="import-in-progress" + ) + + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch("shared_code.blob_operations_metadata.get_credential") + def test_raises_on_http_error(self, mock_get_credential, mock_blob_service_client): + mock_container_client = MagicMock() + mock_properties = MagicMock() + mock_properties.metadata = {'stage': 'import-external'} + mock_container_client.get_container_properties.return_value = mock_properties + mock_container_client.set_container_metadata.side_effect = HttpResponseError("Service Error") + mock_blob_service_client.return_value.get_container_client.return_value = mock_container_client + + with pytest.raises(HttpResponseError): + update_container_stage( + account_name="storageaccount", + request_id="request-123", + new_stage="import-in-progress" + ) + + +class TestGetContainerMetadata: + + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch("shared_code.blob_operations_metadata.get_credential") + def test_returns_all_metadata(self, mock_get_credential, mock_blob_service_client): + expected_metadata = { + 'stage': 'import-in-progress', + 'workspace_id': 'ws-123', + 'request_type': 'import', + 'created_at': '2024-01-01T00:00:00', + 'stage_history': 'import-external,import-in-progress' + } + + mock_container_client = MagicMock() + mock_properties = MagicMock() + mock_properties.metadata = expected_metadata + mock_container_client.get_container_properties.return_value = mock_properties + mock_blob_service_client.return_value.get_container_client.return_value = mock_container_client + + metadata = get_container_metadata( + account_name="storageaccount", + request_id="request-123" + ) + + assert metadata == expected_metadata + + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch("shared_code.blob_operations_metadata.get_credential") + def test_raises_when_container_not_found(self, mock_get_credential, mock_blob_service_client): + mock_container_client = MagicMock() + mock_container_client.get_container_properties.side_effect = ResourceNotFoundError("Container not found") + mock_blob_service_client.return_value.get_container_client.return_value = mock_container_client + + with pytest.raises(ResourceNotFoundError): + get_container_metadata( + account_name="storageaccount", + request_id="nonexistent-request" + ) + + +class TestStageTransitions: + + ABAC_ALLOWED_STAGES = ['import-external', 'import-in-progress', 'export-approved'] + + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch("shared_code.blob_operations_metadata.get_credential") + def test_import_stage_transition_updates_history(self, mock_get_credential, mock_blob_service_client): + mock_container_client = MagicMock() + + current_metadata = { + 'stage': 'import-external', + 'stage_history': 'import-external' + } + mock_properties = MagicMock() + mock_properties.metadata = current_metadata.copy() + mock_container_client.get_container_properties.return_value = mock_properties + mock_blob_service_client.return_value.get_container_client.return_value = mock_container_client + + update_container_stage( + account_name="storageaccount", + request_id="request-123", + new_stage="import-in-progress" + ) + + call_args = mock_container_client.set_container_metadata.call_args + updated_metadata = call_args.args[0] + + assert updated_metadata['stage'] == "import-in-progress" + assert updated_metadata['stage_history'] == "import-external,import-in-progress" + + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch("shared_code.blob_operations_metadata.get_credential") + def test_scan_result_metadata_added_on_approval(self, mock_get_credential, mock_blob_service_client): + mock_container_client = MagicMock() + mock_properties = MagicMock() + mock_properties.metadata = { + 'stage': 'import-in-progress', + 'stage_history': 'import-external,import-in-progress' + } + mock_container_client.get_container_properties.return_value = mock_properties + mock_blob_service_client.return_value.get_container_client.return_value = mock_container_client + + update_container_stage( + account_name="storageaccount", + request_id="request-123", + new_stage="import-approved", + additional_metadata={ + "scan_result": "clean", + "scan_completed_at": "2024-01-01T12:00:00Z" + } + ) + + call_args = mock_container_client.set_container_metadata.call_args + updated_metadata = call_args.args[0] + + assert updated_metadata['stage'] == "import-approved" + assert updated_metadata['scan_result'] == "clean" + assert "import-approved" not in self.ABAC_ALLOWED_STAGES diff --git a/airlock_processor/tests/test_blob_created_trigger.py b/airlock_processor/tests/test_blob_created_trigger.py new file mode 100644 index 0000000000..e4269ed353 --- /dev/null +++ b/airlock_processor/tests/test_blob_created_trigger.py @@ -0,0 +1,110 @@ +import json +from mock import MagicMock, patch +import pytest + +import azure.functions as func + +from shared_code import constants +from BlobCreatedTrigger import main + + +def _make_service_bus_message(topic: str, request_id: str, blob_name: str = "test.txt"): + subject = f"/blobServices/default/containers/{request_id}/blobs/{blob_name}" + body = json.dumps({"topic": topic, "subject": subject}) + encoded = body.encode("utf-8") + msg = MagicMock(spec=func.ServiceBusMessage) + msg.get_body.return_value = encoded + return msg + + +def _mock_blob_client(): + mock_client = MagicMock() + mock_client.get_blob_properties.return_value = {"metadata": {"copied_from": '["container-prev"]'}} + return mock_client + + +class TestV2BlobCreated(): + + @patch("BlobCreatedTrigger.get_blob_client_from_blob_info", return_value=_mock_blob_client()) + @patch("shared_code.blob_operations_metadata.get_container_metadata", return_value={"stage": constants.STAGE_IMPORT_APPROVED, "workspace_id": "ws01"}) + @patch("BlobCreatedTrigger.get_blob_info_from_topic_and_subject") + def test_v2_import_approved_emits_step_result(self, mock_get_blob_info, mock_get_metadata, mock_blob_client): + topic = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/stalairlockgtre123" + request_id = "req-001" + mock_get_blob_info.return_value = ("stalairlockgtre123", request_id, "test.txt") + + step_result = MagicMock() + deletion_event = MagicMock() + + msg = _make_service_bus_message(topic, request_id) + main(msg=msg, stepResultEvent=step_result, dataDeletionEvent=deletion_event) + + step_result.set.assert_called_once() + event_data = step_result.set.call_args[0][0] + assert event_data.get_json()["completed_step"] == constants.STAGE_APPROVAL_INPROGRESS + assert event_data.get_json()["new_status"] == constants.STAGE_APPROVED + + @patch("BlobCreatedTrigger.get_blob_client_from_blob_info", return_value=_mock_blob_client()) + @patch("shared_code.blob_operations_metadata.get_container_metadata", return_value={"stage": constants.STAGE_EXPORT_APPROVED, "workspace_id": "ws01"}) + @patch("BlobCreatedTrigger.get_blob_info_from_topic_and_subject") + def test_v2_export_approved_emits_step_result(self, mock_get_blob_info, mock_get_metadata, mock_blob_client): + topic = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/stalairlocktre123" + request_id = "req-002" + mock_get_blob_info.return_value = ("stalairlocktre123", request_id, "test.txt") + + step_result = MagicMock() + deletion_event = MagicMock() + + msg = _make_service_bus_message(topic, request_id) + main(msg=msg, stepResultEvent=step_result, dataDeletionEvent=deletion_event) + + step_result.set.assert_called_once() + event_data = step_result.set.call_args[0][0] + assert event_data.get_json()["completed_step"] == constants.STAGE_APPROVAL_INPROGRESS + assert event_data.get_json()["new_status"] == constants.STAGE_APPROVED + + @patch("shared_code.blob_operations_metadata.get_container_metadata", return_value={"stage": constants.STAGE_IMPORT_EXTERNAL, "workspace_id": "ws01"}) + @patch("BlobCreatedTrigger.get_blob_info_from_topic_and_subject") + def test_v2_non_terminal_stage_does_not_emit_step_result(self, mock_get_blob_info, mock_get_metadata): + topic = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/stalairlocktre123" + request_id = "req-003" + mock_get_blob_info.return_value = ("stalairlocktre123", request_id, "test.txt") + + step_result = MagicMock() + deletion_event = MagicMock() + + msg = _make_service_bus_message(topic, request_id) + main(msg=msg, stepResultEvent=step_result, dataDeletionEvent=deletion_event) + + step_result.set.assert_not_called() + + @patch("shared_code.blob_operations_metadata.get_container_metadata", return_value={"stage": constants.STAGE_IMPORT_REJECTED, "workspace_id": "ws01"}) + @patch("BlobCreatedTrigger.get_blob_info_from_topic_and_subject") + def test_v2_rejected_stage_does_not_emit_step_result(self, mock_get_blob_info, mock_get_metadata): + topic = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/stalairlockgtre123" + request_id = "req-005" + mock_get_blob_info.return_value = ("stalairlockgtre123", request_id, "test.txt") + + step_result = MagicMock() + deletion_event = MagicMock() + + msg = _make_service_bus_message(topic, request_id) + main(msg=msg, stepResultEvent=step_result, dataDeletionEvent=deletion_event) + + step_result.set.assert_not_called() + + @patch("shared_code.blob_operations_metadata.get_container_metadata", side_effect=Exception("not found")) + @patch("BlobCreatedTrigger.get_blob_info_from_topic_and_subject") + def test_v2_metadata_read_failure_reraises(self, mock_get_blob_info, mock_get_metadata): + topic = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/stalairlockgtre123" + request_id = "req-004" + mock_get_blob_info.return_value = ("stalairlockgtre123", request_id, "test.txt") + + step_result = MagicMock() + deletion_event = MagicMock() + + msg = _make_service_bus_message(topic, request_id) + with pytest.raises(Exception): + main(msg=msg, stepResultEvent=step_result, dataDeletionEvent=deletion_event) + + step_result.set.assert_not_called() diff --git a/airlock_processor/tests/test_scan_result_trigger.py b/airlock_processor/tests/test_scan_result_trigger.py new file mode 100644 index 0000000000..816cab548c --- /dev/null +++ b/airlock_processor/tests/test_scan_result_trigger.py @@ -0,0 +1,129 @@ +import json +from unittest.mock import MagicMock, patch + +import azure.functions as func +from azure.core.exceptions import ResourceNotFoundError + +from shared_code import constants +from ScanResultTrigger import main + + +def _make_message(request_id: str = "req-001", verdict: str = constants.NO_THREATS, container: str = None): + # The submitted copy uses the request id; the mutable draft upload carries the -draft suffix. + container = container if container is not None else request_id + blob_uri = f"https://stalairlockgtre123.blob.core.windows.net/{container}/test.txt" + body = json.dumps({"data": {"blobUri": blob_uri, "scanResultType": verdict}}).encode("utf-8") + msg = MagicMock(spec=func.ServiceBusMessage) + msg.get_body.return_value = body + return msg + + +def _blob_client(metadata: dict): + client = MagicMock() + client.get_blob_properties.return_value = {"metadata": metadata} + return client + + +@patch.dict("os.environ", {"ENABLE_MALWARE_SCANNING": "true"}) +@patch("ScanResultTrigger.blob_operations.get_blob_client_from_blob_info") +def test_original_upload_emits_step_result(mock_get_blob_client): + mock_get_blob_client.return_value = _blob_client({}) + output_event = MagicMock() + + main(msg=_make_message(), outputEvent=output_event) + + output_event.set.assert_called_once() + + +@patch.dict("os.environ", {"ENABLE_MALWARE_SCANNING": "true"}) +@patch("ScanResultTrigger.blob_operations.get_blob_client_from_blob_info") +def test_verdict_is_reported_as_a_fact_not_a_status(mock_get_blob_client): + mock_get_blob_client.return_value = _blob_client({}) + output_event = MagicMock() + + main(msg=_make_message(), outputEvent=output_event) + + data = output_event.set.call_args.args[0].get_json() + assert data["scan_result"] == {"clean": True, "message": None} + assert "new_status" not in data + + +@patch.dict("os.environ", {"ENABLE_MALWARE_SCANNING": "true"}) +@patch("ScanResultTrigger.blob_operations.get_blob_client_from_blob_info") +def test_malicious_verdict_carries_the_reason(mock_get_blob_client): + mock_get_blob_client.return_value = _blob_client({}) + output_event = MagicMock() + + main(msg=_make_message(verdict="Malicious"), outputEvent=output_event) + + data = output_event.set.call_args.args[0].get_json() + assert data["scan_result"] == {"clean": False, "message": "Malicious"} + + +@patch.dict("os.environ", {"ENABLE_MALWARE_SCANNING": "true"}) +@patch("ScanResultTrigger.blob_operations.get_blob_client_from_blob_info") +def test_draft_blob_verdict_is_suppressed(mock_get_blob_client): + mock_get_blob_client.return_value = _blob_client({"copied_from": '["container-prev"]'}) + output_event = MagicMock() + + main(msg=_make_message(container="req-001-draft"), outputEvent=output_event) + + output_event.set.assert_not_called() + + +@patch.dict("os.environ", {"ENABLE_MALWARE_SCANNING": "true"}) +@patch("ScanResultTrigger.blob_operations.get_blob_client_from_blob_info") +def test_submitted_copy_verdict_emitted_without_reading_the_blob(mock_get_blob_client): + """The verdict gating review must describe the submitted copy, and must not depend on reading it.""" + mock_get_blob_client.side_effect = ResourceNotFoundError("blob unavailable") + output_event = MagicMock() + + main(msg=_make_message(), outputEvent=output_event) + + output_event.set.assert_called_once() + mock_get_blob_client.assert_not_called() + + +@patch.dict("os.environ", {"ENABLE_MALWARE_SCANNING": "true"}) +@patch("ScanResultTrigger.blob_operations.get_blob_client_from_blob_info") +def test_v1_copied_blob_still_emits(mock_get_blob_client): + mock_get_blob_client.return_value = _blob_client({"copied_from": '["container-prev"]'}) + blob_uri = "https://stalimiptre123.blob.core.windows.net/req-001/test.txt" + body = json.dumps({"data": {"blobUri": blob_uri, "scanResultType": constants.NO_THREATS}}).encode("utf-8") + msg = MagicMock(spec=func.ServiceBusMessage) + msg.get_body.return_value = body + output_event = MagicMock() + + main(msg=msg, outputEvent=output_event) + + output_event.set.assert_called_once() + mock_get_blob_client.assert_not_called() + + +@patch.dict("os.environ", {"ENABLE_MALWARE_SCANNING": "true"}) +def test_export_draft_on_workspace_global_account_is_suppressed(): + # Export drafts live on the workspace-global account, so suppression must not depend on the account name. + account = constants.STORAGE_ACCOUNT_NAME_AIRLOCK_WORKSPACE_GLOBAL + "tre123" + blob_uri = f"https://{account}.blob.core.windows.net/req-001-draft/test.txt" + body = json.dumps({"data": {"blobUri": blob_uri, "scanResultType": constants.NO_THREATS}}).encode("utf-8") + msg = MagicMock(spec=func.ServiceBusMessage) + msg.get_body.return_value = body + output_event = MagicMock() + + main(msg=msg, outputEvent=output_event) + + output_event.set.assert_not_called() + + +@patch.dict("os.environ", {"ENABLE_MALWARE_SCANNING": "true"}) +def test_export_submitted_copy_on_workspace_global_account_emits(): + account = constants.STORAGE_ACCOUNT_NAME_AIRLOCK_WORKSPACE_GLOBAL + "tre123" + blob_uri = f"https://{account}.blob.core.windows.net/req-001/test.txt" + body = json.dumps({"data": {"blobUri": blob_uri, "scanResultType": constants.NO_THREATS}}).encode("utf-8") + msg = MagicMock(spec=func.ServiceBusMessage) + msg.get_body.return_value = body + output_event = MagicMock() + + main(msg=msg, outputEvent=output_event) + + output_event.set.assert_called_once() diff --git a/airlock_processor/tests/test_status_change_queue_trigger.py b/airlock_processor/tests/test_status_change_queue_trigger.py index 3714f79665..396c2f0947 100644 --- a/airlock_processor/tests/test_status_change_queue_trigger.py +++ b/airlock_processor/tests/test_status_change_queue_trigger.py @@ -4,7 +4,7 @@ from mock import MagicMock, patch from pydantic import ValidationError -from StatusChangedQueueTrigger import get_request_files, main, extract_properties, get_source_dest_for_copy, is_require_data_copy +from StatusChangedQueueTrigger import get_request_files, main, extract_properties, get_source_dest_for_copy, is_require_data_copy, get_storage_account_destination_for_copy from azure.functions.servicebus import ServiceBusMessage from shared_code import constants @@ -20,6 +20,18 @@ def test_extract_prop_valid_body_return_all_values(self): assert req_prop.type == "101112" assert req_prop.workspace_id == "ws1" + def test_extract_prop_with_review_workspace_id(self): + message_body = "{ \"data\": { \"request_id\":\"123\",\"new_status\":\"456\" ,\"previous_status\":\"789\" , \"type\":\"101112\", \"workspace_id\":\"ws1\", \"review_workspace_id\":\"rw01\" }}" + message = _mock_service_bus_message(body=message_body) + req_prop = extract_properties(message) + assert req_prop.review_workspace_id == "rw01" + + def test_extract_prop_without_review_workspace_id_defaults_to_none(self): + message_body = "{ \"data\": { \"request_id\":\"123\",\"new_status\":\"456\" ,\"previous_status\":\"789\" , \"type\":\"101112\", \"workspace_id\":\"ws1\" }}" + message = _mock_service_bus_message(body=message_body) + req_prop = extract_properties(message) + assert req_prop.review_workspace_id is None + def test_extract_prop_defaults_missing_previous_status_to_none(self): message_body = "{ \"data\": { \"request_id\":\"123\",\"new_status\":\"draft\", \"type\":\"export\", \"workspace_id\":\"ws1\" }}" message = _mock_service_bus_message(body=message_body) @@ -77,9 +89,10 @@ def test_wrong_type_raises_when_getting_storage_account_properties(self): class TestFileEnumeration(): @patch("StatusChangedQueueTrigger.set_output_event_to_report_request_files") @patch("StatusChangedQueueTrigger.get_request_files") - @patch("StatusChangedQueueTrigger.is_require_data_copy", return_value=False) + @patch("StatusChangedQueueTrigger.blob_operations.create_container") + @patch("StatusChangedQueueTrigger.blob_operations.copy_data") @patch.dict(os.environ, {"TRE_ID": "tre-id"}, clear=True) - def test_get_request_files_should_be_called_on_submit_stage(self, _, mock_get_request_files, mock_set_output_event_to_report_request_files): + def test_get_request_files_should_be_called_on_submit_stage(self, _, __, mock_get_request_files, mock_set_output_event_to_report_request_files): message_body = "{ \"data\": { \"request_id\":\"123\",\"new_status\":\"submitted\" ,\"previous_status\":\"draft\" , \"type\":\"export\", \"workspace_id\":\"ws1\" }}" message = _mock_service_bus_message(body=message_body) main(msg=message, stepResultEvent=MagicMock(), dataDeletionEvent=MagicMock()) @@ -99,22 +112,36 @@ def test_get_request_files_should_not_be_called_if_new_status_is_not_submit(self @patch("StatusChangedQueueTrigger.set_output_event_to_report_failure") @patch("StatusChangedQueueTrigger.get_request_files") @patch("StatusChangedQueueTrigger.handle_status_changed", side_effect=Exception) - def test_get_request_files_should_be_called_when_failing_during_submit_stage(self, _, mock_get_request_files, mock_set_output_event_to_report_failure): + def test_transient_error_during_submit_propagates_for_retry(self, _, mock_get_request_files, mock_set_output_event_to_report_failure): + # A non-deterministic error must escape so Service Bus retries it, rather than failing the request. message_body = "{ \"data\": { \"request_id\":\"123\",\"new_status\":\"submitted\" ,\"previous_status\":\"draft\" , \"type\":\"export\", \"workspace_id\":\"ws1\" }}" message = _mock_service_bus_message(body=message_body) - main(msg=message, stepResultEvent=MagicMock(), dataDeletionEvent=MagicMock()) + with pytest.raises(Exception): + main(msg=message, stepResultEvent=MagicMock(), dataDeletionEvent=MagicMock()) assert mock_get_request_files.called - assert mock_set_output_event_to_report_failure.called + mock_set_output_event_to_report_failure.assert_not_called() @patch("StatusChangedQueueTrigger.blob_operations.get_request_files") @patch.dict(os.environ, {"TRE_ID": "tre-id"}, clear=True) def test_get_request_files_called_with_correct_storage_account(self, mock_get_request_files): source_storage_account_for_submitted_stage = constants.STORAGE_ACCOUNT_NAME_EXPORT_INTERNAL + 'ws1' - message_body = "{ \"data\": { \"request_id\":\"123\",\"new_status\":\"submitted\" ,\"previous_status\":\"draft\" , \"type\":\"export\", \"workspace_id\":\"ws1\" }}" + message_body = "{ \"data\": { \"request_id\":\"123\",\"new_status\": \"submitted\" ,\"previous_status\":\"draft\" , \"type\":\"export\", \"workspace_id\":\"ws1\", \"airlock_version\":1 }}" + message = _mock_service_bus_message(body=message_body) + request_properties = extract_properties(message) + get_request_files(request_properties) + mock_get_request_files.assert_called_with(account_name=source_storage_account_for_submitted_stage, request_id=request_properties.request_id, container_name=None) + + @patch("StatusChangedQueueTrigger.blob_operations.container_exists", return_value=True) + @patch("StatusChangedQueueTrigger.blob_operations.get_request_files") + @patch.dict(os.environ, {"TRE_ID": "tre-id"}, clear=True) + def test_get_request_files_enumerates_the_draft_container_for_v2(self, mock_get_request_files, _): + message_body = "{ \"data\": { \"request_id\":\"123\",\"new_status\": \"submitted\" ,\"previous_status\":\"draft\" , \"type\":\"export\", \"workspace_id\":\"ws1\", \"airlock_version\":2 }}" message = _mock_service_bus_message(body=message_body) request_properties = extract_properties(message) get_request_files(request_properties) - mock_get_request_files.assert_called_with(account_name=source_storage_account_for_submitted_stage, request_id=request_properties.request_id) + # v2 holds pre-submission data in a separate draft container, so enumerating the + # request-id container would find nothing and fail the request before validation runs. + assert mock_get_request_files.call_args.kwargs["container_name"] == "123-draft" class TestFilesDeletion(): @@ -127,6 +154,50 @@ def test_delete_request_files_should_be_called_on_cancel_stage(self, mock_set_ou assert mock_set_output_event_to_trigger_container_deletion.called +class TestImportSubmitUsesReviewWorkspaceId(): + @patch.dict(os.environ, {"TRE_ID": "tre-id"}, clear=True) + def test_import_submit_destination_uses_review_workspace_id(self): + dest = get_storage_account_destination_for_copy( + new_status=constants.STAGE_SUBMITTED, + request_type=constants.IMPORT_TYPE, + short_workspace_id="ws01", + review_workspace_id="rw01" + ) + assert dest == constants.STORAGE_ACCOUNT_NAME_IMPORT_INPROGRESS + "tre-id" + + @patch.dict(os.environ, {"TRE_ID": "tre-id"}, clear=True) + def test_import_submit_destination_falls_back_to_tre_id_when_no_review_workspace_id(self): + dest = get_storage_account_destination_for_copy( + new_status=constants.STAGE_SUBMITTED, + request_type=constants.IMPORT_TYPE, + short_workspace_id="ws01", + review_workspace_id=None + ) + assert dest == constants.STORAGE_ACCOUNT_NAME_IMPORT_INPROGRESS + "tre-id" + + @patch.dict(os.environ, {"TRE_ID": "tre-id"}, clear=True) + def test_export_submit_destination_ignores_review_workspace_id(self): + dest = get_storage_account_destination_for_copy( + new_status=constants.STAGE_SUBMITTED, + request_type=constants.EXPORT_TYPE, + short_workspace_id="ws01", + review_workspace_id="rw01" + ) + assert dest == constants.STORAGE_ACCOUNT_NAME_EXPORT_INPROGRESS + "ws01" + + +class TestImportApproval(): + @patch("StatusChangedQueueTrigger.blob_operations.copy_data") + @patch("StatusChangedQueueTrigger.blob_operations.create_container") + @patch.dict(os.environ, {"TRE_ID": "tre-id"}, clear=True) + def test_import_approval_copies_data_in_legacy_mode(self, mock_create_container, mock_copy_data): + message_body = "{ \"data\": { \"request_id\":\"123\",\"new_status\": \"approval_in_progress\" ,\"previous_status\":\"in_review\" , \"type\":\"import\", \"workspace_id\":\"ws01\", \"airlock_version\":1 }}" + message = _mock_service_bus_message(body=message_body) + main(msg=message, stepResultEvent=MagicMock(), dataDeletionEvent=MagicMock()) + mock_create_container.assert_called_once() + mock_copy_data.assert_called_once() + + class TestMainFailurePaths(): def test_main_raises_json_decode_error_when_invalid_json(self): message = _mock_service_bus_message(body="invalid json") @@ -148,3 +219,137 @@ def _mock_service_bus_message(body: str): encoded_body = str.encode(body, "utf-8") message = ServiceBusMessage(body=encoded_body, message_id="123", user_properties={}, application_properties={}) return message + + +class TestV2MetadataMode(): + + @patch("StatusChangedQueueTrigger.blob_operations.copy_data") + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch.dict(os.environ, {"TRE_ID": "tre-id", "ENABLE_MALWARE_SCANNING": "False"}, clear=True) + def test_v2_import_approval_copies_data_without_step_result(self, mock_blob_svc, mock_copy_data): + message_body = '{ "data": { "request_id":"123","new_status":"approval_in_progress","previous_status":"in_review","type":"import","workspace_id":"ws01","airlock_version":2 }}' + message = _mock_service_bus_message(body=message_body) + step_result = MagicMock() + main(msg=message, stepResultEvent=step_result, dataDeletionEvent=MagicMock()) + mock_copy_data.assert_called_once() + step_result.set.assert_not_called() + + @patch("StatusChangedQueueTrigger.blob_operations.copy_data") + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch.dict(os.environ, {"TRE_ID": "tre-id", "ENABLE_MALWARE_SCANNING": "False"}, clear=True) + def test_v2_export_approval_copies_data_without_step_result(self, mock_blob_svc, mock_copy_data): + message_body = '{ "data": { "request_id":"123","new_status":"approval_in_progress","previous_status":"in_review","type":"export","workspace_id":"ws01","airlock_version":2 }}' + message = _mock_service_bus_message(body=message_body) + step_result = MagicMock() + main(msg=message, stepResultEvent=step_result, dataDeletionEvent=MagicMock()) + mock_copy_data.assert_called_once() + step_result.set.assert_not_called() + + @patch("StatusChangedQueueTrigger.blob_operations.delete_container") + @patch("StatusChangedQueueTrigger.blob_operations.copy_data") + @patch("StatusChangedQueueTrigger.blob_operations.container_exists", return_value=True) + @patch("StatusChangedQueueTrigger.blob_operations.get_request_files", return_value=[{"name": "test.txt", "size": 100}]) + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch.dict(os.environ, {"TRE_ID": "tre-id", "ENABLE_MALWARE_SCANNING": "False"}, clear=True) + def test_v2_submit_with_scanning_disabled_emits_in_review(self, mock_blob_svc, mock_get_files, mock_exists, mock_copy, mock_delete): + message_body = '{ "data": { "request_id":"123","new_status":"submitted","previous_status":"draft","type":"import","workspace_id":"ws01","airlock_version":2 }}' + message = _mock_service_bus_message(body=message_body) + step_result = MagicMock() + main(msg=message, stepResultEvent=step_result, dataDeletionEvent=MagicMock()) + # The draft container is copied out and removed before the request can be reported as reviewable. + mock_copy.assert_called_once() + assert mock_copy.call_args.kwargs["source_container"] == "123-draft" + mock_delete.assert_called_once_with("stalairlocktre-id", "123-draft") + assert step_result.set.call_count == 1 + second_call_event = step_result.set.call_args_list[0][0][0] + assert second_call_event.get_json()["completed_step"] == constants.STAGE_SUBMITTED + assert second_call_event.get_json()["new_status"] == constants.STAGE_IN_REVIEW + assert second_call_event.get_json()["request_files"] == [{"name": "test.txt", "size": 100}] + + @patch("StatusChangedQueueTrigger.blob_operations.container_exists", return_value=True) + @patch("StatusChangedQueueTrigger.blob_operations.get_request_files", return_value=[]) + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch.dict(os.environ, {"TRE_ID": "tre-id", "ENABLE_MALWARE_SCANNING": "False"}, clear=True) + def test_v2_submit_rejects_zero_files(self, mock_blob_svc, mock_get_files, _): + message_body = '{ "data": { "request_id":"123","new_status":"submitted","previous_status":"draft","type":"import","workspace_id":"ws01","airlock_version":2 }}' + message = _mock_service_bus_message(body=message_body) + step_result = MagicMock() + main(msg=message, stepResultEvent=step_result, dataDeletionEvent=MagicMock()) + assert step_result.set.call_args_list[-1][0][0].get_json()["new_status"] == constants.STAGE_FAILED + + @patch("StatusChangedQueueTrigger.blob_operations.container_exists", return_value=True) + @patch("StatusChangedQueueTrigger.blob_operations.get_request_files", return_value=[{"name": "a.txt", "size": 1}, {"name": "b.txt", "size": 2}]) + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch.dict(os.environ, {"TRE_ID": "tre-id", "ENABLE_MALWARE_SCANNING": "False"}, clear=True) + def test_v2_submit_rejects_multiple_files(self, mock_blob_svc, mock_get_files, _): + message_body = '{ "data": { "request_id":"123","new_status":"submitted","previous_status":"draft","type":"import","workspace_id":"ws01","airlock_version":2 }}' + message = _mock_service_bus_message(body=message_body) + step_result = MagicMock() + main(msg=message, stepResultEvent=step_result, dataDeletionEvent=MagicMock()) + assert step_result.set.call_args_list[-1][0][0].get_json()["new_status"] == constants.STAGE_FAILED + + @patch("StatusChangedQueueTrigger.blob_operations.get_request_files", return_value=[{"name": "test.txt", "size": 1}]) + @patch("StatusChangedQueueTrigger.blob_operations.delete_container") + @patch("StatusChangedQueueTrigger.blob_operations.copy_data") + @patch("StatusChangedQueueTrigger.blob_operations.container_exists", return_value=True) + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch.dict(os.environ, {"TRE_ID": "tre-id", "ENABLE_MALWARE_SCANNING": "True"}, clear=True) + def test_v2_submit_with_scanning_enabled_does_not_emit_in_review(self, mock_blob_svc, _, mock_copy, mock_delete, mock_files): + message_body = '{ "data": { "request_id":"123","new_status":"submitted","previous_status":"draft","type":"import","workspace_id":"ws01","airlock_version":2 }}' + message = _mock_service_bus_message(body=message_body) + step_result = MagicMock() + main(msg=message, stepResultEvent=step_result, dataDeletionEvent=MagicMock()) + # Scanning is enabled, so the seal emits only a files result and leaves the in_review move to the scan verdict. + assert step_result.set.call_count == 1 + assert step_result.set.call_args.args[0].get_json().get("new_status") != constants.STAGE_IN_REVIEW + + @patch("StatusChangedQueueTrigger.blob_operations.delete_container") + @patch("StatusChangedQueueTrigger.blob_operations.copy_data") + @patch("StatusChangedQueueTrigger.blob_operations.container_exists", return_value=True) + @patch("StatusChangedQueueTrigger.blob_operations.get_request_files", return_value=[{"name": "test.txt", "size": 100}]) + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch.dict(os.environ, {"TRE_ID": "tre-id", "ENABLE_MALWARE_SCANNING": "False"}, clear=True) + def test_v2_submit_with_scanning_disabled_carries_files_with_the_transition(self, mock_blob_svc, mock_get_files, mock_exists, mock_copy, mock_delete): + # With scanning off, BlobCreatedTrigger also advances the request but carries no files, + # so this event must carry them or the race can leave the request with no file metadata. + message_body = '{ "data": { "request_id":"123","new_status":"submitted","previous_status":"draft","type":"import","workspace_id":"ws01","airlock_version":2 }}' + message = _mock_service_bus_message(body=message_body) + step_result = MagicMock() + main(msg=message, stepResultEvent=step_result, dataDeletionEvent=MagicMock()) + + data = step_result.set.call_args.args[0].get_json() + assert data["new_status"] == constants.STAGE_IN_REVIEW + assert data["request_files"] == [{"name": "test.txt", "size": 100}] + + @patch("StatusChangedQueueTrigger.blob_operations.delete_container") + @patch("StatusChangedQueueTrigger.blob_operations.copy_data") + @patch("StatusChangedQueueTrigger.blob_operations.container_exists", side_effect=lambda account, container: not container.endswith("-draft")) + @patch("StatusChangedQueueTrigger.blob_operations.get_request_files", return_value=[{"name": "test.txt", "size": 100}]) + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch.dict(os.environ, {"TRE_ID": "tre-id", "ENABLE_MALWARE_SCANNING": "True"}, clear=True) + def test_v2_redelivered_submit_resumes_from_the_sealed_container(self, mock_blob_svc, mock_get_files, mock_exists, mock_copy, mock_delete): + message_body = '{ "data": { "request_id":"123","new_status":"submitted","previous_status":"draft","type":"import","workspace_id":"ws01","airlock_version":2 }}' + message = _mock_service_bus_message(body=message_body) + step_result = MagicMock() + main(msg=message, stepResultEvent=step_result, dataDeletionEvent=MagicMock()) + + # The data is already sealed, so it must not be copied again, but the result still has to be published + # or the request would stay in Submitted forever. + mock_copy.assert_not_called() + mock_delete.assert_not_called() + assert step_result.set.call_count == 1 + + @patch("StatusChangedQueueTrigger.blob_operations.delete_container") + @patch("StatusChangedQueueTrigger.blob_operations.copy_data") + @patch("StatusChangedQueueTrigger.blob_operations.container_exists", return_value=False) + @patch("StatusChangedQueueTrigger.blob_operations.get_request_files", return_value=[{"name": "test.txt", "size": 100}]) + @patch("shared_code.blob_operations_metadata.BlobServiceClient") + @patch.dict(os.environ, {"TRE_ID": "tre-id", "ENABLE_MALWARE_SCANNING": "True"}, clear=True) + def test_v2_submit_fails_when_no_data_can_be_found(self, mock_blob_svc, mock_get_files, mock_exists, mock_copy, mock_delete): + message_body = '{ "data": { "request_id":"123","new_status":"submitted","previous_status":"draft","type":"import","workspace_id":"ws01","airlock_version":2 }}' + message = _mock_service_bus_message(body=message_body) + step_result = MagicMock() + main(msg=message, stepResultEvent=step_result, dataDeletionEvent=MagicMock()) + + mock_copy.assert_not_called() + assert step_result.set.call_args.args[0].get_json()["new_status"] == constants.STAGE_FAILED diff --git a/api_app/_version.py b/api_app/_version.py index ae62eb6326..31eec62af4 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.26.5" +__version__ = "0.27.28" diff --git a/api_app/api/dependencies/airlock.py b/api_app/api/dependencies/airlock.py index c824352ee5..65e370e970 100644 --- a/api_app/api/dependencies/airlock.py +++ b/api_app/api/dependencies/airlock.py @@ -17,5 +17,8 @@ async def get_airlock_request_by_id(airlock_request_id: UUID4, airlock_request_r raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=strings.STATE_STORE_ENDPOINT_NOT_RESPONDING) -async def get_airlock_request_by_id_from_path(airlock_request_id: UUID4 = Path(...), airlock_request_repo=Depends(get_repository(AirlockRequestRepository))) -> AirlockRequest: - return await get_airlock_request_by_id(airlock_request_id, airlock_request_repo) +async def get_airlock_request_by_id_from_path(airlock_request_id: UUID4 = Path(...), workspace_id: UUID4 = Path(...), airlock_request_repo=Depends(get_repository(AirlockRequestRepository))) -> AirlockRequest: + airlock_request = await get_airlock_request_by_id(airlock_request_id, airlock_request_repo) + if airlock_request.workspaceId != str(workspace_id): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=strings.AIRLOCK_REQUEST_DOES_NOT_EXIST) + return airlock_request diff --git a/api_app/api/routes/airlock.py b/api_app/api/routes/airlock.py index dbf96cc117..b83f064ccc 100644 --- a/api_app/api/routes/airlock.py +++ b/api_app/api/routes/airlock.py @@ -26,6 +26,7 @@ from services.airlock import create_review_vm, review_airlock_request, get_airlock_container_link, get_allowed_actions, save_and_publish_event_airlock_request, update_and_publish_event_airlock_request, \ enrich_requests_with_allowed_actions, get_airlock_requests_by_user_and_workspace, cancel_request, revoke_request +from services.legacy_airlock_guard import ensure_workspace_airlock_version_supported from services.logging import logger airlock_workspace_router = APIRouter(dependencies=[Depends(require_workspace_owner_or_researcher_or_airlock_manager)]) @@ -41,7 +42,10 @@ async def create_draft_request(airlock_request_input: AirlockRequestInCreate, us if workspace.properties.get("enable_airlock") is False: raise HTTPException(status_code=status_code.HTTP_405_METHOD_NOT_ALLOWED, detail=strings.AIRLOCK_NOT_ENABLED_IN_WORKSPACE) try: - airlock_request = airlock_request_repo.create_airlock_request_item(airlock_request_input, workspace.id, user) + # Missing versions identify unmigrated v1 workspaces. + airlock_version = workspace.properties.get("airlock_version", 1) + ensure_workspace_airlock_version_supported(workspace.properties, default_version=1) + airlock_request = airlock_request_repo.create_airlock_request_item(airlock_request_input, workspace.id, user, airlock_version=airlock_version) await save_and_publish_event_airlock_request(airlock_request, airlock_request_repo, user, workspace) allowed_actions = get_allowed_actions(airlock_request, user, airlock_request_repo) return AirlockRequestWithAllowedUserActions(airlockRequest=airlock_request, allowedUserActions=allowed_actions) diff --git a/api_app/api/routes/migrations.py b/api_app/api/routes/migrations.py index 26a27353bd..ef205e5db7 100644 --- a/api_app/api/routes/migrations.py +++ b/api_app/api/routes/migrations.py @@ -1,7 +1,8 @@ from fastapi import APIRouter, Depends, HTTPException, status from auth.rbac import require_tre_admin from resources import strings -from models.schemas.migrations import MigrationOutList +from models.schemas.migrations import Migration, MigrationOutList +from db.repositories.workspaces import WorkspaceRepository from services.logging import logger migrations_core_router = APIRouter(dependencies=[Depends(require_tre_admin)]) @@ -16,12 +17,12 @@ async def migrate_database(): try: migrations = list() - # ADD MIGRATIONS HERE - # Examples of migrations can be found in this file: - # https://github.com/microsoft/AzureTRE/blob/v0.22.0/api_app/api/routes/migrations.py#L32-L84 - # and this folder: - # https://github.com/microsoft/AzureTRE/tree/v0.22.0/api_app/db/migrations - logger.info("No migrations exist.") + # Preserve legacy storage routing for pre-v2 workspaces, which the bundle would otherwise + # redeploy as v2 and destroy their legacy storage. + workspace_repo = await WorkspaceRepository.create() + migrated_ids = await workspace_repo.set_default_airlock_version_for_legacy_workspaces() + logger.info(f"Set default airlock_version=1 on {len(migrated_ids)} legacy workspace(s).") + migrations.append(Migration(issueNumber="5048", status=f"Set airlock_version=1 on {len(migrated_ids)} legacy workspace(s)")) return MigrationOutList(migrations=migrations) except Exception as e: diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index 04ea2d6513..b43603043d 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -13,6 +13,9 @@ from db.repositories.resources_history import ResourceHistoryRepository from db.repositories.user_resources import UserResourceRepository from db.repositories.workspaces import WorkspaceRepository +from db.repositories.airlock_requests import AirlockRequestRepository +from services.legacy_airlock_guard import ensure_airlock_version_change_allowed, ensure_workspace_airlock_version_supported +from services.airlock import delete_workspace_airlock_containers from db.repositories.workspace_services import WorkspaceServiceRepository from models.domain.resource import ResourceType from models.domain.workspace import WorkspaceAuth, WorkspaceRole @@ -99,6 +102,8 @@ async def create_workspace(workspace_create: WorkspaceInCreate, response: Respon # TODO: This requires Directory.ReadAll ( Application.Read.All ) to be enabled in the Azure AD application to enable a users workspaces to be listed. This should be made optional. auth_info = extract_auth_information(workspace_create.properties) workspace, resource_template = await workspace_repo.create_workspace_item(workspace_create, auth_info, user.id, user.roles) + # Validate the resolved (template-derived) airlock version rather than the raw request. + ensure_workspace_airlock_version_supported(workspace.properties, default_version=1) except (ValidationError, ValueError) as e: logger.exception("Failed to create workspace model instance") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) @@ -128,8 +133,10 @@ async def create_workspace(workspace_create: WorkspaceInCreate, response: Respon @workspaces_core_router.patch("/workspaces/{workspace_id}", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_UPDATE_WORKSPACE, dependencies=[Depends(require_tre_admin)]) -async def patch_workspace(resource_patch: ResourcePatch, response: Response, user=Depends(require_tre_admin), workspace=Depends(get_workspace_by_id_from_path), workspace_repo: WorkspaceRepository = Depends(get_repository(WorkspaceRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), etag: str = Header(...), force_version_update: bool = False) -> OperationInResponse: +async def patch_workspace(resource_patch: ResourcePatch, response: Response, user=Depends(require_tre_admin), workspace=Depends(get_workspace_by_id_from_path), workspace_repo: WorkspaceRepository = Depends(get_repository(WorkspaceRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), airlock_request_repo=Depends(get_repository(AirlockRequestRepository)), etag: str = Header(...), force_version_update: bool = False) -> OperationInResponse: try: + await ensure_airlock_version_change_allowed(workspace, resource_patch, airlock_request_repo) + ensure_workspace_airlock_version_supported({**workspace.properties, **(resource_patch.properties or {})}, default_version=1) is_disablement = resource_patch.isEnabled is not None and not resource_patch.isEnabled if is_disablement: await cascaded_update_resource(resource_patch, workspace, user, force_version_update, resource_template_repo=resource_template_repo, resource_history_repo=resource_history_repo, resource_repo=workspace_repo) @@ -151,12 +158,14 @@ async def patch_workspace(resource_patch: ResourcePatch, response: Response, use raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.ETAG_CONFLICT) except ValidationError as v: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=v.message) + except ValueError as ve: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(ve)) except (MajorVersionUpdateDenied, TargetTemplateVersionDoesNotExist, VersionDowngradeDenied) as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) @workspaces_core_router.delete("/workspaces/{workspace_id}", response_model=OperationInResponse, name=strings.API_DELETE_WORKSPACE, dependencies=[Depends(require_tre_admin)]) -async def delete_workspace(response: Response, user=Depends(require_tre_admin), workspace=Depends(get_workspace_by_id_from_path), operations_repo=Depends(get_repository(OperationRepository)), workspace_repo=Depends(get_repository(WorkspaceRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> OperationInResponse: +async def delete_workspace(response: Response, user=Depends(require_tre_admin), workspace=Depends(get_workspace_by_id_from_path), operations_repo=Depends(get_repository(OperationRepository)), workspace_repo=Depends(get_repository(WorkspaceRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), airlock_request_repo=Depends(get_repository(AirlockRequestRepository))) -> OperationInResponse: if await delete_validation(workspace, workspace_repo): operation = await send_uninstall_message( resource=workspace, @@ -169,6 +178,9 @@ async def delete_workspace(response: Response, user=Depends(require_tre_admin), is_cascade=True ) + # Shared storage outlives the workspace, so remove its containers once uninstall is queued. + await delete_workspace_airlock_containers(workspace, airlock_request_repo) + response.headers["Location"] = construct_location_header(operation) return OperationInResponse(operation=operation) diff --git a/api_app/core/config.py b/api_app/core/config.py index d2f1cf1fa4..91da3b661c 100644 --- a/api_app/core/config.py +++ b/api_app/core/config.py @@ -69,6 +69,9 @@ AIRLOCK_SAS_TOKEN_EXPIRY_PERIOD_IN_HOURS: int = config("AIRLOCK_SAS_TOKEN_EXPIRY_PERIOD_IN_HOURS", default=1) ENABLE_AIRLOCK_EMAIL_CHECK: bool = config("ENABLE_AIRLOCK_EMAIL_CHECK", cast=bool, default=False) +ENABLE_LEGACY_AIRLOCK: bool = config("ENABLE_LEGACY_AIRLOCK", cast=bool, default=True) + +APP_GATEWAY_FQDN: str = config("APP_GATEWAY_FQDN", default="") API_ROOT_SCOPE: str = f"api://{API_CLIENT_ID}/user_impersonation" diff --git a/api_app/core/credentials.py b/api_app/core/credentials.py index 427f62b529..00898d2646 100644 --- a/api_app/core/credentials.py +++ b/api_app/core/credentials.py @@ -7,6 +7,7 @@ DefaultAzureCredential, ManagedIdentityCredential, ChainedTokenCredential, + ClientAssertionCredential, ) from azure.identity.aio import ( DefaultAzureCredential as DefaultAzureCredentialASync, @@ -14,6 +15,21 @@ ChainedTokenCredential as ChainedTokenCredentialASync, ) +# Workload-identity token-exchange audience. Sovereign clouds use their own audience, +# and it must match the audience on the workspace signer's federated credential. + + +def _token_exchange_audience() -> str: + host = urlparse(AAD_AUTHORITY_URL).netloc.lower() + if host.endswith(".us"): + return "api://AzureADTokenExchangeUSGov/.default" + if host.endswith(".cn"): + return "api://AzureADTokenExchangeChina/.default" + return "api://AzureADTokenExchange/.default" + + +TOKEN_EXCHANGE_AUDIENCE = _token_exchange_audience() # nosec B105 - token exchange audience, not a secret + def get_credential() -> TokenCredential: if MANAGED_IDENTITY_CLIENT_ID: @@ -54,3 +70,18 @@ async def get_credential_async_context() -> TokenCredential: credential = await get_credential_async() yield credential await credential.close() + + +def get_airlock_signer_credential(signer_client_id: str, tenant_id: str) -> TokenCredential: + """Authenticate as a workspace's federated airlock signer.""" + managed_identity = ManagedIdentityCredential(client_id=MANAGED_IDENTITY_CLIENT_ID) + + def _get_managed_identity_assertion() -> str: + return managed_identity.get_token(TOKEN_EXCHANGE_AUDIENCE).token + + return ClientAssertionCredential( + tenant_id=tenant_id, + client_id=signer_client_id, + func=_get_managed_identity_assertion, + authority=urlparse(AAD_AUTHORITY_URL).netloc, + ) diff --git a/api_app/db/repositories/airlock_requests.py b/api_app/db/repositories/airlock_requests.py index 21184c6e61..54fee89d75 100644 --- a/api_app/db/repositories/airlock_requests.py +++ b/api_app/db/repositories/airlock_requests.py @@ -19,8 +19,19 @@ from db.repositories.base import BaseRepository from services.logging import logger +_UNSET = object() + class AirlockRequestRepository(BaseRepository): + FINAL_AIRLOCK_STATUSES = [ + AirlockRequestStatus.Approved, + AirlockRequestStatus.Rejected, + AirlockRequestStatus.Blocked, + AirlockRequestStatus.Cancelled, + AirlockRequestStatus.Failed, + AirlockRequestStatus.Revoked + ] + @classmethod async def create(cls): cls = AirlockRequestRepository() @@ -60,6 +71,28 @@ async def update_airlock_request_item(self, original_request: AirlockRequest, ne def airlock_requests_query(): return 'SELECT * FROM c' + async def get_data_retaining_airlock_request_ids_for_workspace(self, workspace_id: str) -> List[str]: + # Any request may still have a container, including cancelled ones whose async deletion + # failed or has not completed. Cleanup is idempotent (missing containers are ignored), + # so return every request to avoid orphaning data when the workspace is deleted. + query = "SELECT c.id FROM c WHERE c.workspaceId = @workspaceId" + parameters = [ + {"name": "@workspaceId", "value": str(workspace_id)} + ] + requests = await self.query(query=query, parameters=parameters) + return [request["id"] for request in requests] + + async def get_in_flight_airlock_request_ids_for_workspace(self, workspace_id: str) -> List[str]: + # Requests in a final state keep their data but will never move between stages, + # so only in-flight requests block a change of storage layout. + query = "SELECT c.id FROM c WHERE c.workspaceId = @workspaceId AND NOT ARRAY_CONTAINS(@finalStatuses, c.status)" + parameters = [ + {"name": "@workspaceId", "value": str(workspace_id)}, + {"name": "@finalStatuses", "value": [status.value for status in self.FINAL_AIRLOCK_STATUSES]} + ] + requests = await self.query(query=query, parameters=parameters) + return [request["id"] for request in requests] + def validate_status_update(self, current_status: AirlockRequestStatus, new_status: AirlockRequestStatus) -> bool: # Define valid transitions @@ -107,7 +140,7 @@ def validate_status_update(self, current_status: AirlockRequestStatus, new_statu allowed_transitions = valid_transitions.get(current_status, set()) return new_status in allowed_transitions - def create_airlock_request_item(self, airlock_request_input: AirlockRequestInCreate, workspace_id: str, user) -> AirlockRequest: + def create_airlock_request_item(self, airlock_request_input: AirlockRequestInCreate, workspace_id: str, user, airlock_version: int = 1) -> AirlockRequest: full_airlock_request_id = str(uuid.uuid4()) resource_spec_parameters = {**self.get_airlock_request_spec_params()} @@ -123,7 +156,8 @@ def create_airlock_request_item(self, airlock_request_input: AirlockRequestInCre updatedBy=user, updatedWhen=datetime.now(UTC).timestamp(), properties=resource_spec_parameters, - reviews=[] + reviews=[], + airlock_version=airlock_version ) return airlock_request @@ -194,21 +228,24 @@ async def update_airlock_request( request_files: Optional[List[AirlockFile]] = None, status_message: Optional[str] = None, airlock_review: Optional[AirlockReview] = None, - review_user_resource: Optional[AirlockReviewUserResource] = None) -> AirlockRequest: - updated_request = self._build_updated_request( - original_request=original_request, + review_user_resource: Optional[AirlockReviewUserResource] = None, + scan_result=_UNSET) -> AirlockRequest: + # Preserve every field when rebuilding after an ETag conflict. + update_fields = dict( new_status=new_status, request_files=request_files, status_message=status_message, airlock_review=airlock_review, review_user_resource=review_user_resource, + scan_result=scan_result, updated_by=updated_by) + updated_request = self._build_updated_request(original_request=original_request, **update_fields) try: db_response = await self.update_airlock_request_item(original_request, updated_request, updated_by, {"previousStatus": original_request.status}) except CosmosAccessConditionFailedError: logger.warning(f"ETag mismatch for request ID: '{original_request.id}'. Retrying.") original_request = await self.get_airlock_request_by_id(original_request.id) - updated_request = self._build_updated_request(original_request=original_request, new_status=new_status, request_files=request_files, status_message=status_message, airlock_review=airlock_review) + updated_request = self._build_updated_request(original_request=original_request, **update_fields) db_response = await self.update_airlock_request_item(original_request, updated_request, updated_by, {"previousStatus": original_request.status}) return db_response @@ -251,6 +288,7 @@ def _build_updated_request( status_message: Optional[Optional[str]] = None, airlock_review: Optional[AirlockReview] = None, review_user_resource: Optional[AirlockReviewUserResource] = None, + scan_result=_UNSET, updated_by: Optional[Union[User, dict]] = None) -> AirlockRequest: updated_request = copy.deepcopy(original_request) @@ -258,6 +296,9 @@ def _build_updated_request( self._validate_status_update(current_status=original_request.status, new_status=new_status) updated_request.status = new_status + if scan_result is not _UNSET: + updated_request.scanResult = scan_result + if status_message is not None: updated_request.statusMessage = status_message diff --git a/api_app/db/repositories/workspaces.py b/api_app/db/repositories/workspaces.py index 14656c0a63..7717a71fc2 100644 --- a/api_app/db/repositories/workspaces.py +++ b/api_app/db/repositories/workspaces.py @@ -9,6 +9,7 @@ from models.domain.authentication import User import resources.strings as strings +from resources import constants from core import config, credentials from azure.core.exceptions import HttpResponseError from db.errors import EntityDoesNotExist, InvalidInput, ResourceIsNotDeployed, StorageAccountNameGenerationTimeout, StorageAccountNameCheckFailed @@ -65,6 +66,18 @@ async def get_active_workspaces(self) -> List[Workspace]: workspaces = await self.query(query=query, parameters=parameters) return TypeAdapter(List[Workspace]).validate_python(workspaces) + async def set_default_airlock_version_for_legacy_workspaces(self) -> List[str]: + # The bundle now defaults to v2, so pre-v2 workspaces must be stamped v1 explicitly + # or a later redeploy would migrate them - and destroy their legacy storage. + query = 'SELECT * FROM c WHERE c.resourceType = @resourceType AND NOT IS_DEFINED(c.properties.airlock_version)' + parameters = [{'name': '@resourceType', 'value': ResourceType.Workspace}] + migrated = [] + for workspace in await self.query(query=query, parameters=parameters): + workspace['properties']['airlock_version'] = 1 + await self.update_item_dict(workspace) + migrated.append(workspace['id']) + return migrated + async def get_deployed_workspace_by_id(self, workspace_id: str, operations_repo: OperationRepository) -> Workspace: workspace = await self.get_workspace_by_id(workspace_id) @@ -128,6 +141,15 @@ async def name_check(): auto_app_registration_param = {"register_aad_application": self.automatically_create_application_registration(workspace_input.properties)} workspace_owner_param = {"workspace_owner_object_id": self.get_workspace_owner(workspace_input.properties, workspace_owner_object_id)} + # Derive airlock_version from the template: a template that doesn't declare it is legacy (v1) and + # must not be stamped v2, which would route airlock to consolidated storage it never provisioned. + if "airlock_version" in template.properties: + template_default = template.properties["airlock_version"].default + default_airlock_version = template_default if template_default is not None else constants.DEFAULT_AIRLOCK_VERSION + else: + default_airlock_version = 1 + airlock_version_param = {"airlock_version": workspace_input.properties.get("airlock_version", default_airlock_version)} + # we don't want something in the input to overwrite the system parameters, # so dict.update can't work. Priorities from right to left. resource_spec_parameters = {**workspace_input.properties, @@ -135,6 +157,7 @@ async def name_check(): **address_spaces_param, **auto_app_registration_param, **workspace_owner_param, + **airlock_version_param, **auth_info, **self.get_workspace_spec_params(full_workspace_id)} diff --git a/api_app/event_grid/event_sender.py b/api_app/event_grid/event_sender.py index 588a74e20a..fedb2e858b 100644 --- a/api_app/event_grid/event_sender.py +++ b/api_app/event_grid/event_sender.py @@ -5,21 +5,32 @@ from models.domain.events import AirlockNotificationRequestData, AirlockNotificationWorkspaceData, StatusChangedData, AirlockNotificationData from event_grid.helpers import publish_event from core import config -from models.domain.airlock_request import AirlockRequest, AirlockRequestStatus +from models.domain.airlock_request import AirlockRequest, AirlockRequestStatus, AirlockRequestType from models.domain.workspace import Workspace from services.logging import logger -async def send_status_changed_event(airlock_request: AirlockRequest, previous_status: Optional[AirlockRequestStatus]): +async def send_status_changed_event(airlock_request: AirlockRequest, previous_status: Optional[AirlockRequestStatus], workspace: Optional[Workspace] = None): request_id = airlock_request.id new_status = airlock_request.status.value previous_status = previous_status.value if previous_status else None request_type = airlock_request.type.value short_workspace_id = airlock_request.workspaceId[-4:] + # V2 ABAC uses full workspace IDs; legacy account names use short IDs. + workspace_id_for_event = airlock_request.workspaceId if airlock_request.airlock_version >= 2 else short_workspace_id + + review_workspace_id = None + if workspace and airlock_request.type == AirlockRequestType.Import: + try: + full_review_ws_id = workspace.properties["airlock_review_config"]["import"]["import_vm_workspace_id"] + review_workspace_id = full_review_ws_id if airlock_request.airlock_version >= 2 else full_review_ws_id[-4:] + except (KeyError, TypeError): + pass + status_changed_event = EventGridEvent( event_type="statusChanged", - data=StatusChangedData(request_id=request_id, new_status=new_status, previous_status=previous_status, type=request_type, workspace_id=short_workspace_id).model_dump(mode="json"), + data=StatusChangedData(request_id=request_id, new_status=new_status, previous_status=previous_status, type=request_type, workspace_id=workspace_id_for_event, review_workspace_id=review_workspace_id, airlock_version=airlock_request.airlock_version).model_dump(mode="json"), subject=f"{request_id}/statusChanged", data_version="2.0" ) diff --git a/api_app/models/domain/airlock_operations.py b/api_app/models/domain/airlock_operations.py index ad397c94fe..4f90d775ab 100644 --- a/api_app/models/domain/airlock_operations.py +++ b/api_app/models/domain/airlock_operations.py @@ -11,6 +11,7 @@ class EventGridMessageData(AzureTREModel): request_id: str = Field(title="", description="") request_files: Optional[List[AirlockFile]] = Field(default=None, title="", description="") status_message: Optional[str] = Field(default=None, title="", description="") + scan_result: Optional[dict] = Field(default=None, title="", description="") class StepResultStatusUpdateMessage(AzureTREModel): diff --git a/api_app/models/domain/airlock_request.py b/api_app/models/domain/airlock_request.py index 23ab8a626c..edd5d59bc5 100644 --- a/api_app/models/domain/airlock_request.py +++ b/api_app/models/domain/airlock_request.py @@ -114,6 +114,8 @@ class AirlockRequest(AzureTREModel): reviews: Optional[List[AirlockReview]] = None etag: Optional[str] = Field(None, title="_etag", alias="_etag") reviewUserResources: Dict[str, AirlockReviewUserResource] = Field(default_factory=dict, title="User resources created for Airlock Reviews") + airlock_version: int = Field(default=1, title="Airlock version", description="1 = legacy per-stage storage, 2 = consolidated metadata-based storage. Requests are created with an explicit version, so a missing value identifies a pre-v2 request whose data is in legacy storage.") + scanResult: Optional[dict] = Field(None, title="Scan result", description="Malware scan verdict recorded independently of the request status. Shape: {clean, message}.") # SQL API CosmosDB saves ETag as an escaped string: https://github.com/microsoft/AzureTRE/issues/1931 @field_validator("etag", mode="before") diff --git a/api_app/models/domain/events.py b/api_app/models/domain/events.py index 6ec48888c2..1d4802dd50 100644 --- a/api_app/models/domain/events.py +++ b/api_app/models/domain/events.py @@ -40,3 +40,5 @@ class StatusChangedData(AzureTREModel): previous_status: Optional[str] = None type: str workspace_id: str + review_workspace_id: Optional[str] = None + airlock_version: int = 1 diff --git a/api_app/resources/constants.py b/api_app/resources/constants.py index c6f60cec0c..5ace54a332 100644 --- a/api_app/resources/constants.py +++ b/api_app/resources/constants.py @@ -4,6 +4,26 @@ IMPORT_TYPE = "import" EXPORT_TYPE = "export" + +# New workspaces default to consolidated storage. +DEFAULT_AIRLOCK_VERSION = 2 +DRAFT_CONTAINER_SUFFIX = "-draft" + +STORAGE_ACCOUNT_NAME_AIRLOCK_CORE = "stalairlock{}" +STORAGE_ACCOUNT_NAME_AIRLOCK_WORKSPACE_GLOBAL = "stalairlockg{}" + +STAGE_IMPORT_EXTERNAL = "import-external" +STAGE_IMPORT_IN_PROGRESS = "import-in-progress" +STAGE_IMPORT_APPROVED = "import-approved" +STAGE_IMPORT_REJECTED = "import-rejected" +STAGE_IMPORT_BLOCKED = "import-blocked" +STAGE_EXPORT_INTERNAL = "export-internal" +STAGE_EXPORT_IN_PROGRESS = "export-in-progress" +STAGE_EXPORT_APPROVED = "export-approved" +STAGE_EXPORT_REJECTED = "export-rejected" +STAGE_EXPORT_BLOCKED = "export-blocked" + +# Retained during legacy storage migration. # Import STORAGE_ACCOUNT_NAME_IMPORT_EXTERNAL = "stalimex{}" STORAGE_ACCOUNT_NAME_IMPORT_INPROGRESS = "stalimip{}" diff --git a/api_app/service_bus/airlock_request_status_update.py b/api_app/service_bus/airlock_request_status_update.py index c4d50b80d0..c3a36b6a76 100644 --- a/api_app/service_bus/airlock_request_status_update.py +++ b/api_app/service_bus/airlock_request_status_update.py @@ -7,7 +7,7 @@ from fastapi import HTTPException from pydantic import ValidationError, TypeAdapter -from api.dependencies.airlock import get_airlock_request_by_id_from_path +from api.dependencies.airlock import get_airlock_request_by_id from services.airlock import update_and_publish_event_airlock_request from services.logging import logger, tracer from db.repositories.workspaces import WorkspaceRepository @@ -94,6 +94,30 @@ async def process_message(self, msg): return complete_message + async def _complete_submission_if_ready(self, airlock_request): + """A submitted request advances only once file validation and the scan verdict are both in.""" + if airlock_request.status != AirlockRequestStatus.Submitted: + return + if not airlock_request.files or airlock_request.scanResult is None: + return + + clean = airlock_request.scanResult.get("clean") + if clean is True: + new_status, status_message = AirlockRequestStatus.InReview, None + elif clean is False: + new_status, status_message = AirlockRequestStatus.BlockingInProgress, airlock_request.scanResult.get("message") + else: + # A malformed verdict must never be read as clean, so leave the request where it is. + logger.error(f"Request {airlock_request.id} has a malformed scan verdict, not advancing: {airlock_request.scanResult}") + return + + logger.info(f"Completing submission for request {airlock_request.id} with status '{new_status}'.") + workspace = await self.workspace_repo.get_workspace_by_id(airlock_request.workspaceId) + await update_and_publish_event_airlock_request( + airlock_request=airlock_request, airlock_request_repo=self.airlock_request_repo, + updated_by=airlock_request.updatedBy, workspace=workspace, + new_status=new_status, status_message=status_message) + async def update_status_in_database(self, step_result_message: StepResultStatusUpdateMessage): """ Updates an airlock request and with the new status from step_result message contents. @@ -103,20 +127,52 @@ async def update_status_in_database(self, step_result_message: StepResultStatusU try: step_result_data = step_result_message.data airlock_request_id = step_result_data.request_id - current_status = step_result_data.completed_step + completed_step = step_result_data.completed_step new_status = AirlockRequestStatus(step_result_data.new_status) if step_result_data.new_status else None status_message = step_result_data.status_message request_files = step_result_data.request_files + scan_result = step_result_data.scan_result # Find the airlock request by id - airlock_request = await get_airlock_request_by_id_from_path(airlock_request_id=airlock_request_id, airlock_request_repo=self.airlock_request_repo) - # Validate that the airlock request status is the same as current status - if airlock_request.status == current_status: + airlock_request = await get_airlock_request_by_id(airlock_request_id=airlock_request_id, airlock_request_repo=self.airlock_request_repo) + if airlock_request.status in AirlockRequestRepository.FINAL_AIRLOCK_STATUSES: + logger.info(f"Discarding step result for request {airlock_request_id} in final status '{airlock_request.status}'.") + return True + + # File enumeration is a fact about the data and can arrive after the transition it + # accompanied, so record it regardless of the current status rather than discarding it. + if request_files and not airlock_request.files: + airlock_request = await self.airlock_request_repo.update_airlock_request( + original_request=airlock_request, + updated_by=airlock_request.updatedBy, + request_files=request_files) + result = True + + if scan_result is not None: + # A verdict is a fact about the data, so it is recorded in any non-final status. + airlock_request = await self.airlock_request_repo.update_airlock_request( + original_request=airlock_request, + updated_by=airlock_request.updatedBy, + scan_result=scan_result) + result = True + elif new_status is None: + # A file-only result carries no transition; the facts above are enough. Acknowledge it + # without republishing, which previously risked a submitted -> submitted event loop. + result = True + elif airlock_request.status == completed_step: workspace = await self.workspace_repo.get_workspace_by_id(airlock_request.workspaceId) # update to new status and send to event grid - await update_and_publish_event_airlock_request(airlock_request=airlock_request, airlock_request_repo=self.airlock_request_repo, updated_by=airlock_request.updatedBy, workspace=workspace, new_status=new_status, request_files=request_files, status_message=status_message) + airlock_request = await update_and_publish_event_airlock_request(airlock_request=airlock_request, airlock_request_repo=self.airlock_request_repo, updated_by=airlock_request.updatedBy, workspace=workspace, new_status=new_status, request_files=request_files, status_message=status_message) result = True + elif airlock_request.status == new_status: + # Redelivery of a result that was already applied. Retrying forever would dead-letter a valid message. + logger.info(f"Step result for request {airlock_request_id} already applied, acknowledging duplicate.") + return True else: - logger.error(strings.STEP_RESULT_MESSAGE_STATUS_DOES_NOT_MATCH.format(airlock_request_id, current_status, airlock_request.status)) + logger.error(strings.STEP_RESULT_MESSAGE_STATUS_DOES_NOT_MATCH.format(airlock_request_id, completed_step, airlock_request.status)) + return result + + await self._complete_submission_if_ready(airlock_request) + except HTTPException as e: if e.status_code == 404: # Marking as true as this message will never succeed anyways and should be removed from the queue. @@ -124,7 +180,7 @@ async def update_status_in_database(self, step_result_message: StepResultStatusU logger.exception(strings.STEP_RESULT_ID_NOT_FOUND.format(airlock_request_id)) if e.status_code == 400: result = True - logger.exception(strings.STEP_RESULT_MESSAGE_INVALID_STATUS.format(airlock_request_id, current_status, new_status)) + logger.exception(strings.STEP_RESULT_MESSAGE_INVALID_STATUS.format(airlock_request_id, completed_step, new_status)) if e.status_code == 503: logger.exception(strings.STATE_STORE_ENDPOINT_NOT_RESPONDING) except Exception: diff --git a/api_app/services/airlock.py b/api_app/services/airlock.py index 3c84f56192..877978fbf0 100644 --- a/api_app/services/airlock.py +++ b/api_app/services/airlock.py @@ -1,6 +1,8 @@ +import asyncio from datetime import datetime, timedelta, UTC from services.logging import logger +from azure.core.exceptions import ResourceNotFoundError from azure.storage.blob import generate_container_sas, ContainerSasPermissions, BlobServiceClient from fastapi import HTTPException, status from core import config, credentials @@ -17,6 +19,7 @@ from typing import Tuple, List, Optional, Union from models.schemas.user_resource import UserResourceInCreate from services.azure_resource_status import get_azure_resource_status +from services.airlock_storage_helper import get_container_name_for_request from services.authentication import get_aad_service from resources import strings, constants @@ -26,7 +29,7 @@ from db.repositories.user_resources import UserResourceRepository from db.repositories.workspace_services import WorkspaceServiceRepository from db.repositories.operations import OperationRepository -from db.repositories.airlock_requests import AirlockRequestRepository +from db.repositories.airlock_requests import AirlockRequestRepository, _UNSET from db.repositories.resource_templates import ResourceTemplateRepository from db.repositories.resources_history import ResourceHistoryRepository @@ -36,37 +39,6 @@ STORAGE_ENDPOINT = config.STORAGE_ENDPOINT_SUFFIX -def get_account_by_request(airlock_request: AirlockRequest, workspace: Workspace) -> str: - tre_id = config.TRE_ID - short_workspace_id = workspace.id[-4:] - if airlock_request.type == constants.IMPORT_TYPE: - if airlock_request.status == AirlockRequestStatus.Draft: - return constants.STORAGE_ACCOUNT_NAME_IMPORT_EXTERNAL.format(tre_id) - elif airlock_request.status == AirlockRequestStatus.Submitted: - return constants.STORAGE_ACCOUNT_NAME_IMPORT_INPROGRESS.format(tre_id) - elif airlock_request.status == AirlockRequestStatus.InReview: - return constants.STORAGE_ACCOUNT_NAME_IMPORT_INPROGRESS.format(tre_id) - elif airlock_request.status == AirlockRequestStatus.Approved: - return constants.STORAGE_ACCOUNT_NAME_IMPORT_APPROVED.format(short_workspace_id) - elif airlock_request.status == AirlockRequestStatus.Rejected: - return constants.STORAGE_ACCOUNT_NAME_IMPORT_REJECTED.format(tre_id) - elif airlock_request.status == AirlockRequestStatus.Blocked: - return constants.STORAGE_ACCOUNT_NAME_IMPORT_BLOCKED.format(tre_id) - else: - if airlock_request.status == AirlockRequestStatus.Draft: - return constants.STORAGE_ACCOUNT_NAME_EXPORT_INTERNAL.format(short_workspace_id) - elif airlock_request.status in AirlockRequestStatus.Submitted: - return constants.STORAGE_ACCOUNT_NAME_EXPORT_INPROGRESS.format(short_workspace_id) - elif airlock_request.status == AirlockRequestStatus.InReview: - return constants.STORAGE_ACCOUNT_NAME_EXPORT_INPROGRESS.format(short_workspace_id) - elif airlock_request.status == AirlockRequestStatus.Approved: - return constants.STORAGE_ACCOUNT_NAME_EXPORT_APPROVED.format(tre_id) - elif airlock_request.status == AirlockRequestStatus.Rejected: - return constants.STORAGE_ACCOUNT_NAME_EXPORT_REJECTED.format(short_workspace_id) - elif airlock_request.status == AirlockRequestStatus.Blocked: - return constants.STORAGE_ACCOUNT_NAME_EXPORT_BLOCKED.format(short_workspace_id) - - def validate_user_allowed_to_access_storage_account(user: User, airlock_request: AirlockRequest): allowed_roles = [] @@ -103,10 +75,48 @@ def get_required_permission(airlock_request: AirlockRequest) -> ContainerSasPerm return ContainerSasPermissions(read=True, list=True) -def get_airlock_request_container_sas_token(account_name: str, - airlock_request: AirlockRequest): +def get_account_by_request(airlock_request: AirlockRequest, workspace: Workspace) -> str: + """Resolve storage account name for v1 (legacy per-stage) airlock requests.""" + tre_id = config.TRE_ID + short_workspace_id = workspace.id[-4:] + if airlock_request.type == constants.IMPORT_TYPE: + if airlock_request.status == AirlockRequestStatus.Draft: + return constants.STORAGE_ACCOUNT_NAME_IMPORT_EXTERNAL.format(tre_id) + elif airlock_request.status == AirlockRequestStatus.Submitted: + return constants.STORAGE_ACCOUNT_NAME_IMPORT_INPROGRESS.format(tre_id) + elif airlock_request.status == AirlockRequestStatus.InReview: + return constants.STORAGE_ACCOUNT_NAME_IMPORT_INPROGRESS.format(tre_id) + elif airlock_request.status == AirlockRequestStatus.Approved: + return constants.STORAGE_ACCOUNT_NAME_IMPORT_APPROVED.format(short_workspace_id) + elif airlock_request.status == AirlockRequestStatus.Rejected: + return constants.STORAGE_ACCOUNT_NAME_IMPORT_REJECTED.format(tre_id) + elif airlock_request.status == AirlockRequestStatus.Blocked: + return constants.STORAGE_ACCOUNT_NAME_IMPORT_BLOCKED.format(tre_id) + else: + if airlock_request.status == AirlockRequestStatus.Draft: + return constants.STORAGE_ACCOUNT_NAME_EXPORT_INTERNAL.format(short_workspace_id) + elif airlock_request.status == AirlockRequestStatus.Submitted: + return constants.STORAGE_ACCOUNT_NAME_EXPORT_INPROGRESS.format(short_workspace_id) + elif airlock_request.status == AirlockRequestStatus.InReview: + return constants.STORAGE_ACCOUNT_NAME_EXPORT_INPROGRESS.format(short_workspace_id) + elif airlock_request.status == AirlockRequestStatus.Approved: + return constants.STORAGE_ACCOUNT_NAME_EXPORT_APPROVED.format(tre_id) + elif airlock_request.status == AirlockRequestStatus.Rejected: + return constants.STORAGE_ACCOUNT_NAME_EXPORT_REJECTED.format(short_workspace_id) + elif airlock_request.status == AirlockRequestStatus.Blocked: + return constants.STORAGE_ACCOUNT_NAME_EXPORT_BLOCKED.format(short_workspace_id) + + +def get_airlock_request_container_sas_token(airlock_request: AirlockRequest, account_name: str, signer_client_id: str = ""): + # Workspace-global SAS tokens use the workspace signer for ABAC isolation. + global_account_name = constants.STORAGE_ACCOUNT_NAME_AIRLOCK_WORKSPACE_GLOBAL.format(config.TRE_ID) + if signer_client_id and account_name == global_account_name: + credential = credentials.get_airlock_signer_credential(signer_client_id, config.AAD_TENANT_ID) + else: + credential = credentials.get_credential() + blob_service_client = BlobServiceClient(account_url=get_account_url(account_name), - credential=credentials.get_credential()) + credential=credential) start = datetime.now(UTC) - timedelta(minutes=15) expiry = datetime.now(UTC) + timedelta(hours=config.AIRLOCK_SAS_TOKEN_EXPIRY_PERIOD_IN_HOURS) @@ -114,11 +124,12 @@ def get_airlock_request_container_sas_token(account_name: str, try: udk = blob_service_client.get_user_delegation_key(key_start_time=start, key_expiry_time=expiry) except Exception: - raise Exception(f"Failed getting user delegation key, has the API identity been granted 'Storage Blob Data Contributor' access to the storage account {account_name}?") + raise Exception(f"Failed getting user delegation key, has the signing identity been granted 'Storage Blob Data Contributor' access to the storage account {account_name}?") required_permission = get_required_permission(airlock_request) + container_name = get_container_name_for_request(airlock_request.id, airlock_request.status) if airlock_request.airlock_version >= 2 else airlock_request.id - token = generate_container_sas(container_name=airlock_request.id, + token = generate_container_sas(container_name=container_name, account_name=account_name, user_delegation_key=udk, permission=required_permission, @@ -126,13 +137,59 @@ def get_airlock_request_container_sas_token(account_name: str, expiry=expiry) return "https://{}.blob.{}/{}?{}" \ - .format(account_name, STORAGE_ENDPOINT, airlock_request.id, token) + .format(account_name, STORAGE_ENDPOINT, container_name, token) def get_account_url(account_name: str) -> str: return f"https://{account_name}.blob.{STORAGE_ENDPOINT}/" +def _delete_container(account_name: str, container_name: str, credential) -> bool: + blob_service_client = BlobServiceClient(account_url=get_account_url(account_name), credential=credential) + try: + blob_service_client.get_container_client(container_name).delete_container() + return True + except ResourceNotFoundError: + return False + + +async def delete_workspace_airlock_containers(workspace: Workspace, airlock_request_repo: AirlockRequestRepository) -> None: + """Delete shared containers without blocking workspace deletion on failures.""" + # Version changes cannot leave v1 data attached to a v2 workspace. + if workspace.properties.get("airlock_version", 1) < 2: + return + + request_ids = await airlock_request_repo.get_data_retaining_airlock_request_ids_for_workspace(workspace.id) + if not request_ids: + return + + core_account = constants.STORAGE_ACCOUNT_NAME_AIRLOCK_CORE.format(config.TRE_ID) + global_account = constants.STORAGE_ACCOUNT_NAME_AIRLOCK_WORKSPACE_GLOBAL.format(config.TRE_ID) + signer_client_id = workspace.properties.get("airlock_signer_client_id", "") + + deleted = 0 + failed = [] + for request_id in request_ids: + # A request may still be in draft, so both container names are candidates. + for container_name in (request_id, f"{request_id}{constants.DRAFT_CONTAINER_SUFFIX}"): + # Status does not identify which shared account holds the container. + for account_name in (core_account, global_account): + # Workspace-global access uses the workspace signer. + credential = (credentials.get_airlock_signer_credential(signer_client_id, config.AAD_TENANT_ID) + if signer_client_id and account_name == global_account + else credentials.get_credential()) + try: + if await asyncio.to_thread(_delete_container, account_name, container_name, credential): + deleted += 1 + except Exception: + logger.exception(f"Failed deleting airlock container {container_name} from {account_name}") + failed.append(request_id) + + logger.info(f"Deleted {deleted} airlock container(s) for workspace {workspace.id}") + if failed: + logger.error(f"Could not delete airlock containers for workspace {workspace.id}, request ids: {sorted(set(failed))}") + + async def review_airlock_request(airlock_review_input: AirlockReviewInCreate, airlock_request: AirlockRequest, user: User, workspace: Workspace, airlock_request_repo: AirlockRequestRepository, user_resource_repo: UserResourceRepository, workspace_service_repo, operation_repo: WorkspaceServiceRepository, resource_template_repo: ResourceTemplateRepository, @@ -168,8 +225,20 @@ async def review_airlock_request(airlock_review_input: AirlockReviewInCreate, ai def get_airlock_container_link(airlock_request: AirlockRequest, user, workspace): validate_user_allowed_to_access_storage_account(user, airlock_request) validate_request_status(airlock_request) - account_name: str = get_account_by_request(airlock_request, workspace) - return get_airlock_request_container_sas_token(account_name, airlock_request) + + if airlock_request.airlock_version >= 2: + from services.airlock_storage_helper import get_storage_account_name_for_request + tre_id = config.TRE_ID + account_name = get_storage_account_name_for_request( + request_type=airlock_request.type.value, + status=airlock_request.status, + tre_id=tre_id + ) + else: + account_name = get_account_by_request(airlock_request, workspace) + + signer_client_id = workspace.properties.get("airlock_signer_client_id", "") if airlock_request.airlock_version >= 2 else "" + return get_airlock_request_container_sas_token(airlock_request, account_name, signer_client_id) async def create_review_vm(airlock_request: AirlockRequest, user: User, workspace: Workspace, user_resource_repo: UserResourceRepository, workspace_service_repo: WorkspaceServiceRepository, @@ -293,7 +362,7 @@ async def save_and_publish_event_airlock_request(airlock_request: AirlockRequest try: logger.debug(f"Sending status changed event for airlock request item: {airlock_request.id}") - await send_status_changed_event(airlock_request=airlock_request, previous_status=None) + await send_status_changed_event(airlock_request=airlock_request, previous_status=None, workspace=workspace) await send_airlock_notification_event(airlock_request, workspace, role_assignment_details) except Exception as e: await airlock_request_repo.delete_item(airlock_request.id) @@ -311,6 +380,7 @@ async def update_and_publish_event_airlock_request( status_message: Optional[str] = None, airlock_review: Optional[AirlockReview] = None, review_user_resource: Optional[AirlockReviewUserResource] = None, + scan_result=_UNSET, ) -> AirlockRequest: try: logger.debug(f"Updating airlock request item: {airlock_request.id}") @@ -321,7 +391,8 @@ async def update_and_publish_event_airlock_request( request_files=request_files, status_message=status_message, airlock_review=airlock_review, - review_user_resource=review_user_resource) + review_user_resource=review_user_resource, + scan_result=scan_result) except Exception as e: logger.exception(f'Failed updating airlock_request item {airlock_request}') # If the validation failed, the error was not related to the saving itself @@ -343,7 +414,7 @@ async def update_and_publish_event_airlock_request( try: logger.debug(f"Sending status changed event for airlock request item: {airlock_request.id}") - await send_status_changed_event(airlock_request=updated_airlock_request, previous_status=airlock_request.status) + await send_status_changed_event(airlock_request=updated_airlock_request, previous_status=airlock_request.status, workspace=workspace) await send_airlock_notification_event(updated_airlock_request, workspace, role_assignment_details) return updated_airlock_request except Exception as e: diff --git a/api_app/services/airlock_storage_helper.py b/api_app/services/airlock_storage_helper.py new file mode 100644 index 0000000000..fed46ec87b --- /dev/null +++ b/api_app/services/airlock_storage_helper.py @@ -0,0 +1,24 @@ +from resources import constants +from models.domain.airlock_request import AirlockRequestStatus + + +def get_container_name_for_request(request_id: str, status: AirlockRequestStatus) -> str: + # Draft data lives in its own container so submission can delete it, revoking any SAS already issued. + if status == AirlockRequestStatus.Draft: + return f"{request_id}{constants.DRAFT_CONTAINER_SUFFIX}" + return request_id + + +def get_storage_account_name_for_request( + request_type: str, + status: AirlockRequestStatus, + tre_id: str +) -> str: + if request_type == constants.IMPORT_TYPE: + if status in [AirlockRequestStatus.Approved, AirlockRequestStatus.ApprovalInProgress]: + return constants.STORAGE_ACCOUNT_NAME_AIRLOCK_WORKSPACE_GLOBAL.format(tre_id) + return constants.STORAGE_ACCOUNT_NAME_AIRLOCK_CORE.format(tre_id) + + if status in [AirlockRequestStatus.Approved, AirlockRequestStatus.ApprovalInProgress]: + return constants.STORAGE_ACCOUNT_NAME_AIRLOCK_CORE.format(tre_id) + return constants.STORAGE_ACCOUNT_NAME_AIRLOCK_WORKSPACE_GLOBAL.format(tre_id) diff --git a/api_app/services/legacy_airlock_guard.py b/api_app/services/legacy_airlock_guard.py new file mode 100644 index 0000000000..4fad7b5964 --- /dev/null +++ b/api_app/services/legacy_airlock_guard.py @@ -0,0 +1,61 @@ +from core import config +from db.repositories.airlock_requests import AirlockRequestRepository +from models.domain.resource import Resource +from models.schemas.resource import ResourcePatch +from services.logging import logger + + +def _truncate_ids(resource_ids: list[str], limit: int = 25) -> list[str]: + if len(resource_ids) <= limit: + return resource_ids + return resource_ids[:limit] + + +def ensure_workspace_airlock_version_supported(properties: dict, default_version: int = 1) -> None: + """Reject unsupported airlock versions. A missing airlock_version means legacy (v1).""" + if not properties: + return + if not properties.get("enable_airlock", True): + return + + airlock_version = properties.get("airlock_version", default_version) + + if not config.ENABLE_LEGACY_AIRLOCK and airlock_version == 1: + raise ValueError( + "Cannot use airlock_version=1 because legacy airlock is disabled in core " + "(enable_legacy_airlock=false). Use airlock_version=2." + ) + + +async def ensure_airlock_version_change_allowed(workspace: Resource, resource_patch: ResourcePatch, request_repo: AirlockRequestRepository) -> None: + """Reject version changes while airlock requests are still in flight.""" + if not resource_patch.properties: + return + new_version = resource_patch.properties.get("airlock_version") + if new_version is None: + return + current_version = workspace.properties.get("airlock_version", 1) + if new_version == current_version: + return + + if new_version < current_version: + # Downgrading destroys the v2 signer and conditioned role assignments that guard existing + # shared containers, with no path to re-grant access to data created under v2. + logger.warning("Blocked airlock_version downgrade %s->%s for workspace %s", current_version, new_version, workspace.id) + raise ValueError( + f"Cannot change airlock_version from {current_version} to {new_version}: downgrading is not " + f"supported because it removes access to data created under the newer version." + ) + + request_ids = await request_repo.get_in_flight_airlock_request_ids_for_workspace(workspace.id) + if request_ids: + logger.warning( + "Blocked airlock_version change %s->%s for workspace %s due to %d in-flight airlock request(s)", + current_version, new_version, workspace.id, len(request_ids) + ) + raise ValueError( + f"Cannot change airlock_version from {current_version} to {new_version} while " + f"{len(request_ids)} airlock request(s) are still in progress in this workspace " + f"(their data would be left behind in the previous version's storage). " + f"Let them complete or cancel them first. Request ids: {_truncate_ids(request_ids)}" + ) diff --git a/api_app/tests_ma/test_api/dependencies/test_airlock.py b/api_app/tests_ma/test_api/dependencies/test_airlock.py new file mode 100644 index 0000000000..52ab0e6226 --- /dev/null +++ b/api_app/tests_ma/test_api/dependencies/test_airlock.py @@ -0,0 +1,41 @@ +from mock import AsyncMock +import pytest +from fastapi import HTTPException + +from api.dependencies.airlock import get_airlock_request_by_id_from_path +from models.domain.airlock_request import AirlockRequest, AirlockRequestType + +pytestmark = pytest.mark.asyncio + +WORKSPACE_ID = "abc000d3-82da-4bfc-b6e9-9a7853ef753e" +OTHER_WORKSPACE_ID = "d1f2a3b4-1111-2222-3333-444455556666" +AIRLOCK_REQUEST_ID = "af89dccd-cdf8-4e47-8cfe-995faeac0f09" + + +def _request(workspace_id): + return AirlockRequest( + id=AIRLOCK_REQUEST_ID, + workspaceId=workspace_id, + type=AirlockRequestType.Import, + businessJustification="test") + + +async def test_returns_request_belonging_to_the_workspace_in_the_path(): + repo = AsyncMock() + repo.get_airlock_request_by_id.return_value = _request(WORKSPACE_ID) + + result = await get_airlock_request_by_id_from_path( + airlock_request_id=AIRLOCK_REQUEST_ID, workspace_id=WORKSPACE_ID, airlock_request_repo=repo) + + assert result.id == AIRLOCK_REQUEST_ID + + +async def test_rejects_request_belonging_to_another_workspace(): + repo = AsyncMock() + repo.get_airlock_request_by_id.return_value = _request(OTHER_WORKSPACE_ID) + + with pytest.raises(HTTPException) as exc: + await get_airlock_request_by_id_from_path( + airlock_request_id=AIRLOCK_REQUEST_ID, workspace_id=WORKSPACE_ID, airlock_request_repo=repo) + + assert exc.value.status_code == 404 diff --git a/api_app/tests_ma/test_api/test_routes/test_airlock.py b/api_app/tests_ma/test_api/test_routes/test_airlock.py index 7d8f295768..2624935bf8 100644 --- a/api_app/tests_ma/test_api/test_routes/test_airlock.py +++ b/api_app/tests_ma/test_api/test_routes/test_airlock.py @@ -184,6 +184,12 @@ async def test_post_airlock_request_with_airlock_disabled_returns_405(self, _, a response = await client.post(app.url_path_for(strings.API_CREATE_AIRLOCK_REQUEST, workspace_id=WORKSPACE_ID), json=sample_airlock_request_input_data) assert response.status_code == status.HTTP_405_METHOD_NOT_ALLOWED + @patch("services.legacy_airlock_guard.config.ENABLE_LEGACY_AIRLOCK", False) + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id", return_value=sample_workspace(workspace_properties={"airlock_version": 1})) + async def test_post_airlock_request_on_v1_workspace_with_legacy_disabled_returns_400(self, _, app, client, sample_airlock_request_input_data): + response = await client.post(app.url_path_for(strings.API_CREATE_AIRLOCK_REQUEST, workspace_id=WORKSPACE_ID), json=sample_airlock_request_input_data) + assert response.status_code == status.HTTP_400_BAD_REQUEST + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id", return_value=sample_workspace(workspace_properties={})) @patch("api.routes.airlock.save_and_publish_event_airlock_request") async def test_post_airlock_request_with_enable_airlock_property_missing_returns_201(self, _, __, app, client, sample_airlock_request_input_data): @@ -217,6 +223,19 @@ async def test_post_submit_airlock_request_submits_airlock_request_returns_200(s assert response.json()["airlockRequest"]["id"] == AIRLOCK_REQUEST_ID assert response.json()["airlockRequest"]["status"] == AirlockRequestStatus.Submitted + @patch("api.routes.airlock.AirlockRequestRepository.read_item_by_id", return_value=sample_airlock_request_object()) + @patch("api.routes.airlock.update_and_publish_event_airlock_request") + async def test_post_submit_does_not_apply_a_recorded_scan_result(self, update_mock, _, app, client): + submitted = sample_airlock_request_object(status=AirlockRequestStatus.Submitted) + submitted.scanResult = {"clean": True, "message": None} + update_mock.return_value = submitted + + response = await client.post(app.url_path_for(strings.API_SUBMIT_AIRLOCK_REQUEST, workspace_id=WORKSPACE_ID, airlock_request_id=AIRLOCK_REQUEST_ID)) + + assert response.status_code == status.HTTP_200_OK + assert response.json()["airlockRequest"]["status"] == AirlockRequestStatus.Submitted + assert update_mock.call_count == 1 + @patch("api.routes.airlock.AirlockRequestRepository.read_item_by_id", side_effect=EntityDoesNotExist) async def test_post_submit_airlock_request_if_request_not_found_returns_404(self, _, app, client): response = await client.post(app.url_path_for(strings.API_SUBMIT_AIRLOCK_REQUEST, workspace_id=WORKSPACE_ID, airlock_request_id=AIRLOCK_REQUEST_ID)) diff --git a/api_app/tests_ma/test_api/test_routes/test_migrations.py b/api_app/tests_ma/test_api/test_routes/test_migrations.py index dff2c94d9f..718669a52d 100644 --- a/api_app/tests_ma/test_api/test_routes/test_migrations.py +++ b/api_app/tests_ma/test_api/test_routes/test_migrations.py @@ -36,10 +36,24 @@ def _prepare(self, app, admin_user): app.dependency_overrides = {} # [POST] /migrations/ + @patch("api.routes.migrations.WorkspaceRepository.create") @patch("api.routes.migrations.logger.info") - async def test_post_migrations_returns_202_on_successful(self, logging, client, app): + async def test_post_migrations_returns_202_on_successful(self, logging, workspace_repo, client, app): + workspace_repo.return_value.set_default_airlock_version_for_legacy_workspaces.return_value = [] response = await client.post(app.url_path_for(strings.API_MIGRATE_DATABASE)) logging.assert_called() if response.status_code != status.HTTP_202_ACCEPTED: raise AssertionError(f"Expected status code {status.HTTP_202_ACCEPTED}, but got {response.status_code}") + + # [POST] /migrations/ + @patch("api.routes.migrations.WorkspaceRepository.create") + @patch("api.routes.migrations.logger.info") + async def test_post_migrations_stamps_legacy_workspaces_with_v1(self, logging, workspace_repo, client, app): + # Without this the bundle's v2 default would migrate a pre-v2 workspace on its next deploy. + workspace_repo.return_value.set_default_airlock_version_for_legacy_workspaces.return_value = ["ws-1", "ws-2"] + response = await client.post(app.url_path_for(strings.API_MIGRATE_DATABASE)) + + workspace_repo.return_value.set_default_airlock_version_for_legacy_workspaces.assert_awaited_once() + assert response.status_code == status.HTTP_202_ACCEPTED + assert "2 legacy workspace(s)" in response.json()["migrations"][0]["status"] diff --git a/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py b/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py index fc17efebc9..65914b00d2 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py @@ -469,3 +469,40 @@ async def test_get_airlock_requests_for_airlock_manager_argument_compatibility( assert isinstance(result, list), f"Test case {i} should return a list" except TypeError as e: pytest.fail(f"Test case {i} failed with TypeError: {str(e)}. Parameters: {test_kwargs}") + + +@patch("db.repositories.airlock_requests.AirlockRequestRepository.get_airlock_request_by_id", return_value=airlock_request_mock(status=DRAFT)) +@patch("db.repositories.airlock_requests.AirlockRequestRepository.update_airlock_request_item") +async def test_update_airlock_request_retry_preserves_all_update_fields(update_item_mock, _, airlock_request_repo): + update_item_mock.side_effect = [CosmosAccessConditionFailedError, None] + verdict = {"new_status": "in_review", "status_message": None} + + await airlock_request_repo.update_airlock_request( + original_request=airlock_request_mock(status=DRAFT), + updated_by=create_test_user(), + scan_result=verdict) + + retried_request = update_item_mock.call_args_list[1].args[1] + assert retried_request.scanResult == verdict + + +@pytest.mark.asyncio +@patch("db.repositories.airlock_requests.AirlockRequestRepository.query", return_value=[]) +async def test_get_in_flight_requests_excludes_every_final_status(query_mock, airlock_request_repo): + """Final states never move between stages, so they must not block a change of storage layout.""" + await airlock_request_repo.get_in_flight_airlock_request_ids_for_workspace(WORKSPACE_ID) + + final_statuses = query_mock.call_args.kwargs["parameters"][1]["value"] + assert set(final_statuses) == {status.value for status in AirlockRequestRepository.FINAL_AIRLOCK_STATUSES} + assert AirlockRequestStatus.Rejected.value in final_statuses + assert AirlockRequestStatus.Approved.value in final_statuses + + +@pytest.mark.asyncio +@patch("db.repositories.airlock_requests.AirlockRequestRepository.query", return_value=[]) +async def test_data_retaining_query_includes_cancelled_requests(query_mock, airlock_request_repo): + """Workspace-deletion cleanup must cover cancelled requests whose async container deletion may have failed.""" + await airlock_request_repo.get_data_retaining_airlock_request_ids_for_workspace(WORKSPACE_ID) + + query = query_mock.call_args.kwargs["query"] + assert "status" not in query diff --git a/api_app/tests_ma/test_db/test_repositories/test_workpaces_repository.py b/api_app/tests_ma/test_db/test_repositories/test_workpaces_repository.py index f7d637599c..73781f66f6 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_workpaces_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_workpaces_repository.py @@ -11,6 +11,7 @@ from db.repositories.workspaces import WorkspaceRepository from models.domain.operation import Status from models.domain.resource import ResourceType +from models.domain.resource_template import Property from models.domain.workspace import Workspace from models.schemas.workspace import WorkspaceInCreate @@ -148,6 +149,88 @@ async def test_create_workspace_item_creates_a_workspace_with_the_right_values(m assert workspace.properties["workspace_owner_object_id"] == "test_object_id" +@pytest.mark.asyncio +@patch('db.repositories.workspaces.generate_new_cidr') +@patch('db.repositories.workspaces.WorkspaceRepository.validate_input_against_template') +@patch('db.repositories.workspaces.WorkspaceRepository.is_workspace_storage_account_available') +@patch('core.config.RESOURCE_LOCATION', "useast2") +@patch('core.config.TRE_ID', "9876") +async def test_create_workspace_item_persists_default_airlock_version_when_omitted(mock_is_workspace_storage_account_available, validate_input_mock, new_cidr_mock, workspace_repo, basic_workspace_request, basic_resource_template): + workspace_to_create = basic_workspace_request + workspace_to_create.properties.pop("airlock_version", None) + workspace_to_create.properties["auth_type"] = "Automatic" + mock_is_workspace_storage_account_available.return_value = AsyncMock().return_value + mock_is_workspace_storage_account_available.return_value.return_value = False + basic_resource_template.properties["airlock_version"] = Property(default=2, enum=[1, 2]) + validate_input_mock.return_value = basic_resource_template + new_cidr_mock.return_value = "1.2.3.4/24" + + workspace, _ = await workspace_repo.create_workspace_item(workspace_to_create, {}, "test_object_id", ["test_role"]) + + assert workspace.properties["airlock_version"] == 2 + + +@pytest.mark.asyncio +@patch('db.repositories.workspaces.generate_new_cidr') +@patch('db.repositories.workspaces.WorkspaceRepository.validate_input_against_template') +@patch('db.repositories.workspaces.WorkspaceRepository.is_workspace_storage_account_available') +@patch('core.config.RESOURCE_LOCATION', "useast2") +@patch('core.config.TRE_ID', "9876") +async def test_create_workspace_item_defaults_manual_auth_to_airlock_v2(mock_is_workspace_storage_account_available, validate_input_mock, new_cidr_mock, workspace_repo, basic_workspace_request, basic_resource_template): + workspace_to_create = basic_workspace_request + workspace_to_create.properties.pop("airlock_version", None) + workspace_to_create.properties["auth_type"] = "Manual" + mock_is_workspace_storage_account_available.return_value = AsyncMock().return_value + mock_is_workspace_storage_account_available.return_value.return_value = False + basic_resource_template.properties["airlock_version"] = Property(default=2, enum=[1, 2]) + validate_input_mock.return_value = basic_resource_template + new_cidr_mock.return_value = "1.2.3.4/24" + + workspace, _ = await workspace_repo.create_workspace_item(workspace_to_create, {}, "test_object_id", ["test_role"]) + + assert workspace.properties["airlock_version"] == 2 + + +@pytest.mark.asyncio +@patch('db.repositories.workspaces.generate_new_cidr') +@patch('db.repositories.workspaces.WorkspaceRepository.validate_input_against_template') +@patch('db.repositories.workspaces.WorkspaceRepository.is_workspace_storage_account_available') +@patch('core.config.RESOURCE_LOCATION', "useast2") +@patch('core.config.TRE_ID', "9876") +async def test_create_workspace_item_preserves_explicit_airlock_version(mock_is_workspace_storage_account_available, validate_input_mock, new_cidr_mock, workspace_repo, basic_workspace_request, basic_resource_template): + workspace_to_create = basic_workspace_request + workspace_to_create.properties["airlock_version"] = 1 + mock_is_workspace_storage_account_available.return_value = AsyncMock().return_value + mock_is_workspace_storage_account_available.return_value.return_value = False + validate_input_mock.return_value = basic_resource_template + new_cidr_mock.return_value = "1.2.3.4/24" + + workspace, _ = await workspace_repo.create_workspace_item(workspace_to_create, {}, "test_object_id", ["test_role"]) + + assert workspace.properties["airlock_version"] == 1 + + +@pytest.mark.asyncio +@patch('db.repositories.workspaces.generate_new_cidr') +@patch('db.repositories.workspaces.WorkspaceRepository.validate_input_against_template') +@patch('db.repositories.workspaces.WorkspaceRepository.is_workspace_storage_account_available') +@patch('core.config.RESOURCE_LOCATION', "useast2") +@patch('core.config.TRE_ID', "9876") +async def test_create_workspace_item_defaults_legacy_template_to_airlock_v1(mock_is_workspace_storage_account_available, validate_input_mock, new_cidr_mock, workspace_repo, basic_workspace_request, basic_resource_template): + # A template that does not declare airlock_version is legacy (v1) and must not be stamped v2. + workspace_to_create = basic_workspace_request + workspace_to_create.properties.pop("airlock_version", None) + mock_is_workspace_storage_account_available.return_value = AsyncMock().return_value + mock_is_workspace_storage_account_available.return_value.return_value = False + basic_resource_template.properties.pop("airlock_version", None) + validate_input_mock.return_value = basic_resource_template + new_cidr_mock.return_value = "1.2.3.4/24" + + workspace, _ = await workspace_repo.create_workspace_item(workspace_to_create, {}, "test_object_id", ["test_role"]) + + assert workspace.properties["airlock_version"] == 1 + + @pytest.mark.asyncio @patch('core.config.RESOURCE_LOCATION', "useast2") @patch('core.config.TRE_ID', "9876") diff --git a/api_app/tests_ma/test_service_bus/test_airlock_request_status_update.py b/api_app/tests_ma/test_service_bus/test_airlock_request_status_update.py index 6404ba122f..85993934e4 100644 --- a/api_app/tests_ma/test_service_bus/test_airlock_request_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_airlock_request_status_update.py @@ -5,6 +5,7 @@ from mock import AsyncMock, patch from service_bus.airlock_request_status_update import AirlockStatusUpdater +from db.repositories.airlock_requests import _UNSET from models.domain.events import AirlockNotificationUserData, AirlockFile from models.domain.airlock_request import AirlockRequest, AirlockRequestStatus, AirlockRequestType from models.domain.workspace import Workspace @@ -130,7 +131,8 @@ async def test_receiving_good_message(_, logging_mock, workspace_repo, airlock_r request_files=None, status_message=None, airlock_review=None, - review_user_resource=None) + review_user_resource=None, + scan_result=_UNSET) assert eg_client().send.call_count == 2 logging_mock.assert_not_called() @@ -188,7 +190,8 @@ async def test_when_updating_and_state_store_exception_error_is_logged(logging_m async def test_when_updating_and_current_status_differs_from_status_in_state_store_error_is_logged(logging_mock, airlock_request_repo, _): service_bus_received_message_mock = ServiceBusReceivedMessageMock(test_sb_step_result_message) - expected_airlock_request = sample_airlock_request(AirlockRequestStatus.Draft) + # Not the completed step and not the message's new status, so this is a genuine mismatch rather than a duplicate. + expected_airlock_request = sample_airlock_request(AirlockRequestStatus.ApprovalInProgress) airlock_request_repo.return_value.get_airlock_request_by_id.return_value = expected_airlock_request airlockStatusUpdater = AirlockStatusUpdater() await airlockStatusUpdater.init_repos() @@ -199,6 +202,128 @@ async def test_when_updating_and_current_status_differs_from_status_in_state_sto logging_mock.assert_called_once_with(expected_error_message) +test_sb_scan_result_message = { + "id": EVENT_ID, + "subject": "main", + "data": { + "completed_step": "submitted", + "request_id": AIRLOCK_REQUEST_ID, + "scan_result": {"clean": True, "message": None} + }, + "eventType": "bla", + "eventTime": "test message", + "topic": "" +} + + +@patch('service_bus.airlock_request_status_update.update_and_publish_event_airlock_request') +@patch('service_bus.airlock_request_status_update.WorkspaceRepository.create') +@patch('service_bus.airlock_request_status_update.AirlockRequestRepository.create') +@patch('services.logging.logger.error') +async def test_scan_result_for_draft_request_is_recorded_without_status_change(logging_mock, airlock_request_repo, _, update_and_publish_mock): + service_bus_received_message_mock = ServiceBusReceivedMessageMock(test_sb_scan_result_message) + + expected_airlock_request = sample_airlock_request(AirlockRequestStatus.Draft) + airlock_request_repo.return_value.get_airlock_request_by_id.return_value = expected_airlock_request + airlock_request_repo.return_value.update_airlock_request.return_value = expected_airlock_request + airlockStatusUpdater = AirlockStatusUpdater() + await airlockStatusUpdater.init_repos() + complete_message = await airlockStatusUpdater.process_message(service_bus_received_message_mock) + + assert complete_message is True + airlock_request_repo.return_value.update_airlock_request.assert_called_once_with( + original_request=expected_airlock_request, + updated_by=expected_airlock_request.updatedBy, + scan_result={"clean": True, "message": None}) + update_and_publish_mock.assert_not_called() + logging_mock.assert_not_called() + + +@patch('service_bus.airlock_request_status_update.update_and_publish_event_airlock_request') +@patch('service_bus.airlock_request_status_update.WorkspaceRepository.create') +@patch('service_bus.airlock_request_status_update.AirlockRequestRepository.create') +async def test_verdict_arriving_after_file_validation_completes_submission(airlock_request_repo, workspace_repo, update_and_publish_mock): + service_bus_received_message_mock = ServiceBusReceivedMessageMock(test_sb_scan_result_message) + + submitted = sample_airlock_request(AirlockRequestStatus.Submitted) + scanned = sample_airlock_request(AirlockRequestStatus.Submitted) + scanned.scanResult = {"clean": True, "message": None} + airlock_request_repo.return_value.get_airlock_request_by_id.return_value = submitted + airlock_request_repo.return_value.update_airlock_request.return_value = scanned + workspace_repo.return_value.get_workspace_by_id.return_value = sample_workspace() + + airlockStatusUpdater = AirlockStatusUpdater() + await airlockStatusUpdater.init_repos() + complete_message = await airlockStatusUpdater.process_message(service_bus_received_message_mock) + + assert complete_message is True + assert update_and_publish_mock.call_args.kwargs["new_status"] == AirlockRequestStatus.InReview + + +@patch('service_bus.airlock_request_status_update.update_and_publish_event_airlock_request') +@patch('service_bus.airlock_request_status_update.WorkspaceRepository.create') +@patch('service_bus.airlock_request_status_update.AirlockRequestRepository.create') +async def test_file_validation_arriving_after_verdict_completes_submission(airlock_request_repo, workspace_repo, update_and_publish_mock): + service_bus_received_message_mock = ServiceBusReceivedMessageMock(test_sb_step_result_message_with_files) + + scanned = sample_airlock_request(AirlockRequestStatus.Submitted) + scanned.scanResult = {"clean": True, "message": None} + airlock_request_repo.return_value.get_airlock_request_by_id.return_value = scanned + workspace_repo.return_value.get_workspace_by_id.return_value = sample_workspace() + update_and_publish_mock.return_value = scanned + + airlockStatusUpdater = AirlockStatusUpdater() + await airlockStatusUpdater.init_repos() + complete_message = await airlockStatusUpdater.process_message(service_bus_received_message_mock) + + assert complete_message is True + assert update_and_publish_mock.call_args.kwargs["new_status"] == AirlockRequestStatus.InReview + + +@patch('service_bus.airlock_request_status_update.update_and_publish_event_airlock_request') +@patch('service_bus.airlock_request_status_update.WorkspaceRepository.create') +@patch('service_bus.airlock_request_status_update.AirlockRequestRepository.create') +async def test_malicious_verdict_blocks_a_validated_submission(airlock_request_repo, workspace_repo, update_and_publish_mock): + message = json.loads(json.dumps(test_sb_scan_result_message)) + message["data"]["scan_result"] = {"clean": False, "message": "Malicious"} + service_bus_received_message_mock = ServiceBusReceivedMessageMock(message) + + submitted = sample_airlock_request(AirlockRequestStatus.Submitted) + scanned = sample_airlock_request(AirlockRequestStatus.Submitted) + scanned.scanResult = {"clean": False, "message": "Malicious"} + airlock_request_repo.return_value.get_airlock_request_by_id.return_value = submitted + airlock_request_repo.return_value.update_airlock_request.return_value = scanned + workspace_repo.return_value.get_workspace_by_id.return_value = sample_workspace() + + airlockStatusUpdater = AirlockStatusUpdater() + await airlockStatusUpdater.init_repos() + complete_message = await airlockStatusUpdater.process_message(service_bus_received_message_mock) + + assert complete_message is True + assert update_and_publish_mock.call_args.kwargs["new_status"] == AirlockRequestStatus.BlockingInProgress + assert update_and_publish_mock.call_args.kwargs["status_message"] == "Malicious" + + +@pytest.mark.parametrize("message", [test_sb_step_result_message, test_sb_scan_result_message]) +@pytest.mark.parametrize("final_status", [AirlockRequestStatus.Cancelled, AirlockRequestStatus.Failed, AirlockRequestStatus.Blocked, AirlockRequestStatus.Approved, AirlockRequestStatus.Rejected]) +@patch('service_bus.airlock_request_status_update.WorkspaceRepository.create') +@patch('service_bus.airlock_request_status_update.AirlockRequestRepository.create') +@patch('services.logging.logger.error') +async def test_scan_result_for_final_request_is_discarded_not_dead_lettered(logging_mock, airlock_request_repo, _, final_status, message): + # A late verdict must not mutate a request that has already reached a final state. + service_bus_received_message_mock = ServiceBusReceivedMessageMock(message) + + expected_airlock_request = sample_airlock_request(final_status) + airlock_request_repo.return_value.get_airlock_request_by_id.return_value = expected_airlock_request + airlockStatusUpdater = AirlockStatusUpdater() + await airlockStatusUpdater.init_repos() + complete_message = await airlockStatusUpdater.process_message(service_bus_received_message_mock) + + assert complete_message is True + airlock_request_repo.return_value.update_airlock_request.assert_not_called() + logging_mock.assert_not_called() + + @patch('service_bus.airlock_request_status_update.WorkspaceRepository.create') @patch('service_bus.airlock_request_status_update.AirlockRequestRepository.create') @patch('services.logging.logger.exception') @@ -214,3 +339,98 @@ async def test_when_updating_and_status_update_is_illegal_error_is_logged(sb_cli assert complete_message is True expected_error_message = strings.STEP_RESULT_MESSAGE_INVALID_STATUS.format(test_sb_step_result_message_with_invalid_status["data"]["request_id"], test_sb_step_result_message_with_invalid_status["data"]["completed_step"], test_sb_step_result_message_with_invalid_status["data"]["new_status"]) logging_mock.assert_called_once_with(expected_error_message) + + +test_sb_step_result_message_with_files = { + "id": EVENT_ID, + "subject": "main", + "data": { + "completed_step": "submitted", + "request_id": AIRLOCK_REQUEST_ID, + "request_files": [{"name": "test.txt", "size": 5}] + }, + "eventType": "bla", + "eventTime": "test message", + "topic": "" +} + + +@patch('service_bus.airlock_request_status_update.WorkspaceRepository.create') +@patch('service_bus.airlock_request_status_update.AirlockRequestRepository.create') +@patch('services.logging.logger.error') +async def test_duplicate_step_result_already_applied_is_acknowledged(logging_mock, airlock_request_repo, _): + # Redelivery of an already-applied result must not be retried until it dead-letters. + service_bus_received_message_mock = ServiceBusReceivedMessageMock(test_sb_step_result_message) + + already_advanced = sample_airlock_request(AirlockRequestStatus.InReview) + airlock_request_repo.return_value.get_airlock_request_by_id.return_value = already_advanced + airlockStatusUpdater = AirlockStatusUpdater() + await airlockStatusUpdater.init_repos() + complete_message = await airlockStatusUpdater.process_message(service_bus_received_message_mock) + + assert complete_message is True + airlock_request_repo.return_value.update_airlock_request.assert_not_called() + logging_mock.assert_not_called() + + +@pytest.mark.parametrize("malformed", [{"clean": "false"}, {"clean": "true"}, {"clean": None}, {}]) +@patch('service_bus.airlock_request_status_update.update_and_publish_event_airlock_request') +@patch('service_bus.airlock_request_status_update.WorkspaceRepository.create') +@patch('service_bus.airlock_request_status_update.AirlockRequestRepository.create') +async def test_malformed_scan_verdict_does_not_advance_request(airlock_request_repo, _, update_and_publish_mock, malformed): + # A non-boolean verdict must never be read as clean. + request = sample_airlock_request(AirlockRequestStatus.Submitted) + request.files = [AirlockFile(name="test.txt", size=100)] + request.scanResult = malformed + + airlockStatusUpdater = AirlockStatusUpdater() + await airlockStatusUpdater.init_repos() + await airlockStatusUpdater._complete_submission_if_ready(request) + + update_and_publish_mock.assert_not_called() + + +@patch('service_bus.airlock_request_status_update.WorkspaceRepository.create') +@patch('service_bus.airlock_request_status_update.AirlockRequestRepository.create') +async def test_late_file_result_is_persisted_not_discarded(airlock_request_repo, _): + # With scanning disabled the destination BlobCreated event can advance the request first, + # so the file enumeration arrives afterwards and must not be thrown away. + message = json.loads(json.dumps(test_sb_step_result_message)) + message["data"]["request_files"] = [{"name": "test.txt", "size": 100}] + service_bus_received_message_mock = ServiceBusReceivedMessageMock(message) + + already_advanced = sample_airlock_request(AirlockRequestStatus.InReview) + already_advanced.files = [] + airlock_request_repo.return_value.get_airlock_request_by_id.return_value = already_advanced + airlock_request_repo.return_value.update_airlock_request.return_value = already_advanced + + airlockStatusUpdater = AirlockStatusUpdater() + await airlockStatusUpdater.init_repos() + complete_message = await airlockStatusUpdater.process_message(service_bus_received_message_mock) + + assert complete_message is True + persisted = airlock_request_repo.return_value.update_airlock_request.call_args.kwargs["request_files"] + assert persisted[0].name == "test.txt" + + +@patch('service_bus.airlock_request_status_update.update_and_publish_event_airlock_request') +@patch('service_bus.airlock_request_status_update.WorkspaceRepository.create') +@patch('service_bus.airlock_request_status_update.AirlockRequestRepository.create') +async def test_file_only_result_does_not_republish_a_status_event(airlock_request_repo, _, update_and_publish_mock): + # A file-only result (new_status None) must be acknowledged as a fact, never re-published as a transition. + message = json.loads(json.dumps(test_sb_step_result_message)) + message["data"]["new_status"] = None + message["data"]["request_files"] = [{"name": "test.txt", "size": 100}] + service_bus_received_message_mock = ServiceBusReceivedMessageMock(message) + + request = sample_airlock_request(AirlockRequestStatus.Submitted) + request.files = [] + airlock_request_repo.return_value.get_airlock_request_by_id.return_value = request + airlock_request_repo.return_value.update_airlock_request.return_value = request + + airlockStatusUpdater = AirlockStatusUpdater() + await airlockStatusUpdater.init_repos() + complete_message = await airlockStatusUpdater.process_message(service_bus_received_message_mock) + + assert complete_message is True + update_and_publish_mock.assert_not_called() diff --git a/api_app/tests_ma/test_services/test_airlock.py b/api_app/tests_ma/test_services/test_airlock.py index 65c61e6d62..541dd49c9c 100644 --- a/api_app/tests_ma/test_services/test_airlock.py +++ b/api_app/tests_ma/test_services/test_airlock.py @@ -24,6 +24,7 @@ AIRLOCK_REVIEW_ID = "96d909c5-e913-4c05-ae53-668a702ba2e5" USER_RESOURCE_ID = "cce59042-1dee-42dc-9388-6db846feeb3b" WORKSPACE_SERVICE_ID = "30f2fefa-e7bb-4e5b-93aa-e50bb037502a" +REVIEW_WORKSPACE_ID = "def111e4-93eb-4afc-c7fa-0b8964fg864f" CURRENT_TIME = time.time() ALL_ROLES = AzureADAuthorization.WORKSPACE_ROLES_DICT.keys() @@ -48,11 +49,32 @@ def sample_workspace(): resourcePath="test") -def sample_airlock_request(status=AirlockRequestStatus.Draft): +def sample_workspace_with_review_config(): + return Workspace( + id=WORKSPACE_ID, + templateName='template name', + templateVersion='1.0', + etag='', + properties={ + "client_id": "12345", + "display_name": "my research workspace", + "description": "for science!", + "airlock_review_config": { + "import": { + "import_vm_workspace_id": REVIEW_WORKSPACE_ID, + "import_vm_workspace_service_id": WORKSPACE_SERVICE_ID, + "import_vm_user_resource_template_name": "test-template" + } + }}, + resourcePath="test") + + +def sample_airlock_request(status=AirlockRequestStatus.Draft, airlock_version=2): airlock_request = AirlockRequest( id=AIRLOCK_REQUEST_ID, workspaceId=WORKSPACE_ID, type=AirlockRequestType.Import, + airlock_version=airlock_version, reviewUserResources={"user-guid-here": sample_airlock_user_resource_object()}, files=[AirlockFile( name="data.txt", @@ -82,10 +104,10 @@ def sample_airlock_user_resource_object(): ) -def sample_status_changed_event(new_status="draft", previous_status=None): +def sample_status_changed_event(new_status="draft", previous_status=None, review_workspace_id=None, airlock_version=2): status_changed_event = EventGridEvent( event_type="statusChanged", - data=StatusChangedData(request_id=AIRLOCK_REQUEST_ID, new_status=new_status, previous_status=previous_status, type=AirlockRequestType.Import, workspace_id=WORKSPACE_ID[-4:]).model_dump(mode="json"), + data=StatusChangedData(request_id=AIRLOCK_REQUEST_ID, new_status=new_status, previous_status=previous_status, type=AirlockRequestType.Import, workspace_id=WORKSPACE_ID, review_workspace_id=review_workspace_id, airlock_version=airlock_version).model_dump(mode="json"), subject=f"{AIRLOCK_REQUEST_ID}/statusChanged", data_version="2.0" ) @@ -401,6 +423,30 @@ async def test_update_and_publish_event_airlock_request_updates_item(_, event_gr assert actual_airlock_notification_event.data == airlock_notification_event_mock.data +@pytest.mark.asyncio +@patch("event_grid.helpers.EventGridPublisherClient", return_value=AsyncMock()) +@patch("services.aad_authentication.AzureADAuthorization.get_workspace_user_emails_by_role_assignment", return_value={"WorkspaceResearcher": ["researcher@outlook.com"], "WorkspaceOwner": ["owner@outlook.com"], "AirlockManager": ["manager@outlook.com"]}) +async def test_update_and_publish_event_includes_review_workspace_id_for_import(_, event_grid_publisher_client_mock, + airlock_request_repo_mock): + airlock_request_mock = sample_airlock_request() + updated_airlock_request_mock = sample_airlock_request(status=AirlockRequestStatus.Submitted) + status_changed_event_mock = sample_status_changed_event(new_status="submitted", previous_status="draft", review_workspace_id=REVIEW_WORKSPACE_ID) + airlock_request_repo_mock.update_airlock_request = AsyncMock(return_value=updated_airlock_request_mock) + event_grid_sender_client_mock = event_grid_publisher_client_mock.return_value + event_grid_sender_client_mock.send = AsyncMock() + + await update_and_publish_event_airlock_request( + airlock_request=airlock_request_mock, + airlock_request_repo=airlock_request_repo_mock, + updated_by=create_test_user(), + new_status=AirlockRequestStatus.Submitted, + workspace=sample_workspace_with_review_config()) + + actual_status_changed_event = event_grid_sender_client_mock.send.await_args_list[0].args[0][0] + assert actual_status_changed_event.data == status_changed_event_mock.data + assert actual_status_changed_event.data["review_workspace_id"] == REVIEW_WORKSPACE_ID + + @pytest.mark.asyncio @patch("services.airlock.send_status_changed_event") @patch("services.airlock.send_airlock_notification_event") @@ -588,6 +634,25 @@ async def test_delete_review_user_resource_disables_the_resource_before_deletion disable_user_resource.assert_called_once() +@patch("services.airlock.validate_request_status") +@patch("services.airlock.validate_user_allowed_to_access_storage_account") +@patch("services.airlock.get_airlock_request_container_sas_token", return_value="https://stalairlockgtest.blob.core.windows.net/container?sas") +def test_get_airlock_container_link_v2_resolves_correct_account_for_approved_import(mock_sas, mock_validate_user, mock_validate_status): + from services.airlock import get_airlock_container_link + + request = sample_airlock_request(status=AirlockRequestStatus.Approved) + request.type = AirlockRequestType.Import + request.airlock_version = 2 + + workspace = sample_workspace() + result = get_airlock_container_link(request, None, workspace) + + assert result == "https://stalairlockgtest.blob.core.windows.net/container?sas" + mock_sas.assert_called_once() + account_name = mock_sas.call_args[0][1] + assert account_name.startswith("stalairlockg") + + # --- Graph / role-assignment error separation tests --- @pytest.mark.asyncio @@ -732,3 +797,109 @@ def test_authenticated_user_with_empty_email_persists_and_builds_notification(): assert notification.created_by.email == "" assert notification.updated_by.email == "" + + +@patch("services.airlock.generate_container_sas", return_value="sas") +@patch("services.airlock.BlobServiceClient") +@patch("services.airlock.credentials") +def test_sas_token_uses_signer_credential_for_global_account(mock_credentials, mock_bsc, _mock_gen_sas): + from services.airlock import get_airlock_request_container_sas_token + from resources import constants + from core import config + global_account = constants.STORAGE_ACCOUNT_NAME_AIRLOCK_WORKSPACE_GLOBAL.format(config.TRE_ID) + request = sample_airlock_request(status=AirlockRequestStatus.Approved) + + get_airlock_request_container_sas_token(request, global_account, signer_client_id="signer-client-id") + + mock_credentials.get_airlock_signer_credential.assert_called_once() + mock_credentials.get_credential.assert_not_called() + + +@patch("services.airlock.generate_container_sas", return_value="sas") +@patch("services.airlock.BlobServiceClient") +@patch("services.airlock.credentials") +def test_sas_token_uses_default_credential_when_no_signer(mock_credentials, mock_bsc, _mock_gen_sas): + from services.airlock import get_airlock_request_container_sas_token + from resources import constants + from core import config + core_account = constants.STORAGE_ACCOUNT_NAME_AIRLOCK_CORE.format(config.TRE_ID) + request = sample_airlock_request(status=AirlockRequestStatus.Approved) + + get_airlock_request_container_sas_token(request, core_account, signer_client_id="") + + mock_credentials.get_credential.assert_called_once() + mock_credentials.get_airlock_signer_credential.assert_not_called() + + +@pytest.mark.asyncio +@patch("services.airlock.credentials") +@patch("services.airlock.BlobServiceClient") +async def test_delete_workspace_airlock_containers_deletes_from_both_accounts(mock_bsc, _mock_credentials): + from services.airlock import delete_workspace_airlock_containers + from resources import constants + from core import config + + workspace = MagicMock() + workspace.id = WORKSPACE_ID + workspace.properties = {"airlock_signer_client_id": "signer-client-id", "airlock_version": 2} + repo = AsyncMock() + repo.get_data_retaining_airlock_request_ids_for_workspace.return_value = ["req-1"] + + await delete_workspace_airlock_containers(workspace, repo) + + urls = [c.kwargs["account_url"] for c in mock_bsc.call_args_list] + assert any(constants.STORAGE_ACCOUNT_NAME_AIRLOCK_CORE.format(config.TRE_ID) in u for u in urls) + assert any(constants.STORAGE_ACCOUNT_NAME_AIRLOCK_WORKSPACE_GLOBAL.format(config.TRE_ID) in u for u in urls) + # Both accounts, and both the sealed and draft container names. + containers = [c.args[0] for c in mock_bsc.return_value.get_container_client.call_args_list] + assert "req-1" in containers and "req-1-draft" in containers + assert mock_bsc.return_value.get_container_client.return_value.delete_container.call_count == 4 + + +@pytest.mark.asyncio +@patch("services.airlock.credentials") +@patch("services.airlock.BlobServiceClient") +async def test_delete_workspace_airlock_containers_noop_without_requests(mock_bsc, _mock_credentials): + from services.airlock import delete_workspace_airlock_containers + + workspace = MagicMock() + workspace.id = WORKSPACE_ID + workspace.properties = {"airlock_version": 2} + repo = AsyncMock() + repo.get_data_retaining_airlock_request_ids_for_workspace.return_value = [] + + await delete_workspace_airlock_containers(workspace, repo) + + mock_bsc.assert_not_called() + + +@pytest.mark.asyncio +@patch("services.airlock.credentials") +@patch("services.airlock.BlobServiceClient") +async def test_delete_workspace_airlock_containers_skips_legacy_workspace(mock_bsc, _mock_credentials): + from services.airlock import delete_workspace_airlock_containers + + workspace = MagicMock() + workspace.id = WORKSPACE_ID + workspace.properties = {"airlock_version": 1} + repo = AsyncMock() + + await delete_workspace_airlock_containers(workspace, repo) + + repo.get_data_retaining_airlock_request_ids_for_workspace.assert_not_called() + mock_bsc.assert_not_called() + + +@pytest.mark.asyncio +@patch("services.airlock.credentials") +@patch("services.airlock.BlobServiceClient") +async def test_delete_workspace_airlock_containers_does_not_raise_on_failure(mock_bsc, _mock_credentials): + from services.airlock import delete_workspace_airlock_containers + + workspace = MagicMock() + workspace.id = WORKSPACE_ID + workspace.properties = {"airlock_version": 2} + repo = AsyncMock() + repo.get_data_retaining_airlock_request_ids_for_workspace.return_value = ["req-1"] + mock_bsc.return_value.get_container_client.return_value.delete_container.side_effect = Exception("boom") + await delete_workspace_airlock_containers(workspace, repo) diff --git a/api_app/tests_ma/test_services/test_airlock_storage_helper.py b/api_app/tests_ma/test_services/test_airlock_storage_helper.py new file mode 100644 index 0000000000..07719871b1 --- /dev/null +++ b/api_app/tests_ma/test_services/test_airlock_storage_helper.py @@ -0,0 +1,140 @@ +from models.domain.airlock_request import AirlockRequestStatus +import pytest +from services.airlock_storage_helper import get_storage_account_name_for_request, get_container_name_for_request +from resources import constants + + +class TestGetStorageAccountNameForRequestConsolidatedMode: + + class TestImportRequestsConsolidated: + + def test_import_draft_uses_core_storage(self): + account = get_storage_account_name_for_request( + constants.IMPORT_TYPE, AirlockRequestStatus.Draft, "tre123" + ) + assert account == "stalairlocktre123" + + def test_import_submitted_uses_core_storage(self): + account = get_storage_account_name_for_request( + constants.IMPORT_TYPE, AirlockRequestStatus.Submitted, "tre123" + ) + assert account == "stalairlocktre123" + + def test_import_in_review_uses_core_storage(self): + account = get_storage_account_name_for_request( + constants.IMPORT_TYPE, AirlockRequestStatus.InReview, "tre123" + ) + assert account == "stalairlocktre123" + + def test_import_approved_uses_workspace_global_storage(self): + account = get_storage_account_name_for_request( + constants.IMPORT_TYPE, AirlockRequestStatus.Approved, "tre123" + ) + assert account == "stalairlockgtre123" + + def test_import_approval_in_progress_uses_workspace_global_storage(self): + account = get_storage_account_name_for_request( + constants.IMPORT_TYPE, AirlockRequestStatus.ApprovalInProgress, "tre123" + ) + assert account == "stalairlockgtre123" + + def test_import_rejected_uses_core_storage(self): + account = get_storage_account_name_for_request( + constants.IMPORT_TYPE, AirlockRequestStatus.Rejected, "tre123" + ) + assert account == "stalairlocktre123" + + def test_import_blocked_uses_core_storage(self): + account = get_storage_account_name_for_request( + constants.IMPORT_TYPE, AirlockRequestStatus.Blocked, "tre123" + ) + assert account == "stalairlocktre123" + + class TestExportRequestsConsolidated: + + def test_export_draft_uses_workspace_global_storage(self): + account = get_storage_account_name_for_request( + constants.EXPORT_TYPE, AirlockRequestStatus.Draft, "tre123" + ) + assert account == "stalairlockgtre123" + + def test_export_submitted_uses_workspace_global_storage(self): + account = get_storage_account_name_for_request( + constants.EXPORT_TYPE, AirlockRequestStatus.Submitted, "tre123" + ) + assert account == "stalairlockgtre123" + + def test_export_in_review_uses_workspace_global_storage(self): + account = get_storage_account_name_for_request( + constants.EXPORT_TYPE, AirlockRequestStatus.InReview, "tre123" + ) + assert account == "stalairlockgtre123" + + def test_export_approved_uses_core_storage(self): + account = get_storage_account_name_for_request( + constants.EXPORT_TYPE, AirlockRequestStatus.Approved, "tre123" + ) + assert account == "stalairlocktre123" + + def test_export_approval_in_progress_uses_core_storage(self): + account = get_storage_account_name_for_request( + constants.EXPORT_TYPE, AirlockRequestStatus.ApprovalInProgress, "tre123" + ) + assert account == "stalairlocktre123" + + def test_export_rejected_uses_workspace_global_storage(self): + account = get_storage_account_name_for_request( + constants.EXPORT_TYPE, AirlockRequestStatus.Rejected, "tre123" + ) + assert account == "stalairlockgtre123" + + def test_export_blocked_uses_workspace_global_storage(self): + account = get_storage_account_name_for_request( + constants.EXPORT_TYPE, AirlockRequestStatus.Blocked, "tre123" + ) + assert account == "stalairlockgtre123" + + +class TestABACStageConstants: + + def test_import_external_stage_constant_value(self): + assert constants.STAGE_IMPORT_EXTERNAL == "import-external" + + def test_import_in_progress_stage_constant_value(self): + assert constants.STAGE_IMPORT_IN_PROGRESS == "import-in-progress" + + def test_export_approved_stage_constant_value(self): + assert constants.STAGE_EXPORT_APPROVED == "export-approved" + + def test_import_approved_stage_constant_value(self): + assert constants.STAGE_IMPORT_APPROVED == "import-approved" + + def test_import_rejected_stage_constant_value(self): + assert constants.STAGE_IMPORT_REJECTED == "import-rejected" + + def test_import_blocked_stage_constant_value(self): + assert constants.STAGE_IMPORT_BLOCKED == "import-blocked" + + def test_export_internal_stage_constant_value(self): + assert constants.STAGE_EXPORT_INTERNAL == "export-internal" + + def test_export_in_progress_stage_constant_value(self): + assert constants.STAGE_EXPORT_IN_PROGRESS == "export-in-progress" + + def test_export_rejected_stage_constant_value(self): + assert constants.STAGE_EXPORT_REJECTED == "export-rejected" + + def test_export_blocked_stage_constant_value(self): + assert constants.STAGE_EXPORT_BLOCKED == "export-blocked" + + +class TestGetContainerNameForRequest: + + def test_draft_uses_its_own_container(self): + assert get_container_name_for_request("req-1", AirlockRequestStatus.Draft) == "req-1-draft" + + @pytest.mark.parametrize("status", [ + AirlockRequestStatus.Submitted, AirlockRequestStatus.InReview, + AirlockRequestStatus.Approved, AirlockRequestStatus.Rejected, AirlockRequestStatus.Blocked]) + def test_sealed_stages_use_the_request_id(self, status): + assert get_container_name_for_request("req-1", status) == "req-1" diff --git a/api_app/tests_ma/test_services/test_legacy_airlock_guard.py b/api_app/tests_ma/test_services/test_legacy_airlock_guard.py new file mode 100644 index 0000000000..fc0713c2a4 --- /dev/null +++ b/api_app/tests_ma/test_services/test_legacy_airlock_guard.py @@ -0,0 +1,104 @@ +from unittest.mock import AsyncMock, patch +import pytest + +from services.legacy_airlock_guard import ( + ensure_airlock_version_change_allowed, + ensure_workspace_airlock_version_supported, +) +from models.schemas.resource import ResourcePatch + + +def _workspace(airlock_version=None): + ws = AsyncMock() + ws.id = "0b9c8928-9f25-4522-8f48-595105516531" + ws.properties = {} if airlock_version is None else {"airlock_version": airlock_version} + return ws + + +@pytest.mark.asyncio +async def test_ensure_airlock_version_change_allowed_noop_when_version_unchanged(): + request_repo = AsyncMock() + await ensure_airlock_version_change_allowed(_workspace(1), ResourcePatch(properties={"airlock_version": 1}), request_repo) + request_repo.get_in_flight_airlock_request_ids_for_workspace.assert_not_called() + + +@pytest.mark.asyncio +async def test_ensure_airlock_version_change_allowed_noop_when_no_version_in_patch(): + request_repo = AsyncMock() + await ensure_airlock_version_change_allowed(_workspace(1), ResourcePatch(properties={"display_name": "x"}), request_repo) + request_repo.get_in_flight_airlock_request_ids_for_workspace.assert_not_called() + + +@pytest.mark.asyncio +async def test_ensure_airlock_version_change_allowed_permits_change_when_no_in_flight(): + request_repo = AsyncMock() + request_repo.get_in_flight_airlock_request_ids_for_workspace.return_value = [] + await ensure_airlock_version_change_allowed(_workspace(1), ResourcePatch(properties={"airlock_version": 2}), request_repo) + + +@pytest.mark.asyncio +async def test_ensure_airlock_version_change_allowed_blocks_upgrade_with_in_flight_requests(): + request_repo = AsyncMock() + request_repo.get_in_flight_airlock_request_ids_for_workspace.return_value = ["req-1"] + with pytest.raises(ValueError): + await ensure_airlock_version_change_allowed(_workspace(1), ResourcePatch(properties={"airlock_version": 2}), request_repo) + + +def test_ensure_workspace_airlock_version_supported_allows_when_legacy_enabled(): + with patch("services.legacy_airlock_guard.config.ENABLE_LEGACY_AIRLOCK", new=True): + ensure_workspace_airlock_version_supported({"enable_airlock": True, "airlock_version": 1}) + + +def test_ensure_workspace_airlock_version_supported_allows_v2_when_legacy_disabled(): + with patch("services.legacy_airlock_guard.config.ENABLE_LEGACY_AIRLOCK", new=False): + ensure_workspace_airlock_version_supported({"enable_airlock": True, "airlock_version": 2}) + + +def test_ensure_workspace_airlock_version_supported_blocks_v1_when_legacy_disabled(): + with patch("services.legacy_airlock_guard.config.ENABLE_LEGACY_AIRLOCK", new=False): + with pytest.raises(ValueError): + ensure_workspace_airlock_version_supported({"enable_airlock": True, "airlock_version": 1}) + + +def test_ensure_workspace_airlock_version_supported_blocks_v1_when_enable_airlock_omitted(): + with patch("services.legacy_airlock_guard.config.ENABLE_LEGACY_AIRLOCK", new=False): + with pytest.raises(ValueError): + ensure_workspace_airlock_version_supported({"airlock_version": 1}) + + +def test_unstamped_workspace_defaults_to_v1_legacy(): + # A missing airlock_version defaults to legacy (v1), so it is blocked when legacy is disabled. + with patch("services.legacy_airlock_guard.config.ENABLE_LEGACY_AIRLOCK", new=False): + with pytest.raises(ValueError): + ensure_workspace_airlock_version_supported({"enable_airlock": True}) + + +def test_unstamped_workspace_is_treated_as_v1_when_validating_an_existing_one(): + with patch("services.legacy_airlock_guard.config.ENABLE_LEGACY_AIRLOCK", new=False): + with pytest.raises(ValueError): + ensure_workspace_airlock_version_supported({"enable_airlock": True}, default_version=1) + + +def test_ensure_workspace_airlock_version_supported_allows_v2_on_manual_auth(): + with patch("services.legacy_airlock_guard.config.ENABLE_LEGACY_AIRLOCK", new=True): + ensure_workspace_airlock_version_supported({"enable_airlock": True, "airlock_version": 2, "auth_type": "Manual"}) + + +def test_ensure_workspace_airlock_version_supported_allows_v2_on_automatic_auth(): + with patch("services.legacy_airlock_guard.config.ENABLE_LEGACY_AIRLOCK", new=True): + ensure_workspace_airlock_version_supported({"enable_airlock": True, "airlock_version": 2, "auth_type": "Automatic"}) + + +def test_unspecified_version_with_manual_auth_defaults_to_v1_and_is_blocked_when_legacy_disabled(): + with patch("services.legacy_airlock_guard.config.ENABLE_LEGACY_AIRLOCK", new=False): + with pytest.raises(ValueError): + ensure_workspace_airlock_version_supported({"enable_airlock": True, "auth_type": "Manual"}) + + +@pytest.mark.asyncio +async def test_ensure_airlock_version_change_allowed_blocks_downgrade(): + request_repo = AsyncMock() + with pytest.raises(ValueError, match="downgrading is not supported"): + await ensure_airlock_version_change_allowed(_workspace(2), ResourcePatch(properties={"airlock_version": 1}), request_repo) + # A downgrade must be rejected outright, before even checking in-flight requests. + request_repo.get_in_flight_airlock_request_ids_for_workspace.assert_not_called() diff --git a/config.sample.yaml b/config.sample.yaml index 95fedf2835..7be113d5bf 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -16,7 +16,6 @@ management: # encryption_kv_name: __CHANGE_ME__ # Azure Resource Manager credentials used for CI/CD pipelines arm_subscription_id: __CHANGE_ME__ - # If you want to override the currently signed in credentials # You would do this if running commands like `make terraform-install DIR=./templates/workspaces/base` # arm_tenant_id: __CHANGE_ME__ @@ -38,6 +37,8 @@ tre: resource_processor_vmss_sku: Standard_B2s enable_swagger: true enable_airlock_malware_scanning: true + # WARNING: Disabling legacy airlock deletes all v1 storage accounts and their data on core redeployment. + enable_legacy_airlock: false # Set to true if want to ensure users have an email address before airlock request is created # Used if rely on email notifications for governance purposes @@ -58,7 +59,6 @@ tre: # Set to true if TreAdmins should be able to assign and de-assign users to workspaces via the UI user_management_enabled: false - # Uncomment to enable DNS Security policy on the system, and add any known DNS names that you need to allow # DNS queries on, in addition to those in the core list in core/terraform/allowed-dns.json # Note, these need to be fully qualified, i.e. they end in a dot(.) @@ -101,6 +101,7 @@ ui_config: ui_site_name: "Azure TRE" # Footer text shown in the bottom left hand corner of the TRE portal ui_footer_text: "Azure Trusted Research Environment" + #developer_settings: # Locks will not be added to stateful resources so they can be easily removed # stateful_resources_locked: false diff --git a/config_schema.json b/config_schema.json index abfaf97217..56eba4a703 100644 --- a/config_schema.json +++ b/config_schema.json @@ -85,6 +85,11 @@ "description": "Require email check for airlock.", "type": "boolean" }, + "enable_legacy_airlock": { + "description": "Deploy v1 legacy per-stage airlock storage accounts in core. Required for workspaces using airlock_version=1.", + "type": "boolean", + "default": true + }, "core_address_space": { "description": "TRE core address spaces.", "type": "string" diff --git a/core/terraform/airlock/data.tf b/core/terraform/airlock/data.tf index dbec1db64c..6b8b48b775 100644 --- a/core/terraform/airlock/data.tf +++ b/core/terraform/airlock/data.tf @@ -2,10 +2,15 @@ data "local_file" "airlock_processor_version" { filename = "${path.root}/../../airlock_processor/_version.py" } +data "azurerm_private_dns_zone" "blobcore" { + name = split("/", var.blob_core_dns_zone_id)[8] + resource_group_name = var.resource_group_name +} + data "azurerm_monitor_diagnostic_categories" "eventgrid_custom_topics" { resource_id = azurerm_eventgrid_topic.airlock_notification.id } data "azurerm_monitor_diagnostic_categories" "eventgrid_system_topics" { - resource_id = azurerm_eventgrid_system_topic.export_approved_blob_created.id + resource_id = azurerm_eventgrid_system_topic.airlock_blob_created.id } diff --git a/core/terraform/airlock/eventgrid_topics.tf b/core/terraform/airlock/eventgrid_topics.tf index 4041b56240..b8799ab662 100644 --- a/core/terraform/airlock/eventgrid_topics.tf +++ b/core/terraform/airlock/eventgrid_topics.tf @@ -191,136 +191,6 @@ resource "azurerm_role_assignment" "servicebus_sender_scan_result" { } # System topic -resource "azurerm_eventgrid_system_topic" "import_inprogress_blob_created" { - name = local.import_inprogress_sys_topic_name - location = var.location - resource_group_name = var.resource_group_name - source_resource_id = azurerm_storage_account.sa_import_in_progress.id - topic_type = "Microsoft.Storage.StorageAccounts" - - identity { - type = "SystemAssigned" - } - - tags = merge(var.tre_core_tags, { - Publishers = "airlock;import-in-progress-sa" - }) - - depends_on = [ - azurerm_storage_account.sa_import_in_progress - ] - - lifecycle { ignore_changes = [tags] } -} - -resource "azurerm_role_assignment" "servicebus_sender_import_inprogress_blob_created" { - scope = var.airlock_servicebus.id - role_definition_name = "Azure Service Bus Data Sender" - principal_id = azurerm_eventgrid_system_topic.import_inprogress_blob_created.identity[0].principal_id - - depends_on = [ - azurerm_eventgrid_system_topic.import_inprogress_blob_created - ] -} - - -resource "azurerm_eventgrid_system_topic" "import_rejected_blob_created" { - name = local.import_rejected_sys_topic_name - location = var.location - resource_group_name = var.resource_group_name - source_resource_id = azurerm_storage_account.sa_import_rejected.id - topic_type = "Microsoft.Storage.StorageAccounts" - - identity { - type = "SystemAssigned" - } - - tags = merge(var.tre_core_tags, { - Publishers = "airlock;import-rejected-sa" - }) - - depends_on = [ - azurerm_storage_account.sa_import_rejected, - ] - - lifecycle { ignore_changes = [tags] } -} - -resource "azurerm_role_assignment" "servicebus_sender_import_rejected_blob_created" { - scope = var.airlock_servicebus.id - role_definition_name = "Azure Service Bus Data Sender" - principal_id = azurerm_eventgrid_system_topic.import_rejected_blob_created.identity[0].principal_id - - depends_on = [ - azurerm_eventgrid_system_topic.import_rejected_blob_created - ] -} - -resource "azurerm_eventgrid_system_topic" "import_blocked_blob_created" { - name = local.import_blocked_sys_topic_name - location = var.location - resource_group_name = var.resource_group_name - source_resource_id = azurerm_storage_account.sa_import_blocked.id - topic_type = "Microsoft.Storage.StorageAccounts" - - identity { - type = "SystemAssigned" - } - - tags = merge(var.tre_core_tags, { - Publishers = "airlock;import-blocked-sa" - }) - - depends_on = [ - azurerm_storage_account.sa_import_blocked, - ] - - lifecycle { ignore_changes = [tags] } -} - -resource "azurerm_role_assignment" "servicebus_sender_import_blocked_blob_created" { - scope = var.airlock_servicebus.id - role_definition_name = "Azure Service Bus Data Sender" - principal_id = azurerm_eventgrid_system_topic.import_blocked_blob_created.identity[0].principal_id - - depends_on = [ - azurerm_eventgrid_system_topic.import_blocked_blob_created - ] -} - - -resource "azurerm_eventgrid_system_topic" "export_approved_blob_created" { - name = local.export_approved_sys_topic_name - location = var.location - resource_group_name = var.resource_group_name - source_resource_id = azurerm_storage_account.sa_export_approved.id - topic_type = "Microsoft.Storage.StorageAccounts" - - identity { - type = "SystemAssigned" - } - - tags = merge(var.tre_core_tags, { - Publishers = "airlock;export-approved-sa" - }) - - depends_on = [ - azurerm_storage_account.sa_export_approved, - ] - - lifecycle { ignore_changes = [tags] } -} - -resource "azurerm_role_assignment" "servicebus_sender_export_approved_blob_created" { - scope = var.airlock_servicebus.id - role_definition_name = "Azure Service Bus Data Sender" - principal_id = azurerm_eventgrid_system_topic.export_approved_blob_created.identity[0].principal_id - - depends_on = [ - azurerm_eventgrid_system_topic.export_approved_blob_created - ] -} - # Custom topic (for airlock notifications) resource "azurerm_eventgrid_topic" "airlock_notification" { name = local.notification_topic_name @@ -442,25 +312,9 @@ resource "azurerm_eventgrid_event_subscription" "scan_result" { ] } -resource "azurerm_eventgrid_event_subscription" "import_inprogress_blob_created" { - name = local.import_inprogress_eventgrid_subscription_name - scope = azurerm_storage_account.sa_import_in_progress.id - - service_bus_topic_endpoint_id = azurerm_servicebus_topic.blob_created.id - - delivery_identity { - type = "SystemAssigned" - } - - depends_on = [ - azurerm_eventgrid_system_topic.import_inprogress_blob_created, - azurerm_role_assignment.servicebus_sender_import_inprogress_blob_created - ] -} - -resource "azurerm_eventgrid_event_subscription" "import_rejected_blob_created" { - name = local.import_rejected_eventgrid_subscription_name - scope = azurerm_storage_account.sa_import_rejected.id +resource "azurerm_eventgrid_event_subscription" "airlock_blob_created" { + name = "airlock-blob-created-${var.tre_id}" + scope = azurerm_storage_account.sa_airlock_core.id service_bus_topic_endpoint_id = azurerm_servicebus_topic.blob_created.id @@ -468,18 +322,17 @@ resource "azurerm_eventgrid_event_subscription" "import_rejected_blob_created" { type = "SystemAssigned" } - # Todo add Dead_letter + included_event_types = ["Microsoft.Storage.BlobCreated"] depends_on = [ - azurerm_eventgrid_system_topic.import_rejected_blob_created, - azurerm_role_assignment.servicebus_sender_import_rejected_blob_created + azurerm_eventgrid_system_topic.airlock_blob_created, + azurerm_role_assignment.servicebus_sender_airlock_blob_created ] } - -resource "azurerm_eventgrid_event_subscription" "import_blocked_blob_created" { - name = local.import_blocked_eventgrid_subscription_name - scope = azurerm_storage_account.sa_import_blocked.id +resource "azurerm_eventgrid_event_subscription" "airlock_workspace_global_blob_created" { + name = "airlock-blob-created-global-${var.tre_id}" + scope = azurerm_storage_account.sa_airlock_workspace_global.id service_bus_topic_endpoint_id = azurerm_servicebus_topic.blob_created.id @@ -487,27 +340,11 @@ resource "azurerm_eventgrid_event_subscription" "import_blocked_blob_created" { type = "SystemAssigned" } - # Todo add Dead_letter + included_event_types = ["Microsoft.Storage.BlobCreated"] depends_on = [ - azurerm_eventgrid_system_topic.import_blocked_blob_created, - azurerm_role_assignment.servicebus_sender_import_blocked_blob_created - ] -} - -resource "azurerm_eventgrid_event_subscription" "export_approved_blob_created" { - name = local.export_approved_eventgrid_subscription_name - scope = azurerm_storage_account.sa_export_approved.id - - service_bus_topic_endpoint_id = azurerm_servicebus_topic.blob_created.id - - delivery_identity { - type = "SystemAssigned" - } - - depends_on = [ - azurerm_eventgrid_system_topic.export_approved_blob_created, - azurerm_role_assignment.servicebus_sender_export_approved_blob_created + azurerm_eventgrid_system_topic.airlock_workspace_global_blob_created, + azurerm_role_assignment.servicebus_sender_airlock_workspace_global_blob_created ] } @@ -537,12 +374,18 @@ resource "azurerm_monitor_diagnostic_setting" "eventgrid_custom_topics" { } resource "azurerm_monitor_diagnostic_setting" "eventgrid_system_topics" { - for_each = { - (azurerm_eventgrid_system_topic.import_inprogress_blob_created.name) = azurerm_eventgrid_system_topic.import_inprogress_blob_created.id, - (azurerm_eventgrid_system_topic.import_rejected_blob_created.name) = azurerm_eventgrid_system_topic.import_rejected_blob_created.id, - (azurerm_eventgrid_system_topic.import_blocked_blob_created.name) = azurerm_eventgrid_system_topic.import_blocked_blob_created.id, - (azurerm_eventgrid_system_topic.export_approved_blob_created.name) = azurerm_eventgrid_system_topic.export_approved_blob_created.id, - } + for_each = merge( + { + (azurerm_eventgrid_system_topic.airlock_blob_created.name) = azurerm_eventgrid_system_topic.airlock_blob_created.id, + (azurerm_eventgrid_system_topic.airlock_workspace_global_blob_created.name) = azurerm_eventgrid_system_topic.airlock_workspace_global_blob_created.id, + }, + var.enable_legacy_airlock ? { + (azurerm_eventgrid_system_topic.import_inprogress_blob_created[0].name) = azurerm_eventgrid_system_topic.import_inprogress_blob_created[0].id, + (azurerm_eventgrid_system_topic.import_rejected_blob_created[0].name) = azurerm_eventgrid_system_topic.import_rejected_blob_created[0].id, + (azurerm_eventgrid_system_topic.import_blocked_blob_created[0].name) = azurerm_eventgrid_system_topic.import_blocked_blob_created[0].id, + (azurerm_eventgrid_system_topic.export_approved_blob_created[0].name) = azurerm_eventgrid_system_topic.export_approved_blob_created[0].id, + } : {} + ) name = "${each.key}-diagnostics" target_resource_id = each.value diff --git a/core/terraform/airlock/eventgrid_topics_v1.tf b/core/terraform/airlock/eventgrid_topics_v1.tf new file mode 100644 index 0000000000..cce4b3564d --- /dev/null +++ b/core/terraform/airlock/eventgrid_topics_v1.tf @@ -0,0 +1,266 @@ +resource "azurerm_eventgrid_system_topic" "import_inprogress_blob_created" { + count = var.enable_legacy_airlock ? 1 : 0 + name = local.import_inprogress_sys_topic_name + location = var.location + resource_group_name = var.resource_group_name + source_resource_id = azurerm_storage_account.sa_import_in_progress[0].id + topic_type = "Microsoft.Storage.StorageAccounts" + + identity { + type = "SystemAssigned" + } + + tags = merge(var.tre_core_tags, { + Publishers = "airlock;import-in-progress-sa" + }) + + depends_on = [ + azurerm_storage_account.sa_import_in_progress + ] + + lifecycle { ignore_changes = [tags] } +} + +resource "azurerm_role_assignment" "servicebus_sender_import_inprogress_blob_created" { + count = var.enable_legacy_airlock ? 1 : 0 + scope = var.airlock_servicebus.id + role_definition_name = "Azure Service Bus Data Sender" + principal_id = azurerm_eventgrid_system_topic.import_inprogress_blob_created[0].identity[0].principal_id + + depends_on = [ + azurerm_eventgrid_system_topic.import_inprogress_blob_created + ] +} + + +resource "azurerm_eventgrid_system_topic" "import_rejected_blob_created" { + count = var.enable_legacy_airlock ? 1 : 0 + name = local.import_rejected_sys_topic_name + location = var.location + resource_group_name = var.resource_group_name + source_resource_id = azurerm_storage_account.sa_import_rejected[0].id + topic_type = "Microsoft.Storage.StorageAccounts" + + identity { + type = "SystemAssigned" + } + + tags = merge(var.tre_core_tags, { + Publishers = "airlock;import-rejected-sa" + }) + + depends_on = [ + azurerm_storage_account.sa_import_rejected, + ] + + lifecycle { ignore_changes = [tags] } +} + +resource "azurerm_role_assignment" "servicebus_sender_import_rejected_blob_created" { + count = var.enable_legacy_airlock ? 1 : 0 + scope = var.airlock_servicebus.id + role_definition_name = "Azure Service Bus Data Sender" + principal_id = azurerm_eventgrid_system_topic.import_rejected_blob_created[0].identity[0].principal_id + + depends_on = [ + azurerm_eventgrid_system_topic.import_rejected_blob_created + ] +} + +resource "azurerm_eventgrid_system_topic" "import_blocked_blob_created" { + count = var.enable_legacy_airlock ? 1 : 0 + name = local.import_blocked_sys_topic_name + location = var.location + resource_group_name = var.resource_group_name + source_resource_id = azurerm_storage_account.sa_import_blocked[0].id + topic_type = "Microsoft.Storage.StorageAccounts" + + identity { + type = "SystemAssigned" + } + + tags = merge(var.tre_core_tags, { + Publishers = "airlock;import-blocked-sa" + }) + + depends_on = [ + azurerm_storage_account.sa_import_blocked, + ] + + lifecycle { ignore_changes = [tags] } +} + +resource "azurerm_role_assignment" "servicebus_sender_import_blocked_blob_created" { + count = var.enable_legacy_airlock ? 1 : 0 + scope = var.airlock_servicebus.id + role_definition_name = "Azure Service Bus Data Sender" + principal_id = azurerm_eventgrid_system_topic.import_blocked_blob_created[0].identity[0].principal_id + + depends_on = [ + azurerm_eventgrid_system_topic.import_blocked_blob_created + ] +} + + +resource "azurerm_eventgrid_system_topic" "export_approved_blob_created" { + count = var.enable_legacy_airlock ? 1 : 0 + name = local.export_approved_sys_topic_name + location = var.location + resource_group_name = var.resource_group_name + source_resource_id = azurerm_storage_account.sa_export_approved[0].id + topic_type = "Microsoft.Storage.StorageAccounts" + + identity { + type = "SystemAssigned" + } + + tags = merge(var.tre_core_tags, { + Publishers = "airlock;export-approved-sa" + }) + + depends_on = [ + azurerm_storage_account.sa_export_approved, + ] + + lifecycle { ignore_changes = [tags] } +} + +resource "azurerm_role_assignment" "servicebus_sender_export_approved_blob_created" { + count = var.enable_legacy_airlock ? 1 : 0 + scope = var.airlock_servicebus.id + role_definition_name = "Azure Service Bus Data Sender" + principal_id = azurerm_eventgrid_system_topic.export_approved_blob_created[0].identity[0].principal_id + + depends_on = [ + azurerm_eventgrid_system_topic.export_approved_blob_created + ] +} + +resource "azurerm_eventgrid_event_subscription" "import_inprogress_blob_created" { + count = var.enable_legacy_airlock ? 1 : 0 + name = local.import_inprogress_eventgrid_subscription_name + scope = azurerm_storage_account.sa_import_in_progress[0].id + + service_bus_topic_endpoint_id = azurerm_servicebus_topic.blob_created.id + + delivery_identity { + type = "SystemAssigned" + } + + depends_on = [ + azurerm_eventgrid_system_topic.import_inprogress_blob_created, + azurerm_role_assignment.servicebus_sender_import_inprogress_blob_created + ] +} + +resource "azurerm_eventgrid_event_subscription" "import_rejected_blob_created" { + count = var.enable_legacy_airlock ? 1 : 0 + name = local.import_rejected_eventgrid_subscription_name + scope = azurerm_storage_account.sa_import_rejected[0].id + + service_bus_topic_endpoint_id = azurerm_servicebus_topic.blob_created.id + + delivery_identity { + type = "SystemAssigned" + } + + depends_on = [ + azurerm_eventgrid_system_topic.import_rejected_blob_created, + azurerm_role_assignment.servicebus_sender_import_rejected_blob_created + ] +} + +resource "azurerm_eventgrid_event_subscription" "import_blocked_blob_created" { + count = var.enable_legacy_airlock ? 1 : 0 + name = local.import_blocked_eventgrid_subscription_name + scope = azurerm_storage_account.sa_import_blocked[0].id + + service_bus_topic_endpoint_id = azurerm_servicebus_topic.blob_created.id + + delivery_identity { + type = "SystemAssigned" + } + + depends_on = [ + azurerm_eventgrid_system_topic.import_blocked_blob_created, + azurerm_role_assignment.servicebus_sender_import_blocked_blob_created + ] +} + +resource "azurerm_eventgrid_event_subscription" "export_approved_blob_created" { + count = var.enable_legacy_airlock ? 1 : 0 + name = local.export_approved_eventgrid_subscription_name + scope = azurerm_storage_account.sa_export_approved[0].id + + service_bus_topic_endpoint_id = azurerm_servicebus_topic.blob_created.id + + delivery_identity { + type = "SystemAssigned" + } + + depends_on = [ + azurerm_eventgrid_system_topic.export_approved_blob_created, + azurerm_role_assignment.servicebus_sender_export_approved_blob_created + ] +} + +# These resources predate the enable_legacy_airlock toggle; preserve their state addresses. +moved { + from = azurerm_eventgrid_system_topic.import_inprogress_blob_created + to = azurerm_eventgrid_system_topic.import_inprogress_blob_created[0] +} + +moved { + from = azurerm_role_assignment.servicebus_sender_import_inprogress_blob_created + to = azurerm_role_assignment.servicebus_sender_import_inprogress_blob_created[0] +} + +moved { + from = azurerm_eventgrid_system_topic.import_rejected_blob_created + to = azurerm_eventgrid_system_topic.import_rejected_blob_created[0] +} + +moved { + from = azurerm_role_assignment.servicebus_sender_import_rejected_blob_created + to = azurerm_role_assignment.servicebus_sender_import_rejected_blob_created[0] +} + +moved { + from = azurerm_eventgrid_system_topic.import_blocked_blob_created + to = azurerm_eventgrid_system_topic.import_blocked_blob_created[0] +} + +moved { + from = azurerm_role_assignment.servicebus_sender_import_blocked_blob_created + to = azurerm_role_assignment.servicebus_sender_import_blocked_blob_created[0] +} + +moved { + from = azurerm_eventgrid_system_topic.export_approved_blob_created + to = azurerm_eventgrid_system_topic.export_approved_blob_created[0] +} + +moved { + from = azurerm_role_assignment.servicebus_sender_export_approved_blob_created + to = azurerm_role_assignment.servicebus_sender_export_approved_blob_created[0] +} + +moved { + from = azurerm_eventgrid_event_subscription.import_inprogress_blob_created + to = azurerm_eventgrid_event_subscription.import_inprogress_blob_created[0] +} + +moved { + from = azurerm_eventgrid_event_subscription.import_rejected_blob_created + to = azurerm_eventgrid_event_subscription.import_rejected_blob_created[0] +} + +moved { + from = azurerm_eventgrid_event_subscription.import_blocked_blob_created + to = azurerm_eventgrid_event_subscription.import_blocked_blob_created[0] +} + +moved { + from = azurerm_eventgrid_event_subscription.export_approved_blob_created + to = azurerm_eventgrid_event_subscription.export_approved_blob_created[0] +} diff --git a/core/terraform/airlock/identity.tf b/core/terraform/airlock/identity.tf index b4e272c144..0cdb553458 100644 --- a/core/terraform/airlock/identity.tf +++ b/core/terraform/airlock/identity.tf @@ -49,21 +49,6 @@ resource "azurerm_role_assignment" "eventgrid_data_sender_data_deletion" { principal_id = azurerm_user_assigned_identity.airlock_id.principal_id } -resource "azurerm_role_assignment" "airlock_blob_data_contributor" { - count = length(local.airlock_sa_blob_data_contributor) - scope = local.airlock_sa_blob_data_contributor[count.index] - role_definition_name = "Storage Blob Data Contributor" - principal_id = azurerm_user_assigned_identity.airlock_id.principal_id -} - -# This might be considered redundent since we give Virtual Machine Contributor -# at the subscription level, but best to be explicit. -resource "azurerm_role_assignment" "api_sa_data_contributor" { - count = length(local.api_sa_data_contributor) - scope = local.api_sa_data_contributor[count.index] - role_definition_name = "Storage Blob Data Contributor" - principal_id = var.api_principal_id -} # Permissions needed for the Function Host to work correctly. resource "azurerm_role_assignment" "function_host_storage" { diff --git a/core/terraform/airlock/locals.tf b/core/terraform/airlock/locals.tf index 838ddf091a..830471c225 100644 --- a/core/terraform/airlock/locals.tf +++ b/core/terraform/airlock/locals.tf @@ -1,26 +1,14 @@ locals { version = replace(replace(replace(data.local_file.airlock_processor_version.content, "__version__ = \"", ""), "\"", ""), "\n", "") - # STorage AirLock EXternal - import_external_storage_name = lower(replace("stalimex${var.tre_id}", "-", "")) - # STorage AirLock IMport InProgress - import_in_progress_storage_name = lower(replace("stalimip${var.tre_id}", "-", "")) - # STorage AirLock IMport REJected - import_rejected_storage_name = lower(replace("stalimrej${var.tre_id}", "-", "")) - # STorage AirLock IMport BLOCKED - import_blocked_storage_name = lower(replace("stalimblocked${var.tre_id}", "-", "")) - # STorage AirLock EXPort APProved - export_approved_storage_name = lower(replace("stalexapp${var.tre_id}", "-", "")) + airlock_core_storage_name = lower(replace("stalairlock${var.tre_id}", "-", "")) + + airlock_workspace_global_storage_name = lower(replace("stalairlockg${var.tre_id}", "-", "")) # Due to the following issue and Azure not liking delete and immediate recreate under the same name, # we had to change the resource names. https://github.com/hashicorp/terraform-provider-azurerm/issues/17389 topic_name_suffix = "v2-${var.tre_id}" - import_inprogress_sys_topic_name = "evgt-airlock-import-in-progress-${local.topic_name_suffix}" - import_rejected_sys_topic_name = "evgt-airlock-import-rejected-${local.topic_name_suffix}" - import_blocked_sys_topic_name = "evgt-airlock-import-blocked-${local.topic_name_suffix}" - export_approved_sys_topic_name = "evgt-airlock-export-approved-${local.topic_name_suffix}" - step_result_topic_name = "evgt-airlock-step-result-${local.topic_name_suffix}" status_changed_topic_name = "evgt-airlock-status-changed-${local.topic_name_suffix}" notification_topic_name = "evgt-airlock-notification-${local.topic_name_suffix}" @@ -35,31 +23,43 @@ locals { blob_created_al_processor_subscription_name = "airlock-blob-created-airlock-processor" - step_result_eventgrid_subscription_name = "evgs-airlock-update-status" - status_changed_eventgrid_subscription_name = "evgs-airlock-status-changed" - data_deletion_eventgrid_subscription_name = "evgs-airlock-data-deletion" - scan_result_eventgrid_subscription_name = "evgs-airlock-scan-result" + step_result_eventgrid_subscription_name = "evgs-airlock-update-status" + status_changed_eventgrid_subscription_name = "evgs-airlock-status-changed" + data_deletion_eventgrid_subscription_name = "evgs-airlock-data-deletion" + scan_result_eventgrid_subscription_name = "evgs-airlock-scan-result" + + import_external_storage_name = lower(replace("stalimex${var.tre_id}", "-", "")) + import_in_progress_storage_name = lower(replace("stalimip${var.tre_id}", "-", "")) + import_rejected_storage_name = lower(replace("stalimrej${var.tre_id}", "-", "")) + import_blocked_storage_name = lower(replace("stalimblocked${var.tre_id}", "-", "")) + export_approved_storage_name = lower(replace("stalexapp${var.tre_id}", "-", "")) + + import_inprogress_sys_topic_name = "evgt-airlock-import-in-progress-${local.topic_name_suffix}" + import_rejected_sys_topic_name = "evgt-airlock-import-rejected-${local.topic_name_suffix}" + import_blocked_sys_topic_name = "evgt-airlock-import-blocked-${local.topic_name_suffix}" + export_approved_sys_topic_name = "evgt-airlock-export-approved-${local.topic_name_suffix}" + import_inprogress_eventgrid_subscription_name = "evgs-airlock-import-in-progress-blob-created" import_rejected_eventgrid_subscription_name = "evgs-airlock-import-rejected-blob-created" import_blocked_eventgrid_subscription_name = "evgs-airlock-import-blocked-blob-created" export_approved_eventgrid_subscription_name = "evgs-airlock-export-approved-blob-created" - airlock_function_app_name = "func-airlock-processor-${var.tre_id}" - airlock_function_sa_name = lower(replace("stairlockp${var.tre_id}", "-", "")) + airlock_sa_blob_data_contributor = var.enable_legacy_airlock ? [ + azurerm_storage_account.sa_import_external[0].id, + azurerm_storage_account.sa_import_in_progress[0].id, + azurerm_storage_account.sa_import_rejected[0].id, + azurerm_storage_account.sa_export_approved[0].id, + azurerm_storage_account.sa_import_blocked[0].id + ] : [] - airlock_sa_blob_data_contributor = [ - azurerm_storage_account.sa_import_external.id, - azurerm_storage_account.sa_import_in_progress.id, - azurerm_storage_account.sa_import_rejected.id, - azurerm_storage_account.sa_export_approved.id, - azurerm_storage_account.sa_import_blocked.id - ] + api_sa_data_contributor = var.enable_legacy_airlock ? [ + azurerm_storage_account.sa_import_external[0].id, + azurerm_storage_account.sa_import_in_progress[0].id, + azurerm_storage_account.sa_export_approved[0].id + ] : [] - api_sa_data_contributor = [ - azurerm_storage_account.sa_import_external.id, - azurerm_storage_account.sa_import_in_progress.id, - azurerm_storage_account.sa_export_approved.id - ] + airlock_function_app_name = "func-airlock-processor-${var.tre_id}" + airlock_function_sa_name = lower(replace("stairlockp${var.tre_id}", "-", "")) servicebus_connection = "SERVICEBUS_CONNECTION" step_result_eventgrid_connection = "EVENT_GRID_STEP_RESULT_CONNECTION" diff --git a/core/terraform/airlock/storage_accounts.tf b/core/terraform/airlock/storage_accounts.tf index 13b8071ab2..d9d49fe0c9 100644 --- a/core/terraform/airlock/storage_accounts.tf +++ b/core/terraform/airlock/storage_accounts.tf @@ -1,8 +1,7 @@ -# 'External' storage account - drop location for import -resource "azurerm_storage_account" "sa_import_external" { - name = local.import_external_storage_name +resource "azurerm_storage_account" "sa_airlock_core" { + name = local.airlock_core_storage_name location = var.location resource_group_name = var.resource_group_name account_tier = "Standard" @@ -12,11 +11,9 @@ resource "azurerm_storage_account" "sa_import_external" { cross_tenant_replication_enabled = false shared_access_key_enabled = false local_user_enabled = false - # Don't allow anonymous access (unrelated to the 'public' networking rules) - allow_nested_items_to_be_public = false + allow_nested_items_to_be_public = false + public_network_access_enabled = true - # Important! we rely on the fact that the blob craeted events are issued when the creation of the blobs are done. - # This is true ONLY when Hierarchical Namespace is DISABLED is_hns_enabled = false # changing this value is destructive, hence attribute is in lifecycle.ignore_changes block below @@ -38,15 +35,46 @@ resource "azurerm_storage_account" "sa_import_external" { } } + # ABAC limits public access to user-facing stages and requires Private Link for import-in-progress. + network_rules { + default_action = "Allow" + bypass = ["AzureServices"] + } + tags = merge(var.tre_core_tags, { - description = "airlock;import;external" + description = "airlock;core;consolidated" + SecurityControl = "Ignore" }) lifecycle { ignore_changes = [infrastructure_encryption_enabled, tags] } } -resource "azurerm_private_endpoint" "stg_import_external_pe" { - name = "pe-stg-import-external-blob-${var.tre_id}" +resource "azapi_resource_action" "enable_defender_for_storage_core" { + count = var.enable_malware_scanning ? 1 : 0 + type = "Microsoft.Security/defenderForStorageSettings@2022-12-01-preview" + resource_id = "${azurerm_storage_account.sa_airlock_core.id}/providers/Microsoft.Security/defenderForStorageSettings/current" + method = "PUT" + + body = { + properties = { + isEnabled = true + malwareScanning = { + onUpload = { + isEnabled = true + capGBPerMonth = 5000 + }, + scanResultsEventGridTopicResourceId = azurerm_eventgrid_topic.scan_result[0].id + } + sensitiveDataDiscovery = { + isEnabled = false + } + overrideSubscriptionLevelSettings = true + } + } +} + +resource "azurerm_private_endpoint" "stg_airlock_core_pe_processor" { + name = "pe-stg-airlock-processor-${var.tre_id}" location = var.location resource_group_name = var.resource_group_name subnet_id = var.airlock_storage_subnet_id @@ -55,89 +83,90 @@ resource "azurerm_private_endpoint" "stg_import_external_pe" { lifecycle { ignore_changes = [tags] } private_dns_zone_group { - name = "pdzg-stg-import-external-blob-${var.tre_id}" + name = "pdzg-stg-airlock-processor-${var.tre_id}" private_dns_zone_ids = [var.blob_core_dns_zone_id] } private_service_connection { - name = "psc-stg-import-external-blob-${var.tre_id}" - private_connection_resource_id = azurerm_storage_account.sa_import_external.id + name = "psc-stg-airlock-processor-${var.tre_id}" + private_connection_resource_id = azurerm_storage_account.sa_airlock_core.id is_manual_connection = false subresource_names = ["Blob"] } } -# 'Approved' export -resource "azurerm_storage_account" "sa_export_approved" { - name = local.export_approved_storage_name - location = var.location - resource_group_name = var.resource_group_name - account_tier = "Standard" - account_replication_type = "LRS" - table_encryption_key_type = var.enable_cmk_encryption ? "Account" : "Service" - queue_encryption_key_type = var.enable_cmk_encryption ? "Account" : "Service" - cross_tenant_replication_enabled = false - shared_access_key_enabled = false - local_user_enabled = false - - # Don't allow anonymous access (unrelated to the 'public' networking rules) - allow_nested_items_to_be_public = false - - # Important! we rely on the fact that the blob craeted events are issued when the creation of the blobs are done. - # This is true ONLY when Hierarchical Namespace is DISABLED - is_hns_enabled = false - - # changing this value is destructive, hence attribute is in lifecycle.ignore_changes block below - infrastructure_encryption_enabled = true +resource "azurerm_eventgrid_system_topic" "airlock_blob_created" { + name = "evgt-airlock-blob-created-${var.tre_id}" + location = var.location + resource_group_name = var.resource_group_name + source_resource_id = azurerm_storage_account.sa_airlock_core.id + topic_type = "Microsoft.Storage.StorageAccounts" + tags = var.tre_core_tags - dynamic "identity" { - for_each = var.enable_cmk_encryption ? [1] : [] - content { - type = "UserAssigned" - identity_ids = [var.encryption_identity_id] - } + identity { + type = "SystemAssigned" } - dynamic "customer_managed_key" { - for_each = var.enable_cmk_encryption ? [1] : [] - content { - key_vault_key_id = var.encryption_key_versionless_id - user_assigned_identity_id = var.encryption_identity_id - } - } + lifecycle { ignore_changes = [tags] } +} - tags = merge(var.tre_core_tags, { - description = "airlock;export;approved" - }) +resource "azurerm_role_assignment" "servicebus_sender_airlock_blob_created" { + scope = var.airlock_servicebus.id + role_definition_name = "Azure Service Bus Data Sender" + principal_id = azurerm_eventgrid_system_topic.airlock_blob_created.identity[0].principal_id - lifecycle { ignore_changes = [infrastructure_encryption_enabled, tags] } + depends_on = [ + azurerm_eventgrid_system_topic.airlock_blob_created + ] } -resource "azurerm_private_endpoint" "stg_export_approved_pe" { - name = "pe-stg-export-approved-blob-${var.tre_id}" - location = var.location - resource_group_name = var.resource_group_name - subnet_id = var.airlock_storage_subnet_id - tags = var.tre_core_tags - lifecycle { ignore_changes = [tags] } - - private_dns_zone_group { - name = "pdzg-stg-export-approved-blob-${var.tre_id}" - private_dns_zone_ids = [var.blob_core_dns_zone_id] - } +resource "azurerm_role_assignment" "airlock_core_blob_data_contributor" { + scope = azurerm_storage_account.sa_airlock_core.id + role_definition_name = "Storage Blob Data Contributor" + principal_id = azurerm_user_assigned_identity.airlock_id.principal_id +} - private_service_connection { - name = "psc-stg-export-approved-blob-${var.tre_id}" - private_connection_resource_id = azurerm_storage_account.sa_export_approved.id - is_manual_connection = false - subresource_names = ["Blob"] - } +# Blob access is stage-limited; import-in-progress also requires Private Link. +resource "azurerm_role_assignment" "api_core_blob_data_contributor" { + scope = azurerm_storage_account.sa_airlock_core.id + role_definition_name = "Storage Blob Data Contributor" + principal_id = var.api_principal_id + + condition_version = "2.0" + condition = <<-EOT + ( + ( + !(ActionMatches{'Microsoft.Storage/storageAccounts/blobServices/containers/blobs/read'}) + AND !(ActionMatches{'Microsoft.Storage/storageAccounts/blobServices/containers/blobs/write'}) + AND !(ActionMatches{'Microsoft.Storage/storageAccounts/blobServices/containers/blobs/add/action'}) + AND !(ActionMatches{'Microsoft.Storage/storageAccounts/blobServices/containers/blobs/delete'}) + ) + OR + @Resource[Microsoft.Storage/storageAccounts/blobServices/containers/metadata:stage] + StringEquals 'import-external' + OR + ( + ActionMatches{'Microsoft.Storage/storageAccounts/blobServices/containers/blobs/read'} + AND + @Resource[Microsoft.Storage/storageAccounts/blobServices/containers/metadata:stage] + StringEquals 'export-approved' + ) + OR + ( + ActionMatches{'Microsoft.Storage/storageAccounts/blobServices/containers/blobs/read'} + AND + @Resource[Microsoft.Storage/storageAccounts/blobServices/containers/metadata:stage] + StringEquals 'import-in-progress' + AND + @Environment[isPrivateLink] BoolEquals true + ) + ) + EOT } -# 'In-Progress' storage account -resource "azurerm_storage_account" "sa_import_in_progress" { - name = local.import_in_progress_storage_name +resource "azurerm_storage_account" "sa_airlock_workspace_global" { + name = local.airlock_workspace_global_storage_name location = var.location resource_group_name = var.resource_group_name account_tier = "Standard" @@ -156,6 +185,11 @@ resource "azurerm_storage_account" "sa_import_in_progress" { # changing this value is destructive, hence attribute is in lifecycle.ignore_changes block below infrastructure_encryption_enabled = true + network_rules { + default_action = var.enable_local_debugging ? "Allow" : "Deny" + bypass = ["AzureServices"] + } + dynamic "identity" { for_each = var.enable_cmk_encryption ? [1] : [] content { @@ -173,22 +207,17 @@ resource "azurerm_storage_account" "sa_import_in_progress" { } tags = merge(var.tre_core_tags, { - description = "airlock;import;in-progress" + description = "airlock;workspace;global" }) - network_rules { - default_action = var.enable_local_debugging ? "Allow" : "Deny" - bypass = ["AzureServices"] - } - lifecycle { ignore_changes = [infrastructure_encryption_enabled, tags] } } -# Enable Airlock Malware Scanning on Core TRE -resource "azapi_resource_action" "enable_defender_for_storage" { + +resource "azapi_resource_action" "enable_defender_for_storage_workspace_global" { count = var.enable_malware_scanning ? 1 : 0 type = "Microsoft.Security/defenderForStorageSettings@2022-12-01-preview" - resource_id = "${azurerm_storage_account.sa_import_in_progress.id}/providers/Microsoft.Security/defenderForStorageSettings/current" + resource_id = "${azurerm_storage_account.sa_airlock_workspace_global.id}/providers/Microsoft.Security/defenderForStorageSettings/current" method = "PUT" body = { @@ -209,170 +238,88 @@ resource "azapi_resource_action" "enable_defender_for_storage" { } } -resource "azurerm_private_endpoint" "stg_import_inprogress_pe" { - name = "pe-stg-import-inprogress-blob-${var.tre_id}" + +resource "azurerm_eventgrid_system_topic" "airlock_workspace_global_blob_created" { + name = "evgt-airlock-blob-created-global-${var.tre_id}" location = var.location resource_group_name = var.resource_group_name - subnet_id = var.airlock_storage_subnet_id + source_resource_id = azurerm_storage_account.sa_airlock_workspace_global.id + topic_type = "Microsoft.Storage.StorageAccounts" tags = var.tre_core_tags - lifecycle { ignore_changes = [tags] } - - private_dns_zone_group { - name = "pdzg-stg-import-inprogress-blob-${var.tre_id}" - private_dns_zone_ids = [var.blob_core_dns_zone_id] + identity { + type = "SystemAssigned" } - private_service_connection { - name = "psc-stg-import-inprogress-blob-${var.tre_id}" - private_connection_resource_id = azurerm_storage_account.sa_import_in_progress.id - is_manual_connection = false - subresource_names = ["Blob"] - } + lifecycle { ignore_changes = [tags] } } +resource "azurerm_role_assignment" "servicebus_sender_airlock_workspace_global_blob_created" { + scope = var.airlock_servicebus.id + role_definition_name = "Azure Service Bus Data Sender" + principal_id = azurerm_eventgrid_system_topic.airlock_workspace_global_blob_created.identity[0].principal_id -# 'Rejected' storage account -resource "azurerm_storage_account" "sa_import_rejected" { - name = local.import_rejected_storage_name - location = var.location - resource_group_name = var.resource_group_name - account_tier = "Standard" - account_replication_type = "LRS" - table_encryption_key_type = var.enable_cmk_encryption ? "Account" : "Service" - queue_encryption_key_type = var.enable_cmk_encryption ? "Account" : "Service" - allow_nested_items_to_be_public = false - cross_tenant_replication_enabled = false - shared_access_key_enabled = false - local_user_enabled = false - - # Important! we rely on the fact that the blob craeted events are issued when the creation of the blobs are done. - # This is true ONLY when Hierarchical Namespace is DISABLED - is_hns_enabled = false - - # changing this value is destructive, hence attribute is in lifecycle.ignore_changes block below - infrastructure_encryption_enabled = true - - dynamic "identity" { - for_each = var.enable_cmk_encryption ? [1] : [] - content { - type = "UserAssigned" - identity_ids = [var.encryption_identity_id] - } - } - - dynamic "customer_managed_key" { - for_each = var.enable_cmk_encryption ? [1] : [] - content { - key_vault_key_id = var.encryption_key_versionless_id - user_assigned_identity_id = var.encryption_identity_id - } - } - - tags = merge(var.tre_core_tags, { - description = "airlock;import;rejected" - }) - - network_rules { - default_action = var.enable_local_debugging ? "Allow" : "Deny" - bypass = ["AzureServices"] - } - - lifecycle { ignore_changes = [infrastructure_encryption_enabled, tags] } + depends_on = [ + azurerm_eventgrid_system_topic.airlock_workspace_global_blob_created + ] } -resource "azurerm_private_endpoint" "stg_import_rejected_pe" { - name = "pe-stg-import-rejected-blob-${var.tre_id}" +resource "azurerm_private_endpoint" "stg_airlock_workspace_global_pe_processor" { + name = "pe-stg-airlock-ws-global-${var.tre_id}" location = var.location resource_group_name = var.resource_group_name subnet_id = var.airlock_storage_subnet_id + tags = var.tre_core_tags - private_dns_zone_group { - name = "pdzg-stg-import-rejected-blob-${var.tre_id}" - private_dns_zone_ids = [var.blob_core_dns_zone_id] - } + lifecycle { ignore_changes = [tags] } private_service_connection { - name = "psc-stg-import-rejected-blob-${var.tre_id}" - private_connection_resource_id = azurerm_storage_account.sa_import_rejected.id + name = "psc-stg-airlock-ws-global-${var.tre_id}" + private_connection_resource_id = azurerm_storage_account.sa_airlock_workspace_global.id is_manual_connection = false subresource_names = ["Blob"] } +} - tags = var.tre_core_tags +# Keep core DNS authoritative when workspaces create account-specific zones. +resource "azurerm_private_dns_zone" "airlock_workspace_global" { + name = "${local.airlock_workspace_global_storage_name}.${data.azurerm_private_dns_zone.blobcore.name}" + resource_group_name = var.resource_group_name + tags = var.tre_core_tags lifecycle { ignore_changes = [tags] } } -# 'Blocked' storage account -resource "azurerm_storage_account" "sa_import_blocked" { - name = local.import_blocked_storage_name - location = var.location - resource_group_name = var.resource_group_name - account_tier = "Standard" - account_replication_type = "LRS" - table_encryption_key_type = var.enable_cmk_encryption ? "Account" : "Service" - queue_encryption_key_type = var.enable_cmk_encryption ? "Account" : "Service" - allow_nested_items_to_be_public = false - cross_tenant_replication_enabled = false - shared_access_key_enabled = false - local_user_enabled = false - - # Important! we rely on the fact that the blob craeted events are issued when the creation of the blobs are done. - # This is true ONLY when Hierarchical Namespace is DISABLED - is_hns_enabled = false - - # changing this value is destructive, hence attribute is in lifecycle.ignore_changes block below - infrastructure_encryption_enabled = true - - dynamic "identity" { - for_each = var.enable_cmk_encryption ? [1] : [] - content { - type = "UserAssigned" - identity_ids = [var.encryption_identity_id] - } - } - - dynamic "customer_managed_key" { - for_each = var.enable_cmk_encryption ? [1] : [] - content { - key_vault_key_id = var.encryption_key_versionless_id - user_assigned_identity_id = var.encryption_identity_id - } - } +resource "azurerm_private_dns_zone_virtual_network_link" "airlock_workspace_global" { + name = "airlock-ws-global-corelink" + resource_group_name = var.resource_group_name + private_dns_zone_name = azurerm_private_dns_zone.airlock_workspace_global.name + virtual_network_id = var.core_vnet_id + tags = var.tre_core_tags - tags = merge(var.tre_core_tags, { - description = "airlock;import;blocked" - }) - - network_rules { - default_action = var.enable_local_debugging ? "Allow" : "Deny" - bypass = ["AzureServices"] - } - - lifecycle { ignore_changes = [infrastructure_encryption_enabled, tags] } + lifecycle { ignore_changes = [tags] } } -resource "azurerm_private_endpoint" "stg_import_blocked_pe" { - name = "pe-stg-import-blocked-blob-${var.tre_id}" - location = var.location +resource "azurerm_private_dns_a_record" "airlock_workspace_global" { + name = "@" + zone_name = azurerm_private_dns_zone.airlock_workspace_global.name resource_group_name = var.resource_group_name - subnet_id = var.airlock_storage_subnet_id - - private_dns_zone_group { - name = "pdzg-stg-import-blocked-blob-${var.tre_id}" - private_dns_zone_ids = [var.blob_core_dns_zone_id] - } - - private_service_connection { - name = "psc-stg-import-blocked-blob-${var.tre_id}" - private_connection_resource_id = azurerm_storage_account.sa_import_blocked.id - is_manual_connection = false - subresource_names = ["Blob"] - } - - tags = var.tre_core_tags + ttl = 10 + records = [azurerm_private_endpoint.stg_airlock_workspace_global_pe_processor.private_service_connection[0].private_ip_address] + tags = var.tre_core_tags lifecycle { ignore_changes = [tags] } } +resource "azurerm_role_assignment" "airlock_workspace_global_blob_data_contributor" { + scope = azurerm_storage_account.sa_airlock_workspace_global.id + role_definition_name = "Storage Blob Data Contributor" + principal_id = azurerm_user_assigned_identity.airlock_id.principal_id +} + +# Blob access is enforced by workspace ABAC role assignments. +resource "azurerm_role_assignment" "api_workspace_global_blob_delegator" { + scope = azurerm_storage_account.sa_airlock_workspace_global.id + role_definition_name = "Storage Blob Delegator" + principal_id = var.api_principal_id +} diff --git a/core/terraform/airlock/storage_accounts_v1.tf b/core/terraform/airlock/storage_accounts_v1.tf new file mode 100644 index 0000000000..c819bb4cdd --- /dev/null +++ b/core/terraform/airlock/storage_accounts_v1.tf @@ -0,0 +1,420 @@ +resource "azurerm_storage_account" "sa_import_external" { + count = var.enable_legacy_airlock ? 1 : 0 + name = local.import_external_storage_name + location = var.location + resource_group_name = var.resource_group_name + account_tier = "Standard" + account_replication_type = "LRS" + table_encryption_key_type = var.enable_cmk_encryption ? "Account" : "Service" + queue_encryption_key_type = var.enable_cmk_encryption ? "Account" : "Service" + cross_tenant_replication_enabled = false + shared_access_key_enabled = false + local_user_enabled = false + allow_nested_items_to_be_public = false + + is_hns_enabled = false + infrastructure_encryption_enabled = true + + dynamic "identity" { + for_each = var.enable_cmk_encryption ? [1] : [] + content { + type = "UserAssigned" + identity_ids = [var.encryption_identity_id] + } + } + + dynamic "customer_managed_key" { + for_each = var.enable_cmk_encryption ? [1] : [] + content { + key_vault_key_id = var.encryption_key_versionless_id + user_assigned_identity_id = var.encryption_identity_id + } + } + + tags = merge(var.tre_core_tags, { + description = "airlock;import;external" + }) + + lifecycle { ignore_changes = [infrastructure_encryption_enabled, tags] } +} + +resource "azurerm_private_endpoint" "stg_import_external_pe" { + count = var.enable_legacy_airlock ? 1 : 0 + name = "pe-stg-import-external-blob-${var.tre_id}" + location = var.location + resource_group_name = var.resource_group_name + subnet_id = var.airlock_storage_subnet_id + tags = var.tre_core_tags + + lifecycle { ignore_changes = [tags] } + + private_dns_zone_group { + name = "pdzg-stg-import-external-blob-${var.tre_id}" + private_dns_zone_ids = [var.blob_core_dns_zone_id] + } + + private_service_connection { + name = "psc-stg-import-external-blob-${var.tre_id}" + private_connection_resource_id = azurerm_storage_account.sa_import_external[0].id + is_manual_connection = false + subresource_names = ["Blob"] + } +} + +resource "azurerm_storage_account" "sa_export_approved" { + count = var.enable_legacy_airlock ? 1 : 0 + name = local.export_approved_storage_name + location = var.location + resource_group_name = var.resource_group_name + account_tier = "Standard" + account_replication_type = "LRS" + table_encryption_key_type = var.enable_cmk_encryption ? "Account" : "Service" + queue_encryption_key_type = var.enable_cmk_encryption ? "Account" : "Service" + cross_tenant_replication_enabled = false + shared_access_key_enabled = false + local_user_enabled = false + allow_nested_items_to_be_public = false + + is_hns_enabled = false + infrastructure_encryption_enabled = true + + dynamic "identity" { + for_each = var.enable_cmk_encryption ? [1] : [] + content { + type = "UserAssigned" + identity_ids = [var.encryption_identity_id] + } + } + + dynamic "customer_managed_key" { + for_each = var.enable_cmk_encryption ? [1] : [] + content { + key_vault_key_id = var.encryption_key_versionless_id + user_assigned_identity_id = var.encryption_identity_id + } + } + + tags = merge(var.tre_core_tags, { + description = "airlock;export;approved" + }) + + lifecycle { ignore_changes = [infrastructure_encryption_enabled, tags] } +} + +resource "azurerm_private_endpoint" "stg_export_approved_pe" { + count = var.enable_legacy_airlock ? 1 : 0 + name = "pe-stg-export-approved-blob-${var.tre_id}" + location = var.location + resource_group_name = var.resource_group_name + subnet_id = var.airlock_storage_subnet_id + tags = var.tre_core_tags + + lifecycle { ignore_changes = [tags] } + + private_dns_zone_group { + name = "pdzg-stg-export-approved-blob-${var.tre_id}" + private_dns_zone_ids = [var.blob_core_dns_zone_id] + } + + private_service_connection { + name = "psc-stg-export-approved-blob-${var.tre_id}" + private_connection_resource_id = azurerm_storage_account.sa_export_approved[0].id + is_manual_connection = false + subresource_names = ["Blob"] + } +} + +resource "azurerm_storage_account" "sa_import_in_progress" { + count = var.enable_legacy_airlock ? 1 : 0 + name = local.import_in_progress_storage_name + location = var.location + resource_group_name = var.resource_group_name + account_tier = "Standard" + account_replication_type = "LRS" + table_encryption_key_type = var.enable_cmk_encryption ? "Account" : "Service" + queue_encryption_key_type = var.enable_cmk_encryption ? "Account" : "Service" + allow_nested_items_to_be_public = false + cross_tenant_replication_enabled = false + shared_access_key_enabled = false + local_user_enabled = false + + is_hns_enabled = false + infrastructure_encryption_enabled = true + + dynamic "identity" { + for_each = var.enable_cmk_encryption ? [1] : [] + content { + type = "UserAssigned" + identity_ids = [var.encryption_identity_id] + } + } + + dynamic "customer_managed_key" { + for_each = var.enable_cmk_encryption ? [1] : [] + content { + key_vault_key_id = var.encryption_key_versionless_id + user_assigned_identity_id = var.encryption_identity_id + } + } + + tags = merge(var.tre_core_tags, { + description = "airlock;import;in-progress" + }) + + network_rules { + default_action = var.enable_local_debugging ? "Allow" : "Deny" + bypass = ["AzureServices"] + } + + lifecycle { ignore_changes = [infrastructure_encryption_enabled, tags] } +} + +resource "azapi_resource_action" "enable_defender_for_storage" { + count = var.enable_legacy_airlock && var.enable_malware_scanning ? 1 : 0 + type = "Microsoft.Security/defenderForStorageSettings@2022-12-01-preview" + resource_id = "${azurerm_storage_account.sa_import_in_progress[0].id}/providers/Microsoft.Security/defenderForStorageSettings/current" + method = "PUT" + + body = { + properties = { + isEnabled = true + malwareScanning = { + onUpload = { + isEnabled = true + capGBPerMonth = 5000 + }, + scanResultsEventGridTopicResourceId = azurerm_eventgrid_topic.scan_result[0].id + } + sensitiveDataDiscovery = { + isEnabled = false + } + overrideSubscriptionLevelSettings = true + } + } +} + +resource "azurerm_private_endpoint" "stg_import_inprogress_pe" { + count = var.enable_legacy_airlock ? 1 : 0 + name = "pe-stg-import-inprogress-blob-${var.tre_id}" + location = var.location + resource_group_name = var.resource_group_name + subnet_id = var.airlock_storage_subnet_id + tags = var.tre_core_tags + + lifecycle { ignore_changes = [tags] } + + private_dns_zone_group { + name = "pdzg-stg-import-inprogress-blob-${var.tre_id}" + private_dns_zone_ids = [var.blob_core_dns_zone_id] + } + + private_service_connection { + name = "psc-stg-import-inprogress-blob-${var.tre_id}" + private_connection_resource_id = azurerm_storage_account.sa_import_in_progress[0].id + is_manual_connection = false + subresource_names = ["Blob"] + } +} + +resource "azurerm_storage_account" "sa_import_rejected" { + count = var.enable_legacy_airlock ? 1 : 0 + name = local.import_rejected_storage_name + location = var.location + resource_group_name = var.resource_group_name + account_tier = "Standard" + account_replication_type = "LRS" + table_encryption_key_type = var.enable_cmk_encryption ? "Account" : "Service" + queue_encryption_key_type = var.enable_cmk_encryption ? "Account" : "Service" + allow_nested_items_to_be_public = false + cross_tenant_replication_enabled = false + shared_access_key_enabled = false + local_user_enabled = false + + is_hns_enabled = false + infrastructure_encryption_enabled = true + + dynamic "identity" { + for_each = var.enable_cmk_encryption ? [1] : [] + content { + type = "UserAssigned" + identity_ids = [var.encryption_identity_id] + } + } + + dynamic "customer_managed_key" { + for_each = var.enable_cmk_encryption ? [1] : [] + content { + key_vault_key_id = var.encryption_key_versionless_id + user_assigned_identity_id = var.encryption_identity_id + } + } + + tags = merge(var.tre_core_tags, { + description = "airlock;import;rejected" + }) + + network_rules { + default_action = var.enable_local_debugging ? "Allow" : "Deny" + bypass = ["AzureServices"] + } + + lifecycle { ignore_changes = [infrastructure_encryption_enabled, tags] } +} + +resource "azurerm_private_endpoint" "stg_import_rejected_pe" { + count = var.enable_legacy_airlock ? 1 : 0 + name = "pe-stg-import-rejected-blob-${var.tre_id}" + location = var.location + resource_group_name = var.resource_group_name + subnet_id = var.airlock_storage_subnet_id + + private_dns_zone_group { + name = "pdzg-stg-import-rejected-blob-${var.tre_id}" + private_dns_zone_ids = [var.blob_core_dns_zone_id] + } + + private_service_connection { + name = "psc-stg-import-rejected-blob-${var.tre_id}" + private_connection_resource_id = azurerm_storage_account.sa_import_rejected[0].id + is_manual_connection = false + subresource_names = ["Blob"] + } + + tags = var.tre_core_tags + + lifecycle { ignore_changes = [tags] } +} + +resource "azurerm_storage_account" "sa_import_blocked" { + count = var.enable_legacy_airlock ? 1 : 0 + name = local.import_blocked_storage_name + location = var.location + resource_group_name = var.resource_group_name + account_tier = "Standard" + account_replication_type = "LRS" + table_encryption_key_type = var.enable_cmk_encryption ? "Account" : "Service" + queue_encryption_key_type = var.enable_cmk_encryption ? "Account" : "Service" + allow_nested_items_to_be_public = false + cross_tenant_replication_enabled = false + shared_access_key_enabled = false + local_user_enabled = false + + is_hns_enabled = false + infrastructure_encryption_enabled = true + + dynamic "identity" { + for_each = var.enable_cmk_encryption ? [1] : [] + content { + type = "UserAssigned" + identity_ids = [var.encryption_identity_id] + } + } + + dynamic "customer_managed_key" { + for_each = var.enable_cmk_encryption ? [1] : [] + content { + key_vault_key_id = var.encryption_key_versionless_id + user_assigned_identity_id = var.encryption_identity_id + } + } + + tags = merge(var.tre_core_tags, { + description = "airlock;import;blocked" + }) + + network_rules { + default_action = var.enable_local_debugging ? "Allow" : "Deny" + bypass = ["AzureServices"] + } + + lifecycle { ignore_changes = [infrastructure_encryption_enabled, tags] } +} + +resource "azurerm_private_endpoint" "stg_import_blocked_pe" { + count = var.enable_legacy_airlock ? 1 : 0 + name = "pe-stg-import-blocked-blob-${var.tre_id}" + location = var.location + resource_group_name = var.resource_group_name + subnet_id = var.airlock_storage_subnet_id + + private_dns_zone_group { + name = "pdzg-stg-import-blocked-blob-${var.tre_id}" + private_dns_zone_ids = [var.blob_core_dns_zone_id] + } + + private_service_connection { + name = "psc-stg-import-blocked-blob-${var.tre_id}" + private_connection_resource_id = azurerm_storage_account.sa_import_blocked[0].id + is_manual_connection = false + subresource_names = ["Blob"] + } + + tags = var.tre_core_tags + + lifecycle { ignore_changes = [tags] } +} + +resource "azurerm_role_assignment" "airlock_blob_data_contributor" { + count = var.enable_legacy_airlock ? length(local.airlock_sa_blob_data_contributor) : 0 + scope = local.airlock_sa_blob_data_contributor[count.index] + role_definition_name = "Storage Blob Data Contributor" + principal_id = azurerm_user_assigned_identity.airlock_id.principal_id +} + +resource "azurerm_role_assignment" "api_sa_data_contributor" { + count = var.enable_legacy_airlock ? length(local.api_sa_data_contributor) : 0 + scope = local.api_sa_data_contributor[count.index] + role_definition_name = "Storage Blob Data Contributor" + principal_id = var.api_principal_id +} + +# These resources predate the enable_legacy_airlock toggle; preserve their state addresses. +moved { + from = azurerm_storage_account.sa_import_external + to = azurerm_storage_account.sa_import_external[0] +} + +moved { + from = azurerm_private_endpoint.stg_import_external_pe + to = azurerm_private_endpoint.stg_import_external_pe[0] +} + +moved { + from = azurerm_storage_account.sa_export_approved + to = azurerm_storage_account.sa_export_approved[0] +} + +moved { + from = azurerm_private_endpoint.stg_export_approved_pe + to = azurerm_private_endpoint.stg_export_approved_pe[0] +} + +moved { + from = azurerm_storage_account.sa_import_in_progress + to = azurerm_storage_account.sa_import_in_progress[0] +} + +moved { + from = azurerm_private_endpoint.stg_import_inprogress_pe + to = azurerm_private_endpoint.stg_import_inprogress_pe[0] +} + +moved { + from = azurerm_storage_account.sa_import_rejected + to = azurerm_storage_account.sa_import_rejected[0] +} + +moved { + from = azurerm_private_endpoint.stg_import_rejected_pe + to = azurerm_private_endpoint.stg_import_rejected_pe[0] +} + +moved { + from = azurerm_storage_account.sa_import_blocked + to = azurerm_storage_account.sa_import_blocked[0] +} + +moved { + from = azurerm_private_endpoint.stg_import_blocked_pe + to = azurerm_private_endpoint.stg_import_blocked_pe[0] +} diff --git a/core/terraform/airlock/variables.tf b/core/terraform/airlock/variables.tf index 69888118d0..2fe8ca304f 100644 --- a/core/terraform/airlock/variables.tf +++ b/core/terraform/airlock/variables.tf @@ -80,6 +80,10 @@ variable "log_analytics_workspace_id" { variable "blob_core_dns_zone_id" { type = string } +variable "core_vnet_id" { + type = string + description = "Core VNet, linked to the account-specific DNS zone so the airlock processor resolves the shared global workspace storage account to the core private endpoint" +} variable "file_core_dns_zone_id" { type = string } @@ -107,3 +111,9 @@ variable "encryption_key_versionless_id" { type = string description = "Versionless ID of the encryption key in the key vault" } + +variable "enable_legacy_airlock" { + type = bool + default = true + description = "Deploy v1 legacy per-stage airlock storage accounts. Required for workspaces using airlock_version=1." +} diff --git a/core/terraform/api-webapp.tf b/core/terraform/api-webapp.tf index 47afeb83cb..50144dd05e 100644 --- a/core/terraform/api-webapp.tf +++ b/core/terraform/api-webapp.tf @@ -63,6 +63,7 @@ resource "azurerm_linux_web_app" "api" { MICROSOFT_GRAPH_URL = module.terraform_azurerm_environment_configuration.microsoft_graph_endpoint STORAGE_ENDPOINT_SUFFIX = module.terraform_azurerm_environment_configuration.storage_suffix ENABLE_AIRLOCK_EMAIL_CHECK = var.enable_airlock_email_check + ENABLE_LEGACY_AIRLOCK = var.enable_legacy_airlock LOGGING_LEVEL = var.logging_level OTEL_RESOURCE_ATTRIBUTES = "service.name=api,service.version=${local.version}" OTEL_EXPERIMENTAL_RESOURCE_DETECTORS = "azure_app_service" diff --git a/core/terraform/main.tf b/core/terraform/main.tf index b2f9a6f225..9b0039ad01 100644 --- a/core/terraform/main.tf +++ b/core/terraform/main.tf @@ -136,9 +136,9 @@ module "appgateway" { depends_on = [ module.network, + azurerm_private_endpoint.api_private_endpoint, azurerm_key_vault.kv, azurerm_role_assignment.keyvault_deployer_role, - azurerm_private_endpoint.api_private_endpoint, azurerm_key_vault_key.tre_encryption[0] ] } @@ -159,10 +159,12 @@ module "airlock_resources" { airlock_servicebus_fqdn = azurerm_servicebus_namespace.sb.endpoint applicationinsights_connection_string = module.azure_monitor.app_insights_connection_string enable_malware_scanning = var.enable_airlock_malware_scanning + enable_legacy_airlock = var.enable_legacy_airlock arm_environment = var.arm_environment tre_core_tags = local.tre_core_tags log_analytics_workspace_id = module.azure_monitor.log_analytics_workspace_id blob_core_dns_zone_id = module.network.blob_core_dns_zone_id + core_vnet_id = module.network.core_vnet_id file_core_dns_zone_id = module.network.file_core_dns_zone_id queue_core_dns_zone_id = module.network.queue_core_dns_zone_id table_core_dns_zone_id = module.network.table_core_dns_zone_id @@ -218,6 +220,7 @@ module "resource_processor_vmss_porter" { auto_grant_workspace_consent = var.auto_grant_workspace_consent enable_airlock_malware_scanning = var.enable_airlock_malware_scanning airlock_malware_scan_result_topic_name = module.airlock_resources.airlock_malware_scan_result_topic_name + enable_legacy_airlock = var.enable_legacy_airlock firewall_policy_id = module.firewall.firewall_policy_id depends_on = [ diff --git a/core/terraform/resource_processor/vmss_porter/locals.tf b/core/terraform/resource_processor/vmss_porter/locals.tf index 7c0ec06402..b9da52f0b5 100644 --- a/core/terraform/resource_processor/vmss_porter/locals.tf +++ b/core/terraform/resource_processor/vmss_porter/locals.tf @@ -19,6 +19,7 @@ locals { auto_grant_workspace_consent = var.auto_grant_workspace_consent enable_airlock_malware_scanning = var.enable_airlock_malware_scanning airlock_malware_scan_result_topic_name = var.airlock_malware_scan_result_topic_name + enable_legacy_airlock = var.enable_legacy_airlock core_api_client_id = var.core_api_client_id firewall_policy_id = var.firewall_policy_id }) diff --git a/core/terraform/resource_processor/vmss_porter/variables.tf b/core/terraform/resource_processor/vmss_porter/variables.tf index 342c27b207..ff89256c2a 100644 --- a/core/terraform/resource_processor/vmss_porter/variables.tf +++ b/core/terraform/resource_processor/vmss_porter/variables.tf @@ -115,6 +115,11 @@ variable "airlock_malware_scan_result_topic_name" { description = "Name of the topic to publish Airlock malware scan results to" } +variable "enable_legacy_airlock" { + type = bool + description = "Whether the core legacy (v1) airlock storage accounts are deployed. Propagated to workspace bundles so they only provision legacy connectivity when core does." +} + variable "mgmt_storage_account_id" { type = string description = "ID of the management storage account" diff --git a/core/terraform/variables.tf b/core/terraform/variables.tf index e813751745..36588b2e29 100644 --- a/core/terraform/variables.tf +++ b/core/terraform/variables.tf @@ -174,6 +174,12 @@ variable "enable_airlock_malware_scanning" { description = "If False, Airlock requests will skip the malware scanning stage" } +variable "enable_legacy_airlock" { + type = bool + default = true + description = "Deploy v1 legacy per-stage airlock storage accounts in core. Required for workspaces using airlock_version=1." +} + variable "enable_airlock_email_check" { type = bool default = false diff --git a/core/version.txt b/core/version.txt index 12d80d0630..963aa2b5eb 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.16.17" +__version__ = "0.18.9" diff --git a/docs/azure-tre-overview/airlock-legacy.md b/docs/azure-tre-overview/airlock-legacy.md new file mode 100644 index 0000000000..ad19bee9c4 --- /dev/null +++ b/docs/azure-tre-overview/airlock-legacy.md @@ -0,0 +1,143 @@ +# Legacy Airlock Architecture + +!!! warning "Legacy Architecture" + This page documents the legacy airlock architecture that uses per-stage storage accounts. New deployments should use the current [consolidated architecture](airlock.md). This architecture is maintained for backwards compatibility with existing workspaces. + +## Overview + +The legacy airlock architecture uses **separate storage accounts for each stage** of the airlock process. Data is physically copied between storage accounts as the request progresses through stages. This results in 5 core storage accounts and 5 per-workspace storage accounts (10+ total). + +To use the legacy architecture, explicitly set `airlock_version: 1` in your workspace properties (new workspaces default to `2`) and ensure `enable_legacy_airlock: true` is set in your `config.yaml`. + +!!! note "Airlock v2 works with both auth modes" + Airlock v2 (consolidated storage) uses a per-workspace Entra **app registration** as its SAS signer. This signer is created for **every** workspace, including those using manual authentication (`auth_type: Manual`): + it is always created and owned by TRE and needs no directory reads, consent grants or role assignments, so the Application Administrator's `Application.ReadWrite.OwnedBy` permission (granted unconditionally) is sufficient. Manual-auth workspaces therefore default to and fully support `airlock_version: 2`. + +## Storage Accounts + +### Core (TRE-level) + +| Storage Account | Name Pattern | Description | Network Access | +| --- | --- | --- | --- | +| `stalimex` | `stalimex{tre_id}` | Import external — initial upload location | Public (SAS token) | +| `stalimip` | `stalimip{tre_id}` | Import in-progress — during review | TRE Core VNet | +| `stalimrej` | `stalimrej{tre_id}` | Import rejected | TRE Core VNet | +| `stalimblocked` | `stalimblocked{tre_id}` | Import blocked by scan | TRE Core VNet | +| `stalexapp` | `stalexapp{tre_id}` | Export approved — final export location | Public (SAS token) | + +### Workspace-level + +| Storage Account | Name Pattern | Description | Network Access | +| --- | --- | --- | --- | +| `stalimappws` | `stalimappws{short_ws_id}` | Import approved — final import location | Workspace VNet | +| `stalexintws` | `stalexintws{short_ws_id}` | Export internal — initial export upload | Workspace VNet | +| `stalexipws` | `stalexipws{short_ws_id}` | Export in-progress — during review | Workspace VNet | +| `stalexrejws` | `stalexrejws{short_ws_id}` | Export rejected | Workspace VNet | +| `stalexblockedws` | `stalexblockedws{short_ws_id}` | Export blocked by scan | Workspace VNet | + +> Each workspace gets its own set of 5 storage accounts, leading to significant resource proliferation as the number of workspaces grows. + +## Data Flow + +In the legacy architecture, data is **copied between storage accounts** at each stage transition. A typical import request involves up to 3 copies: + +1. External → In-progress (on submit) +2. In-progress → Blocked (if scan fails) OR stay in In-progress (if clean) +3. In-progress → Approved (on approval) OR In-progress → Rejected (on rejection) + +```mermaid +graph LR + subgraph TRE["TRE Core"] + A["stalimex\nimport external"]-->|"Copy on submit"| B + B["stalimip\nimport in-progress"]-->|"Copy if blocked"| D["stalimblocked\nimport blocked"] + B-->|"No issues"| review{"Manual\nApproval"} + review-->|"Copy on reject"| C["stalimrej\nimport rejected"] + end + subgraph Workspace["TRE Workspace"] + review-->|"Copy on approve"| E["stalimappws\nimport approved"] + end + subgraph External["External"] + data("Data to import")-->A + end +``` +> Legacy import data flow — data is copied at each stage transition. + +```mermaid +graph LR + subgraph Workspace["TRE Workspace"] + data("Data to export")-->A + A["stalexintws\nexport internal"]-->|"Copy on submit"| B + B["stalexipws\nexport in-progress"]-->|"Copy if blocked"| D["stalexblockedws\nexport blocked"] + B-->|"No issues"| review{"Manual\nApproval"} + review-->|"Copy on reject"| C["stalexrejws\nexport rejected"] + end + subgraph External["External"] + review-->|"Copy on approve"| E["stalexapp\nexport approved"] + end +``` +> Legacy export data flow — data is copied at each stage transition. + +## Network Architecture + +In the legacy architecture, each storage account has its own network configuration: + +- **External accounts** (`stalimex`, `stalexapp`): Not bound to any VNet, accessible via SAS token through the internet. +- **Core internal accounts** (`stalimip`, `stalimrej`, `stalimblocked`): Bound to the TRE Core VNet. +- **Workspace accounts** (`stalimappws`, `stalexintws`, `stalexipws`, `stalexrejws`, `stalexblockedws`): Bound to the workspace VNet. + +Each storage account has its own private endpoints, EventGrid system topics, and role assignments. + +[![Legacy airlock networking](../assets/airlock-networking.png)](../assets/airlock-networking.png) + +## Airlock Flow + +The following diagram shows the legacy airlock flow with data copies between storage accounts: + +[![Legacy airlock flow](../assets/airlock-swimlanes.png)](../assets/airlock-swimlanes.png) + +## Comparison with Current Architecture + +| Aspect | Current (Consolidated) | Legacy (Per-Stage) | +| --- | --- | --- | +| **Storage accounts** | 2 total | 10+ (5 core + 5 per workspace) | +| **Stage tracking** | Container metadata | Separate storage accounts | +| **Data copies per request** | 1 (on approval only) | Up to 3 | +| **Workspace isolation** | ABAC + shared PE | Dedicated storage per workspace | +| **Private endpoints** | 2 core + 1 per workspace | 5 core + 5 per workspace | +| **EventGrid topics** | 2 system topics | 10+ system topics | +| **Infrastructure cost** | Lower | Higher (more resources) | +| **Stage transition speed** | Near-instant (metadata) | Minutes (data copy) | +| **Scalability** | All workspaces share storage | Linear growth per workspace | + +## Upgrading to Current Architecture + +To upgrade a workspace from the legacy architecture: + +1. Ensure core is deployed with the current codebase (`enable_legacy_airlock: true` to keep legacy infrastructure alongside the new accounts), and upgrade the workspace to the current `tre-workspace-base` version. That upgrade is a non-destructive **minor** bump: the workspace stays on `airlock_version=1` (no v2 infrastructure is + deployed, no data moves) while the v2 airlock module becomes available for the switch below. +2. Let any in-flight requests complete or cancel them: the API rejects an `airlock_version` change while the workspace holds requests that are still in progress (HTTP 400). Requests in a final state do not block the change. +3. Update the workspace `airlock_version` property to `2`. +4. Redeploy the workspace — this switches from the legacy airlock terraform module to the consolidated module. +5. New airlock requests will use the consolidated storage accounts. +6. Once all workspaces are migrated and no legacy requests are in-flight, set `enable_legacy_airlock: false` in `config.yaml` and redeploy core to remove the legacy storage accounts. + +!!! note + Each request has `airlock_version` stamped at creation time, so a request always keeps using the storage it was created against. A workspace cannot be switched to v2 while any of its requests are still in progress, so drain them before changing the version. + +!!! warning + Upgrading a workspace from `airlock_version=1` to `2` **permanently deletes that workspace's per-stage + v1 airlock storage accounts**, including the data of already-completed requests (approved imports, and + retained rejected/blocked/export data). The request records themselves remain in the API — their + metadata, status and history are preserved — but their stored files are gone and any download link will + no longer resolve. Only upgrade a workspace once you no longer need the data held by its completed v1 + requests. Downgrading from `2` to `1` is not supported and is rejected by the API. + +!!! warning + Setting `enable_legacy_airlock: false` and redeploying core **permanently deletes the core v1 airlock storage accounts** (`stalimex`, `stalimip`, `stalimrej`, `stalimblocked`, `stalexapp`) and any data still in them. Terraform destroys them in the same apply that switches the API to v2-only. + +!!! warning + Before disabling, ensure no active `airlock_version=1` workspaces or in-flight v1 requests remain. + Run `POST /migrations` after upgrading the API so pre-v2 workspaces are stamped with an explicit `1`; + otherwise their next redeploy would migrate them to v2 (new workspaces default to `airlock_version=2`) + and destroy their legacy storage. Switching a workspace's `airlock_version` while it holds in-progress + requests is blocked by the API (returns HTTP 400). diff --git a/docs/azure-tre-overview/airlock.md b/docs/azure-tre-overview/airlock.md index f4673ea279..f0dd307fea 100644 --- a/docs/azure-tre-overview/airlock.md +++ b/docs/azure-tre-overview/airlock.md @@ -2,49 +2,124 @@ In a Trusted Research Environment (TRE) the workspaces represent a security boundary that enables researchers to access data, execute analysis, apply algorithms and collect reports. The airlock capability is the only mechanism that allows users to `import` or `export` data, tools or other file based artefacts in a secure fashion with a human approval. This constitutes the mechanism focused on preventing data exfiltration and securing TRE and its workspaces from inappropriate data, while allowing researchers to work on their projects and execute their tasks. -The airlock feature brings several actions: ingress/egress Mechanism; Data movement; Security gates; Approval mechanism and Notifications. As part of TRE's Safe settings all activity must be tracked for auditing purposes. +The airlock feature brings several actions: ingress/egress mechanism, data movement, security gates, approval mechanism and notifications. As part of TRE's Safe Settings all activity must be tracked for auditing purposes. The Airlock feature aims to address these goals: * Prevent unauthorised data import or export. - * Provide a process to allow approved data to be imported through the security boundary of a TRE Workspace. +* Track requests and decisions, supporting cycles of revision, approval or rejection. +* Automatically scan data being imported for security issues. +* Require manual review by the Airlock Manager for data being exported or imported. +* Notify the requesting researcher of progress and required actions. +* Audit all steps within the airlock process. -* TRE provides functionality to track requests and decisions, supporting cycles of revision, approval or rejection. +Typically in a TRE, the Airlock feature would be used to allow a researcher to export the outputs of a research project such as summary results. With the airlock, data to be exported must go through a human review, typically undertaken by a data governance team. -* Data being imported with an airlock import process can be automatically scanned for security issues. +The Airlock feature creates events on every meaningful step of the process, enabling organisations to extend the notification mechanism. -* Data being exported or imported must be manually reviewed by the Airlock Manager. +## Storage Architecture -* Notify the requesting researcher of the process progress and/or required actions. +The airlock uses a consolidated storage architecture with **2 storage accounts** and metadata-based stage management. Each airlock request gets a dedicated container (named with the request ID), and the request's stage is tracked via container metadata rather than by copying data between storage accounts. -* All steps within the airlock process are audited. +```mermaid +graph TB + subgraph External["External"] + researcher["fa:fa-user Researcher"] + reviewer["fa:fa-user-shield Airlock Manager"] + end + + appgw["fa:fa-shield-alt App Gateway"] + + subgraph Core["TRE Core"] + direction TB + subgraph CoreStorage["Core: stalairlock"] + ie{{"stage: import-external"}} + eapp{{"stage: export-approved"}} + iip{{"stage: import-in-progress"}} + irej{{"stage: import-rejected"}} + iblk{{"stage: import-blocked"}} + end + processor["fa:fa-cog Airlock Processor"] + end + + subgraph WSStorage["Workspace: stalairlockg"] + iappr{{"stage: import-approved"}} + eint{{"stage: export-internal"}} + eip{{"stage: export-in-progress"}} + erej{{"stage: export-rejected"}} + eblk{{"stage: export-blocked"}} + end + + subgraph Workspace["TRE Workspace"] + vm["fa:fa-desktop Researcher VM"] + end + + researcher -- "SAS token" --> appgw + reviewer -- "SAS token" --> appgw + appgw -- "Public stages only" --> CoreStorage + processor -. "All stages" .-> CoreStorage + processor -. "All stages" .-> WSStorage + vm -- "Private Endpoint" --> WSStorage + + style Core fill:#1a3d6d,stroke:#0d2240,color:#fff + style CoreStorage fill:#2c5f9e,stroke:#1a3d6d,color:#fff + style WSStorage fill:#8b5c00,stroke:#5c3d00,color:#fff + style External fill:#444,stroke:#333,color:#fff + style Workspace fill:#1a5c1a,stroke:#0d330d,color:#fff + style appgw fill:#0078d4,stroke:#005a9e,color:#fff + style processor fill:#cc7000,stroke:#995300,color:#fff + style vm fill:#2d8a2d,stroke:#1a5c1a,color:#fff + style ie fill:#2d8a2d,stroke:#1a5c1a,color:#fff + style eapp fill:#2d8a2d,stroke:#1a5c1a,color:#fff + style iip fill:#4a6fa5,stroke:#2c5f9e,color:#fff + style irej fill:#b85450,stroke:#8b3e3b,color:#fff + style iblk fill:#b85450,stroke:#8b3e3b,color:#fff + style iappr fill:#2d8a2d,stroke:#1a5c1a,color:#fff + style eint fill:#2d8a2d,stroke:#1a5c1a,color:#fff + style eip fill:#8b7800,stroke:#5c5000,color:#fff + style erej fill:#b85450,stroke:#8b3e3b,color:#fff + style eblk fill:#b85450,stroke:#8b3e3b,color:#fff + style researcher fill:#0078d4,stroke:#005a9e,color:#fff + style reviewer fill:#0078d4,stroke:#005a9e,color:#fff +``` +> Airlock architecture overview. Hexagon shapes represent container metadata stages. Green = user-accessible, yellow = processing, red = terminal. -Typically in a TRE, the Airlock feature would be used to allow a researcher to export the outputs of a research project such as summary results. With the airlock, data to be exported must go through a human review, typically undertaken by a data governance team. +**Storage Accounts:** -The Airlock feature will create events on every meaningful step of the process. This will enable increased flexibility by allowing an organization to extend the notification mechanism. +| Storage Account | Name Pattern | Purpose | +| --- | --- | --- | +| **Core Storage** | `stalairlock{tre_id}` | All core-managed stages: import external, in-progress, rejected, blocked; export approved | +| **Global Workspace Storage** | `stalairlockg{tre_id}` | All workspace-managed stages: import approved; export internal, in-progress, rejected, blocked | + +**Key design principles:** + +- **Metadata over movement** — In-account stage transitions (submitted→in-review, in-review→rejected/blocked) simply update container metadata, providing near-instant transitions. Data is physically copied in only two places: when a request is **submitted** the writable draft container is *sealed* (its file is copied into an immutable + request-id container and the draft is deleted, structurally revoking the researcher's SAS), and again on **approval** when data crosses the core/workspace boundary. +- **ABAC security** — Azure Attribute-Based Access Control conditions restrict which stages each identity can access on the storage account, enforced at the Azure RBAC layer. +- **Shared infrastructure** — All workspaces share the same workspace storage account, with network isolation via per-workspace private endpoints and ABAC conditions filtering by `workspace_id`. ## Ingress/Egress Mechanism The Airlock allows a TRE user to start the `import` or `export` process to a given workspace. A number of milestones must be reached in order to complete a successful import or export. These milestones are defined using the following states: -1. **Draft**: An Airlock request has been created but has not yet started. The TRE User/Researcher has now access to a storage location and they must identify the data to be processed. At this point the airlock import/export processes allow a single file to be processed. However a compressed file may be used (zip). +1. **Draft**: An Airlock request has been created but has not yet started. The TRE User/Researcher has access to a storage container and must upload the data to be processed. At this point the airlock import/export processes allow a single file to be processed. However a compressed file may be used (zip). 2. **Submitted**: The request was submitted by the researcher (not yet processed). 3. **In-Review**: The request is ready to be reviewed. This state can be reached directly from Submitted state or after going through a successful security scan (found clean). 4. **Approval In-progress**: The Airlock request has been approved, however data movement is still ongoing. -5. **Approved**: The Airlock request has been approved. At this state, data has been securely verified and manually reviewed. The data is now in its final location. For an import process the data is now available in the TRE workspace, it can be accessed by the requestor from within the workspace. +5. **Approved**: The Airlock request has been approved. Data has been securely verified and manually reviewed. The data is now in its final location. For an import process the data is available in the TRE workspace and can be accessed by the requestor from within the workspace. 6. **Rejection In-progress**: The Airlock request has been rejected, however data movement is still ongoing. -7. **Rejected**: The Airlock request has been rejected. The data in the process was rejected manually by the Airlock Manager. -8. **Cancelled**: The Airlock request was manually cancelled by the requestor TRE user, a Workspace owner or a TRE administrator. The cancellation is only allowed when the request is not actively changing (i.e. **Draft** or **In-Review** state). +7. **Rejected**: The Airlock request has been rejected. The data was rejected manually by the Airlock Manager. +8. **Cancelled**: The Airlock request was manually cancelled by the requestor, a Workspace Owner, or a TRE administrator. Cancellation is allowed while the request is not actively moving data — that is, in **Draft** or **In-Review** state. 9. **Blocking In-progress**: The Airlock request has been blocked, however data movement is still ongoing. 10. **Blocked By Scan**: The Airlock request has been blocked. The security analysis found issues in the submitted data and consequently quarantined the data. ```mermaid -graph TD - A[Researcher wants to export data from TRE Workspace] -->|Request created| B[Request in state Draft] +graph TD + A[Researcher wants to export data from TRE Workspace] -->|Request created| B[Request in state Draft] B-->|Researcher gets link to storage container and uploads data| B B-->|Request submitted| C[Submitted] - C--> D{Security issues found?} + C--> D{Security issues found?} D-->|Yes| E[Blocking In-progress] D-->|No| G[In-Review] E:::temporary--> F((Blocked By Scan)) @@ -58,148 +133,460 @@ graph TD H-->|Request Canceled| X classDef temporary stroke-dasharray: 5 5 ``` -> Airlock state flow diagram for an Airlock export request +> Airlock state flow diagram for an export request. Import follows the same flow. + +When an airlock process is created the initial state is **Draft** and the airlock processor creates a storage container with the appropriate stage metadata. The user receives a link to this container (URL + SAS token) that they can use to upload data. + +For import, the container is created in core storage (`stalairlock`) with metadata `stage=import-external`. For export, the container is created in global workspace storage (`stalairlockg`) with metadata `stage=export-internal`, accessible only from within the workspace via private endpoint. -When an airlock process is created the initial state is **Draft** and the required infrastructure will get created providing a single container to isolate the data in the request. Once completed, the user will be able to get a link for this container inside the storage account (URL + SAS token) that they can use to upload the desired data to be processed (import or export). +The user uploads a file using any tool of their preference: [Azure Storage Explorer](https://azure.microsoft.com/en-us/features/storage-explorer/) or [AzCopy](https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-v10). -This storage location is external for import (`stalimex`) or internal for export (`stalexint`), however only accessible to the requestor (ex: a TRE user/researcher). -The user will be able to upload a file to the provided storage location, using any tool of their preference: [Azure Storage Explorer](https://azure.microsoft.com/en-us/features/storage-explorer/) or [AzCopy](https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-v10) which is a command line tool. +The user submits the request (TRE API call). On submission the processor **seals** the draft: it copies the draft container's file into a new, immutable container named after the request ID (with `stage=import-in-progress` / `export-in-progress`) and deletes the writable draft container (`-draft`). This structurally revokes +the researcher's upload SAS, so the data under review can no longer be modified. The airlock request is now in state **Submitted**. -The user Submits the request (TRE API call) starting the data movement (to the `stalimip` - import in-progress or `stalexip` - export in-progress). The airlock request is now in state **Submitted**. -If enabled, the Malware Scanning is started. The scan is done using Microsoft Defender for Storage, which is described in detail in the [Microsoft Defender for Storage documentation](https://learn.microsoft.com/en-us/azure/defender-for-cloud/defender-for-storage-introduction). -In the case that security flaws are found, the request state becomes **Blocking In-progress** while the data is moved to blocked storage (either import blocked `stalimblocked` or export blocked `stalexblocked`). In this case, the request is finalized with the state **Blocked By Scan**. -If the Security Scanning does not identify any security flaws, the request state becomes **In-Review**. Simultaneously, a notification is sent to the Airlock Manager user. The user needs to ask for the container URL using the TRE API (SAS token + URL with READ permission). +If enabled, malware scanning is started using Microsoft Defender for Storage +(see [Microsoft Defender for Storage documentation](https://learn.microsoft.com/en-us/azure/defender-for-cloud/defender-for-storage-introduction)). +If security flaws are found, the container metadata is updated to blocked status and the request is finalised with state **Blocked By Scan**. +If no issues are found, the metadata is updated to in-review status and the request state becomes **In-Review**. +A notification is sent to the Airlock Manager. > The Security Scanning can be disabled, changing the request state from **Submitted** straight to **In-Review**. -The Airlock Manager will manually review the data using the tools of their choice available in the TRE workspace. Once review is completed, the Airlock Manager will have to *Approve* or *Reject* the airlock process, through a TRE API call. -At this point, the request will change state to either **Approval In-progress** or **Rejection In-progress**, while the data movement occurs moving afterwards to **Approved** or **Rejected** accordingly. The data will now be in the final storage destination: `stalexapp` - export approved or `stalimapp` - import approved. -With this state change, a notification will be triggered to the requestor including the location of the processed data in the form of an URL + SAS token. +The Airlock Manager manually reviews the data using tools available in the TRE workspace. Once review is completed, the Airlock Manager approves or rejects the request through a TRE API call. For approval, data is copied to the final destination. For rejection, only metadata is updated. -## Data movement +## Data Movement -For any airlock process, there is data movement either **into** a TRE workspace (in import process) or **from** a TRE workspace (in export process). Being a TRE Workspace boundary, there are networking configurations designed to achieve this goal. The data movement will guarantee that the data is automatically verified for security flaws and manually reviewed, before placing data inside the TRE Workspace. -Also, the process guarantees that data is not tampered with throughout the process. +For any airlock process, there is data movement either **into** a TRE workspace (import) or **from** a TRE workspace (export). The data movement guarantees that data is automatically verified for security flaws and manually reviewed before being placed inside or taken outside the TRE Workspace. -In an import process, data will transition from more public locations (yet confined to the requestor) to TRE workspace storage, after guaranteeing security automatically and by manual review. +**Metadata-based stage management** means most transitions are near-instantaneous metadata updates. Data is physically copied in only two places: -In an export process, data will transition from internal locations (available to the requestor) to public locations in the TRE, after going through a manual review. +- **On submit (seal)**: the writable draft container (`-draft`) is copied into a new immutable `` container on the *same* account and the draft is deleted. This revokes the researcher's upload SAS structurally, so content under review cannot be altered. +- **On approval (boundary crossing)**: Core storage → Workspace storage for imports, Workspace storage → Core storage for exports (one server-side copy). -Considering that the Airlock requests may require large data movements, the operations can have longer durations, hence becoming the operations asynchronous. This is why states like **Approval In-progress**, **Rejection In-progress** or **Blocking In-progress** will be set while there are data movement operations. +All other transitions — submitted→in-review, in-review→rejected/blocked — update metadata only, with no data movement. -> The data movement mechanism is data-driven, allowing an organization to extend how request data transitions between +The cross-account approval copy is driven by the `StatusChangedQueueTrigger`: it creates the destination container (with stage metadata) and starts a server-side copy with `start_copy_from_url`, waiting for it to finish (and aborting, so the Service Bus message retries, if it exceeds the copy timeout). Completion is signalled by a +`BlobCreated` Event Grid event on the destination account: the `BlobCreatedTrigger` maps the destination approval stage to the completed step (`V2_STAGE_COMPLETION_MAP`) and emits the `StepResult` that advances the request to **Approved**, plus a data-deletion event for the source container. + +### Import Data Flow + +```mermaid +graph LR + subgraph External["External"] + data("fa:fa-file Data to import") + end + + subgraph CoreStorage["Core: stalairlock"] + A{{"stage: import-external"}} + B{{"stage: import-in-progress"}} + D{{"stage: import-blocked"}} + C{{"stage: import-rejected"}} + end + + subgraph WorkspaceStorage["Workspace: stalairlockg"] + E{{"stage: import-approved"}} + end + + data -- "Upload via SAS" --> A + A == "Submitted - seal (copy + delete draft)" ==> B + B -. "Threat found - metadata only" .-> D + B -. "Clean scan - metadata only" .-> review{"Review"} + review -. "Rejected - metadata only" .-> C + review == "Approved - cross-account copy" ==> E + + style External fill:#444,stroke:#333,color:#fff + style CoreStorage fill:#2c5f9e,stroke:#1a3d6d,color:#fff + style WorkspaceStorage fill:#8b5c00,stroke:#5c3d00,color:#fff + style data fill:#0078d4,stroke:#005a9e,color:#fff + style A fill:#2d8a2d,stroke:#1a5c1a,color:#fff + style B fill:#4a6fa5,stroke:#2c5f9e,color:#fff + style C fill:#b85450,stroke:#8b3e3b,color:#fff + style D fill:#b85450,stroke:#8b3e3b,color:#fff + style E fill:#2d8a2d,stroke:#1a5c1a,color:#fff + style review fill:#6b5900,stroke:#4a3d00,color:#fff +``` +> Import data flow. Dashed lines = metadata-only transitions. Thick lines = data copies: the submit *seal* (same-account, deletes the writable draft) and the cross-account copy on approval. Hexagons = container metadata stages. + +### Export Data Flow + +```mermaid +graph LR + subgraph Workspace["TRE Workspace"] + data("fa:fa-file Data to export") + end + + subgraph WorkspaceStorage["Workspace: stalairlockg"] + A{{"stage: export-internal"}} + B{{"stage: export-in-progress"}} + D{{"stage: export-blocked"}} + C{{"stage: export-rejected"}} + end + + subgraph CoreStorage["Core: stalairlock"] + E{{"stage: export-approved"}} + end + + data -- "Upload via PE" --> A + A == "Submitted - seal (copy + delete draft)" ==> B + B -. "Threat found - metadata only" .-> D + B -. "Clean scan - metadata only" .-> review{"Review"} + review -. "Rejected - metadata only" .-> C + review == "Approved - cross-account copy" ==> E + + style Workspace fill:#1a5c1a,stroke:#0d330d,color:#fff + style WorkspaceStorage fill:#8b5c00,stroke:#5c3d00,color:#fff + style CoreStorage fill:#2c5f9e,stroke:#1a3d6d,color:#fff + style data fill:#2d8a2d,stroke:#1a5c1a,color:#fff + style A fill:#2d8a2d,stroke:#1a5c1a,color:#fff + style B fill:#4a6fa5,stroke:#2c5f9e,color:#fff + style C fill:#b85450,stroke:#8b3e3b,color:#fff + style D fill:#b85450,stroke:#8b3e3b,color:#fff + style E fill:#2d8a2d,stroke:#1a5c1a,color:#fff + style review fill:#6b5900,stroke:#4a3d00,color:#fff +``` +> Export data flow. Dashed lines = metadata-only transitions. Thick lines = data copies: the submit *seal* (same-account, deletes the writable draft) and the cross-account copy on approval. Hexagons = container metadata stages. ## Security Scan -The identified data in an airlock process, will be submitted to a security scan. If the security scan identifies issues the data is quarantined and a report is added to the process metadata. Both the requestor and Workspace Owner are notified. For a successful security scan, the data will remain in state **In-progress**, and accessible to the Workspace Owner. +Data in an airlock process is submitted to a security scan. If the scan identifies issues, the container metadata is updated to blocked status and a report is added to the process metadata. Both the requestor and Workspace Owner are notified. For a successful security scan, data remains accessible to the Workspace Owner for review. + +> * The security scan is optional, behind a feature flag enabled by a script. +> * The outcome of the security scan will be either the in-progress metadata status or blocked metadata status. +> * An airlock process guarantees that the content being imported/exported is secure. + +## Access Control + +The airlock uses Azure Attribute-Based Access Control (ABAC) to restrict access at the storage account level. This ensures that identities can only access containers matching specific stage metadata values. + +```mermaid +graph LR + api["fa:fa-key TRE API"] + proc["fa:fa-cog Airlock Processor"] + wspe["fa:fa-lock Workspace PE"] + + subgraph CoreStorage["Core: stalairlock"] + cs_ie{{"stage: import-external"}} + cs_eapp{{"stage: export-approved"}} + cs_iip{{"stage: import-in-progress"}} + cs_irej{{"stage: import-rejected"}} + cs_iblk{{"stage: import-blocked"}} + end + + subgraph WorkspaceStorage["Workspace: stalairlockg"] + ws_iapp{{"stage: import-approved"}} + ws_eint{{"stage: export-internal"}} + ws_eip{{"stage: export-in-progress"}} + ws_erej{{"stage: export-rejected"}} + ws_eblk{{"stage: export-blocked"}} + end + + api -- "ABAC: import-external OR export-approved" --> CoreStorage + proc == "Unrestricted access" ==> CoreStorage + proc == "Unrestricted access" ==> WorkspaceStorage + wspe -- "ABAC: workspace_id + stage" --> WorkspaceStorage + + style api fill:#b85450,stroke:#8b3e3b,color:#fff + style proc fill:#cc7000,stroke:#995300,color:#fff + style wspe fill:#6a3d9a,stroke:#4a2b6d,color:#fff + style CoreStorage fill:#2c5f9e,stroke:#1a3d6d,color:#fff + style WorkspaceStorage fill:#8b5c00,stroke:#5c3d00,color:#fff + style cs_ie fill:#2d8a2d,stroke:#1a5c1a,color:#fff + style cs_eapp fill:#2d8a2d,stroke:#1a5c1a,color:#fff + style cs_iip fill:#4a6fa5,stroke:#2c5f9e,color:#fff + style cs_irej fill:#8b3e3b,stroke:#6b2e2b,color:#fff + style cs_iblk fill:#8b3e3b,stroke:#6b2e2b,color:#fff + style ws_iapp fill:#2d8a2d,stroke:#1a5c1a,color:#fff + style ws_eint fill:#2d8a2d,stroke:#1a5c1a,color:#fff + style ws_eip fill:#4a6fa5,stroke:#2c5f9e,color:#fff + style ws_erej fill:#8b3e3b,stroke:#6b2e2b,color:#fff + style ws_eblk fill:#8b3e3b,stroke:#6b2e2b,color:#fff +``` +> ABAC access control. The API can only access public stages (green). The Processor has full access. Workspace PEs are scoped by workspace_id. -> * The Security scan will be optional, behind a feature flag enabled by a script -> * The outcome of the security scan will be either the in-progress (`stalexip`) storage or blocked (`stalexblocked`) -> * An airlock process will guarantee that the content being imported/exported is secure. It is envisioned that a set of **security gates** are identified to be executed successfully for a process to be approved. +**Identity access summary:** -## Approval mechanism +| Identity | Core Storage | Workspace Storage | ABAC Condition | +| --- | --- | --- | --- | +| TRE API | `Storage Blob Data Contributor` | — | Only `import-external` and `export-approved` stages | +| Airlock Processor | `Storage Blob Data Contributor` | `Storage Blob Data Contributor` | None (unrestricted) | +| Workspace PE | — | `Storage Blob Data Contributor` | `workspace_id` must match + stage restrictions | -The approval mechanism, is bundled with any airlock process, providing a specific way to `approve` or `reject` the data. This mechanism will allow the Airlock Managers to explicitly approve/reject the process, after having access to the data. The Airlock Manager users will be able to execute a manual review on the data using the tools available to them in a review TRE Workspace. -Once this manual review is executed, Airlock Managers can proactively approve or reject the airlock request. +**Network access:** -The only goal of the Approval mechanism is to provide a cycle of revision, approval or rejection while tracking the decision. +- Core storage allows public access for import-external and export-approved stages via SAS tokens (through the App Gateway). +- Global workspace storage uses `Deny` as the default network action. Access is only possible via per-workspace private endpoints from within the workspace VNet. +- The airlock processor has a private endpoint on the airlock storage subnet for internal processing on both accounts. +- User Delegation SAS tokens inherit the ABAC restrictions of the signing identity, so even a valid SAS token cannot access stages outside the identity's ABAC scope. -This mechanism will provide access to the data in the airlock process, and will be able to use a VM in TRE workspace. The data review will be the Airlock Manager responsibility +### Container Metadata Stages -> * It is envisioned that this mechanism to be more flexible and extensible. -> * The `Airlock Manager` is a role defined at the workspace instance level and assigned to identities. Initially, the `Owner` role will be used. +Each container has a `stage` metadata key that tracks the current stage of the airlock request: + +**Core Storage (`stalairlock`):** + +| Stage | Description | Access | +| --- | --- | --- | +| `import-external` | Initial upload location for imports | Public via SAS | +| `import-in-progress` | After submission, during review | Processor only | +| `import-rejected` | Import rejected by reviewer | Processor only | +| `import-blocked` | Import blocked by security scan | Processor only | +| `export-approved` | Final location for approved exports | Public via SAS | + +**Global Workspace Storage (`stalairlockg`):** + +| Stage | Description | Access | +| --- | --- | --- | +| `import-approved` | Final location for approved imports | Workspace PE | +| `export-internal` | Initial upload location for exports | Workspace PE | +| `export-in-progress` | After submission, during review | Processor only | +| `export-rejected` | Export rejected by reviewer | Processor only | +| `export-blocked` | Export blocked by security scan | Processor only | + +## Approval Mechanism + +The approval mechanism is bundled with any airlock process, providing a specific way to `approve` or `reject` the data. Airlock Managers can explicitly approve/reject the process after reviewing the data using tools available in a review TRE Workspace. + +The only goal of the approval mechanism is to provide a cycle of revision, approval or rejection while tracking the decision. + +> * It is envisioned that this mechanism will be more flexible and extensible. +> * The `Airlock Manager` is a role defined at the workspace instance level and assigned to identities. ## Notifications -Throughout the airlock process, the notification mechanism will notify the relevant people of the process. Both the requestor (TRE User/Researcher) and the Workspace Owner will be notified by email of the relevant process events. +Throughout the airlock process, the notification mechanism notifies the relevant people. Both the requestor (TRE User/Researcher) and the Workspace Owner are notified by email of relevant process events. Whenever the airlock process changes to a state of **Draft**, **Submitted**, **Approved**, **Rejected**, **Approval In-progress**, **Rejection In-progress**, **Blocked By Scan** or **Cancelled**, the process requestor gets notified. -When the state changes to `In-progress` the Workspace Owner (Airlock Manager) gets notified. +When the state changes to **In-Review**, the Workspace Owner (Airlock Manager) gets notified. + +> * The notification mechanism is data-driven, allowing an organisation to extend the notifications behaviour. The mechanism is exemplified with a Logic App determining the notifications logic. +> * Notifications work with all TRE users being Microsoft Entra ID users (guests or not), with email defined — if not, notifications will not be sent. -> * The Notification mechanism is also data-driven, allowing an organization to extend the notifications behavior. The mechanism is exemplified with a Logic App determining the notifications logic. -> * Notifications will work with All TRE users being Microsoft Entra ID users (guests or not), with email defined – if not, notifications will not be sent. +## API Endpoints -## Architecture +The TRE API exposes the following airlock endpoints: -The Airlock feature is supported by infrastructure at the TRE and workspace level, containing a set of storage accounts. Each Airlock request will provision and use unique storage containers with the request id in its name. +| Method | Endpoint | Description | +| --- | --- | --- | +| `POST` | `/api/workspaces/{workspace_id}/requests` | Create an Airlock request (in **Draft**) | +| `GET` | `/api/workspaces/{workspace_id}/requests/{airlock_request_id}/link` | Get the url and token to access an Airlock Request | +| `POST` | `/api/workspaces/{workspace_id}/requests/{airlock_request_id}/submit` | Submit an Airlock request | +| `POST` | `/api/workspaces/{workspace_id}/requests/{airlock_request_id}/review` | Review an Airlock request | +| `POST` | `/api/workspaces/{workspace_id}/requests/{airlock_request_id}/cancel` | Cancel an Airlock request | + +## Airlock Processor + +The **Airlock Processor** is a set of Azure Functions that handle the events created throughout the airlock process: + +- **StatusChangedQueueTrigger** — Consumes status change events from the Service Bus queue and orchestrates container creation, sealing, metadata updates, and cross-account data copies. On **submit** it seals the draft (copies `-draft` into an immutable `` container and deletes the draft). For same-account + transitions (in-review→rejected/blocked) it updates container metadata directly. For the cross-account approval copy it creates the destination container and runs a server-side copy, waiting for it to finish (aborting and letting the message retry on timeout). +- **BlobCreatedTrigger** — Fires when a blob appears in a storage account (via EventGrid → Service Bus). For the v1 per-stage flow it signals each stage's copy completion. For v2 it handles **only** the cross-account approval copies (`V2_STAGE_COMPLETION_MAP`): when the copied blob lands in the approved container it emits the `StepResult` + advancing the request to **Approved** and a data-deletion event for the source. Other v2 completions are emitted directly by the StatusChangedQueueTrigger. +- **ScanResultTrigger** — Consumes malware scan results from Microsoft Defender for Storage. If threats are found, emits a StepResult to block the request. If clean, emits a StepResult to advance to in-review. +- **DataDeletionTrigger** — Cleans up source containers after data has been copied to the destination. + +This event-driven design keeps stage transitions responsive: same-account transitions are metadata-only (except the submit seal, a fast same-account copy that deletes the draft), while the less frequent cross-account approval copy is confirmed via a `BlobCreated` event before the request advances to **Approved**. + +## Airlock Flow + +The following sequence diagram details the airlock feature and its event-driven behaviour: ```mermaid -graph LR - subgraph TRE Workspace - E[(stalimapp
import approved)] - end - subgraph TRE - A[(stalimex
import external)]-->|Request Submitted| B - B[(stalimip
import in-progress)]-->|Security issues found| D[(stalimblocked
import blocked)] - B-->|No security issues found| review{Manual
Approval} - review-->|Rejected| C[(stalimrej
import rejected)] - review-->|Approved| E - end - subgraph External - data(Data to import)-->A - end +sequenceDiagram + participant R as Researcher + participant API as TRE API + participant CS as Core Storage
(stalairlock) + participant WS as Workspace Storage
(stalairlockg) + participant AP as Airlock Processor + participant EG as Event Grid + participant SB as Service Bus + participant DB as Cosmos DB + + Note over R,DB: Creating a Draft Request (Import Example) + R->>API: POST /requests (type=import) + API->>DB: Save request (status: draft) + API->>EG: StatusChangedEvent(draft) + EG->>SB: Queue status change + SB->>AP: Consume event + AP->>CS: Create container with metadata stage=import-external + API-->>R: OK + request details + + Note over R,DB: Getting Upload Link + R->>API: GET /requests/{id}/link + API->>CS: Generate User Delegation SAS (ABAC: import-external) + API-->>R: SAS URL for container + + Note over R,DB: Uploading File + R->>CS: Upload file via SAS token + + Note over R,DB: Submitting Request + R->>API: POST /requests/{id}/submit + API->>DB: Update status → submitted + API->>EG: StatusChangedEvent(submitted) + EG->>SB: Queue status change + SB->>AP: StatusChangedQueueTrigger + AP->>CS: Update metadata → import-in-progress + + Note over R,DB: Security Scan + alt Malware Scanning Enabled + CS-->>EG: Defender scan result + EG->>SB: Queue scan result + SB->>AP: ScanResultTrigger + alt Threat Found + AP->>EG: StepResult(blocking_in_progress) + Note over AP,CS: StatusChangedQueueTrigger updates metadata → import-blocked + AP->>EG: StepResult(blocked) + else No Threat + AP->>EG: StepResult(in-review) + end + else Malware Scanning Disabled + AP->>EG: StepResult(submitted → in-review) + end + AP->>DB: Update status → in-review + AP->>EG: NotificationEvent (to reviewer) + + Note over R,DB: Approval (Synchronous Copy) + R->>API: POST /requests/{id}/review (approve) + API->>DB: Update status → approval_in_progress + API->>EG: StatusChangedEvent(approval_in_progress) + EG->>SB: Queue status change + SB->>AP: StatusChangedQueueTrigger + AP->>WS: Create container with metadata stage=import-approved + AP->>WS: Server-side copy from Core → Workspace storage + Note over AP,WS: Trigger waits (polls) for copy to complete + AP->>EG: StepResult(approved) + AP->>DB: Update status → approved + AP->>EG: DataDeletion event (source container) + AP->>EG: NotificationEvent (to researcher) ``` -> Data movement in an Airlock import request -```mermaid -graph LR - subgraph TRE workspace - data(Data to export)-->A - A[(stalexint
export internal)]-->|Request Submitted| B - B[(stalexip
export in-progress)]-->|Security issues found| D[(stalexblocked
export blocked)] - B-->|No security issues found| review{Manual
Approval} - review-->|Rejected| C[(stalexrej
export rejected)] - end - subgraph External - review-->|Approved| E[(stalexapp
export approved)] - end +## Legacy Airlock + +For details on the legacy airlock architecture (per-stage storage accounts) and migration guidance, see [Legacy Airlock Architecture](airlock-legacy.md). + +## Configuration + +### Core Settings (`config.yaml`) + +The following settings in `config.yaml` control the airlock infrastructure at the TRE core level: + +```yaml +# config.yaml +tre_id: mytre + +# Set to false to remove legacy per-stage storage accounts. +# Default: true (keeps legacy accounts for backward compatibility) +enable_legacy_airlock: false +``` + +| Setting | Type | Default | Description | +| --- | --- | --- | --- | +| `enable_legacy_airlock` | bool | `true` | When `true`, deploys legacy per-stage storage accounts alongside the consolidated accounts for backward compatibility. When `false`, only the consolidated accounts (`stalairlock`, `stalairlockg`) are deployed. See [Legacy Airlock Architecture](airlock-legacy.md) for details. | + +The consolidated storage accounts (`stalairlock{tre_id}` and `stalairlockg{tre_id}`) are **always** provisioned regardless of this setting. + +### Workspace Settings + +The airlock is enabled per workspace via the following properties: + +| Property | Type | Default | Values | Description | +| --- | --- | --- | --- | --- | +| `enable_airlock` | bool | `false` | `true` / `false` | Enables or disables the airlock feature for the workspace | +| `airlock_version` | int | `2` | `1` or `2` | `2` = Consolidated metadata-based storage (recommended), `1` = Legacy per-stage storage accounts | + +The `airlock_version` property only appears when `enable_airlock` is set to `true`. + +**Enabling airlock via the API:** + +```json +PATCH /api/workspaces/{workspace_id} +{ + "properties": { + "enable_airlock": true, + "airlock_version": 2 + } +} ``` -> Data movement in an Airlock export request +**Enabling airlock via the UI:** -TRE: +When creating or updating a workspace, the airlock version is available as a dropdown under the airlock configuration section. -* `stalimex` - storage (st) airlock (al) import (im) external (ex) -* `stalimip` - storage (st) airlock (al) import (im) in-progress (ip) -* `stalimrej` - storage (st) airlock (al) import (im) rejected (rej) -* `stalimblocked` - storage (st) airlock (al) import (im) blocked -* `stalexapp` - storage (st) airlock (al) export (ex) approved (app) +### What Happens at Each Level -Workspace: +```text +config.yaml Workspace Properties +┌─────────────────────────┐ ┌─────────────────────────────┐ +│ enable_legacy_airlock: │ │ enable_airlock: true │ +│ false → v2 infra only│ │ airlock_version: 2 → v2 TF │ +└─────────────────────────┘ └─────────────────────────────┘ + Core Terraform Workspace Terraform +``` -* `stalimapp` - workspace storage (st) airlock (al) import (im) approved (app) -* `stalexint` - workspace storage (st) airlock (al) export (ex) internal (int) -* `stalexip` - workspace storage (st) airlock (al) export (ex) in-progress (ip) -* `stalexrej` - workspace storage (st) airlock (al) export (ex) rejected (rej) -* `stalexblocked` - workspace storage (st) airlock (al) export (ex) blocked +- **Core level** (`enable_legacy_airlock`): Controls whether legacy per-stage storage accounts are also deployed (for backward compatibility only) +- **Workspace level** (`airlock_version`): Controls which workspace Terraform module runs — `airlock_v2/` for consolidated storage with ABAC -> * The external storage accounts (`stalimex`, `stalexapp`), are not bound to any vnet and are accessible (with SAS token) via the internet -> * The internal storage account (`stalexint`) is bound to the workspace vnet, so ONLY TRE Users/Researchers on that workspace can access it -> * The (export) in-progress storage account (`stalexip`) is bound to the workspace vnet -> * The (export) blocked storage account (`stalexblocked`) is bound to the workspace vnet -> * The (export) rejected storage account (`stalexrej`) is bound to the workspace vnet -> * The (import) in-progress storage account (`stalimip`) is bound to the TRE CORE vnet -> * The (import) blocked storage account (`stalimblocked`) is bound to the TRE CORE vnet -> * The (import) rejected storage account (`stalimrej`) is bound to the TRE CORE vnet -> * The (import) approved storage account (`stalimapp`) is bound to the workspace vnet +## Cross-Workspace Isolation -[![Airlock networking](../assets/airlock-networking.png)](../assets/airlock-networking.png) +A common question: if all workspaces share the same storage account (`stalairlockg{tre_id}`), what prevents Workspace A from accessing Workspace B's data? -In the TRE Core, the TRE API will provide the airlock API endpoints allowing to advance the process. The TRE API will expose the following methods: +The answer is **three layers of isolation**: -| Method | Endpoint | Description | -| --- | --- | --- | -| `POST` | `/api/workspaces/{workspace_id}/requests` | Create an Airlock request (in **Draft**) | -| `POST` | `/api/workspaces/{workspace_id}/requests/{airlock_request_id}/link` | Get the url and token to access an Airlock Request | -| `POST` | `/api/workspaces/{workspace_id}/requests/{airlock_request_id}/submit` | Submits an Airlock request | -| `POST` | `/api/workspaces/{workspace_id}/requests/{airlock_request_id}/review` | Reviews an Airlock request | -| `POST` | `/api/workspaces/{workspace_id}/requests/{airlock_request_id}/cancel` | Cancels an Airlock request | +### 1. ABAC Conditions (Azure Attribute-Based Access Control) + +Each workspace deployment creates a role assignment on the global workspace storage account with an ABAC condition that requires **all three** of the following to be true for blob operations: +- The request must come through **that workspace's specific private endpoint** +- The container's `workspace_id` metadata must match **that workspace's ID** +- The container's `stage` metadata must be one of the allowed stages (`import-approved`, `export-internal`, `export-in-progress`) + +```text +ABAC condition (per workspace): + @Environment[Microsoft.Network/privateEndpoints] + == '/subscriptions/.../pe-sa-airlock-ws-global-{workspace_short_id}' + AND + @Resource[...containers/metadata:workspace_id] + == '{workspace_id}' + AND + @Resource[...containers/metadata:stage] + IN ('import-approved', 'export-internal', 'export-in-progress') +``` -Also in the airlock feature there is the **Airlock Processor** which handles the events that are created throughout the process, signalling state changes from blobs created, status changed or security scans finalized. +This means even if Workspace A somehow obtained a SAS token referencing Workspace B's container, the ABAC condition would deny the operation because the private endpoint wouldn't match. -## Airlock flow +### 2. Network Isolation (Private Endpoints) -The following sequence diagram detailing the Airlock feature and its event driven behaviour: +Each workspace creates its own private endpoint to the global workspace storage account, connected to the workspace's VNet. The ABAC condition references this specific private endpoint ID, so requests from a different workspace's PE are rejected. -[![Airlock flow](../assets/airlock-swimlanes.png)](../assets/airlock-swimlanes.png) +### 3. Container Metadata + +The airlock processor stamps every container with `workspace_id` metadata at creation time. This metadata is immutable in practice (only the processor identity can modify it, and researcher identities have no direct access to the storage account). + +```mermaid +graph TB + subgraph WS_A["Workspace A"] + pe_a["PE: pe-sa-airlock-ws-global-ab12"] + end + + subgraph WS_B["Workspace B"] + pe_b["PE: pe-sa-airlock-ws-global-cd34"] + end + + subgraph GlobalStorage["Workspace: stalairlockg"] + c1("req-001
workspace_id: ws-ab12
stage: import-approved") + c2("req-002
workspace_id: ws-cd34
stage: export-internal") + end + + pe_a -- "ABAC: ws-ab12 + PE match" --> c1 + pe_a -. "DENIED by ABAC" .-> c2 + pe_b -. "DENIED by ABAC" .-> c1 + pe_b -- "ABAC: ws-cd34 + PE match" --> c2 + + style WS_A fill:#2c5f9e,stroke:#1a3d6d,color:#fff + style WS_B fill:#8b5c00,stroke:#5c3d00,color:#fff + style GlobalStorage fill:#444,stroke:#333,color:#fff + style pe_a fill:#4a6fa5,stroke:#2c5f9e,color:#fff + style pe_b fill:#cc7000,stroke:#995300,color:#fff + style c1 fill:#4a6fa5,stroke:#2c5f9e,color:#fff + style c2 fill:#cc7000,stroke:#995300,color:#fff +``` +> Cross-workspace isolation. Each workspace can only access containers matching its own workspace_id, through its own private endpoint. ABAC enforces both conditions at the Azure RBAC layer. diff --git a/docs/tre-admins/environment-variables.md b/docs/tre-admins/environment-variables.md index c5139b8035..9e203b6a2a 100644 --- a/docs/tre-admins/environment-variables.md +++ b/docs/tre-admins/environment-variables.md @@ -35,6 +35,7 @@ | `STATEFUL_RESOURCES_LOCKED` | If set to `false` locks on stateful resources won't be created. A recommended setting for developers. | | `KV_PURGE_PROTECTION_ENABLED` | If set to `false` the core Key Vault's purge protection will be disabled so it can be reused upon deletion. A recommended setting for developers. | | `ENABLE_AIRLOCK_MALWARE_SCANNING` | If False, Airlock requests will skip the malware scanning stage. If set to True, Defender for Storage will be enabled. | +| `ENABLE_LEGACY_AIRLOCK` | Set to `true` by default. Deploys the legacy per-stage airlock storage accounts alongside the consolidated accounts. Set to `false` only once no `airlock_version=1` workspaces or in-flight v1 requests remain; doing so permanently deletes the legacy accounts and any data still in them. | | `ENABLE_LOCAL_DEBUGGING` | Set to `false` by default. Setting this to `true` will ensure that Azure resources are accessible from your local development machine. (e.g. ServiceBus and Cosmos) | | `PUBLIC_DEPLOYMENT_IP_ADDRESS` | The public IP address of the machine that is deploying TRE. (Your desktop or the build agents). In certain locations a dynamic script to retrieve this from [https://ipecho.net/plain](https://ipecho.net/plain) does not work. If this is the case, then you can 'hardcode' your IP. | | `RESOURCE_PROCESSOR_VMSS_SKU` | The SKU of the VMMS to use for the resource processing VM. | diff --git a/e2e_tests/airlock/request.py b/e2e_tests/airlock/request.py index 67e340f3e4..6dac3c8072 100644 --- a/e2e_tests/airlock/request.py +++ b/e2e_tests/airlock/request.py @@ -64,7 +64,7 @@ async def upload_blob_using_sas(file_path: str, sas_url: str): _, file_ext = os.path.splitext(file_name) blob_url = f"{storage_account_url}{container_name}/{file_name}?{parsed_sas_url.query}" - LOGGER.info(f"uploading [{file_name}] to container [{blob_url}]") + LOGGER.info(f"uploading [{file_name}] to container [{storage_account_url}{container_name}]") client = BlobClient.from_blob_url(blob_url) with open(file_name, 'rb') as data: @@ -73,6 +73,19 @@ async def upload_blob_using_sas(file_path: str, sas_url: str): return response +async def delete_blob_using_sas(file_path: str, sas_url: str): + parsed_sas_url = urlparse(sas_url) + container_name = parsed_sas_url.path.lstrip("/") + storage_account_url = f"{parsed_sas_url.scheme}://{parsed_sas_url.netloc}/" + file_name = os.path.basename(file_path) + + blob_url = f"{storage_account_url}{container_name}/{file_name}?{parsed_sas_url.query}" + LOGGER.info(f"deleting [{file_name}] from container [{storage_account_url}{container_name}]") + + client = BlobClient.from_blob_url(blob_url) + client.delete_blob() + + async def wait_for_status( request_status: str, workspace_owner_token, workspace_path, request_id, verify ): diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py index 227a6caebb..3513e79ebc 100644 --- a/e2e_tests/conftest.py +++ b/e2e_tests/conftest.py @@ -113,11 +113,11 @@ async def clean_up_test_workspace_service(pre_created_workspace_service_id: str, # Session scope isn't in effect with python-xdist: https://github.com/microsoft/AzureTRE/issues/2868 @pytest.fixture(scope="session") -async def setup_test_workspace(verify) -> Tuple[str, str, str]: +async def setup_test_workspace(verify) -> Tuple[str, str]: pre_created_workspace_id = config.TEST_WORKSPACE_ID - # Set up - uses a pre created app reg as has appropriate roles assigned + auth_type = "Manual" if config.TEST_WORKSPACE_APP_ID else "Automatic" workspace_path, workspace_id = await create_or_get_test_workspace( - auth_type="Manual", verify=verify, pre_created_workspace_id=pre_created_workspace_id, client_id=config.TEST_WORKSPACE_APP_ID, client_secret=config.TEST_WORKSPACE_APP_SECRET) + auth_type=auth_type, verify=verify, pre_created_workspace_id=pre_created_workspace_id, client_id=config.TEST_WORKSPACE_APP_ID, client_secret=config.TEST_WORKSPACE_APP_SECRET) yield workspace_path, workspace_id diff --git a/e2e_tests/test_airlock.py b/e2e_tests/test_airlock.py index 051a5c9d81..a5e5c07028 100644 --- a/e2e_tests/test_airlock.py +++ b/e2e_tests/test_airlock.py @@ -2,11 +2,10 @@ import pytest import asyncio import logging - +from httpx import AsyncClient from azure.core.exceptions import ResourceNotFoundError -from azure.storage.blob import ContainerClient -from airlock.request import post_request, get_request, upload_blob_using_sas, wait_for_status +from airlock.request import post_request, get_request, upload_blob_using_sas, delete_blob_using_sas, wait_for_status from resources.resource import get_resource, post_resource from resources.workspace import get_workspace_auth_details from airlock import strings as airlock_strings @@ -18,6 +17,7 @@ LOGGER = logging.getLogger(__name__) BLOB_FILE_PATH = "./test_airlock_sample.txt" BLOB_NAME = os.path.basename(BLOB_FILE_PATH) +SECOND_BLOB_FILE_PATH = "./test_airlock_second_sample.txt" async def submit_airlock_import_request(workspace_path: str, workspace_owner_token: str, verify: bool): @@ -77,6 +77,160 @@ async def submit_airlock_import_request(workspace_path: str, workspace_owner_tok return request_id, container_url +@pytest.mark.timeout(30 * 60) +@pytest.mark.airlock +async def test_draft_container_is_sealed_after_submit(setup_test_workspace, verify): + """A SAS handed out while the request was in Draft must stop working once it is submitted, + otherwise a researcher could alter the data after it has been scanned and reviewed.""" + workspace_path, workspace_id = setup_test_workspace + workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) + + _, container_url = await submit_airlock_import_request(workspace_path, workspace_owner_token, verify) + + # Submission copies the data out and deletes the draft container, so the old SAS resolves to nothing. + with pytest.raises(ResourceNotFoundError): + await upload_blob_using_sas(BLOB_FILE_PATH, container_url) + + +async def create_draft_import_request_with_file(workspace_path: str, workspace_owner_token: str, verify: bool): + payload = { + "type": airlock_strings.IMPORT, + "businessJustification": "some business justification" + } + request_result = await post_request(payload, f'/api{workspace_path}/requests', workspace_owner_token, verify, 201) + request_id = request_result["airlockRequest"]["id"] + + request_result = await get_request(f'/api{workspace_path}/requests/{request_id}/link', workspace_owner_token, verify, 200) + container_url = request_result["containerUrl"] + + # Container creation is asynchronous, so a successful upload is what confirms it exists. + for _ in range(20): + try: + await asyncio.sleep(5) + upload_response = await upload_blob_using_sas(BLOB_FILE_PATH, container_url) + if "etag" in upload_response: + return request_id, container_url + except ResourceNotFoundError: + await asyncio.sleep(30) + raise Exception("Draft container was not created in time") + + +async def submit_and_expect_failure(workspace_path, workspace_owner_token, request_id, verify, expected_message): + await post_request(None, f'/api{workspace_path}/requests/{request_id}/submit', workspace_owner_token, verify, 200) + await wait_for_status(airlock_strings.FAILED_STATUS, workspace_owner_token, workspace_path, request_id, verify) + + request_result = await get_request(f'/api{workspace_path}/requests/{request_id}', workspace_owner_token, verify, 200) + assert expected_message in request_result["airlockRequest"]["statusMessage"] + + +@pytest.mark.timeout(30 * 60) +@pytest.mark.airlock +async def test_submit_with_no_files_is_rejected(setup_test_workspace, verify): + workspace_path, workspace_id = setup_test_workspace + workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) + + request_id, container_url = await create_draft_import_request_with_file(workspace_path, workspace_owner_token, verify) + await delete_blob_using_sas(BLOB_FILE_PATH, container_url) + + await submit_and_expect_failure(workspace_path, workspace_owner_token, request_id, verify, "did not contain any files") + + +@pytest.mark.timeout(30 * 60) +@pytest.mark.airlock +async def test_submit_with_multiple_files_is_rejected(setup_test_workspace, verify): + workspace_path, workspace_id = setup_test_workspace + workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) + + request_id, container_url = await create_draft_import_request_with_file(workspace_path, workspace_owner_token, verify) + + # upload_blob_using_sas names the blob after the file, so a second distinct name is needed. + with open(SECOND_BLOB_FILE_PATH, "w") as second_file: + second_file.write("a second file in the same request") + await upload_blob_using_sas(SECOND_BLOB_FILE_PATH, container_url) + + await submit_and_expect_failure(workspace_path, workspace_owner_token, request_id, verify, "more than 1 file") + + +@pytest.mark.timeout(30 * 60) +@pytest.mark.airlock +async def test_cancelled_request_reaches_terminal_state(setup_test_workspace, verify): + workspace_path, workspace_id = setup_test_workspace + workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) + + request_id, _ = await create_draft_import_request_with_file(workspace_path, workspace_owner_token, verify) + + request_result = await post_request(None, f'/api{workspace_path}/requests/{request_id}/cancel', workspace_owner_token, verify, 200) + assert request_result["airlockRequest"]["status"] == airlock_strings.CANCELLED_STATUS + + +@pytest.mark.timeout(30 * 60) +@pytest.mark.airlock +async def test_rejected_request_reaches_terminal_state(setup_test_workspace, verify): + workspace_path, workspace_id = setup_test_workspace + workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) + + request_id, _ = await submit_airlock_import_request(workspace_path, workspace_owner_token, verify) + + payload = {"approval": "False", "decisionExplanation": "rejected by the e2e test"} + await post_request(payload, f'/api{workspace_path}/requests/{request_id}/review', workspace_owner_token, verify, 200) + await wait_for_status(airlock_strings.REJECTED_STATUS, workspace_owner_token, workspace_path, request_id, verify) + + +@pytest.mark.timeout(30 * 60) +@pytest.mark.airlock +async def test_container_link_is_refused_once_request_leaves_draft(setup_test_workspace, verify): + """Data must be immutable once submitted, so the API should stop handing out a writable link.""" + workspace_path, workspace_id = setup_test_workspace + workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) + + request_id, _ = await submit_airlock_import_request(workspace_path, workspace_owner_token, verify) + + payload = {"approval": "False", "decisionExplanation": "rejected so the data is no longer reachable"} + await post_request(payload, f'/api{workspace_path}/requests/{request_id}/review', workspace_owner_token, verify, 200) + await wait_for_status(airlock_strings.REJECTED_STATUS, workspace_owner_token, workspace_path, request_id, verify) + + await get_request(f'/api{workspace_path}/requests/{request_id}/link', workspace_owner_token, verify, 400) + + +@pytest.mark.timeout(30 * 60) +@pytest.mark.airlock +async def test_client_supplied_airlock_version_is_ignored(setup_test_workspace, verify): + """The storage layout is decided by the workspace, not by the caller.""" + workspace_path, workspace_id = setup_test_workspace + workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) + + workspace = await get_resource(f"/api{workspace_path}", workspace_owner_token, verify) + expected_version = workspace["workspace"]["properties"].get("airlock_version", 1) + + payload = { + "type": airlock_strings.IMPORT, + "businessJustification": "some business justification", + "airlock_version": 1 if expected_version != 1 else 2, + } + request_result = await post_request(payload, f'/api{workspace_path}/requests', workspace_owner_token, verify, 201) + assert request_result["airlockRequest"]["airlock_version"] == expected_version + + +@pytest.mark.timeout(50 * 60) +@pytest.mark.airlock +async def test_in_progress_data_is_not_reachable_from_the_public_internet(setup_test_workspace, verify): + """Submitted data sits on the in-progress account, which is gated on private link, + so a valid SAS for it must still be refused when presented from outside the vnet.""" + workspace_path, workspace_id = setup_test_workspace + workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) + + request_id, _ = await submit_airlock_import_request(workspace_path, workspace_owner_token, verify) + + request_result = await get_request(f'/api{workspace_path}/requests/{request_id}/link', workspace_owner_token, verify, 200) + base, sas = request_result["containerUrl"].split("?") + blob_url = f"{base}/{BLOB_NAME}?{sas}" + + async with AsyncClient(timeout=30.0, verify=verify) as client: + public_response = await client.get(blob_url) + + assert public_response.status_code == 403, f"in-progress data was readable from the public internet: {public_response.status_code}" + + @pytest.mark.timeout(50 * 60) @pytest.mark.airlock async def test_airlock_review_vm_flow(setup_test_workspace, setup_test_airlock_import_review_workspace_and_guacamole_service, verify): @@ -126,6 +280,11 @@ async def test_airlock_review_vm_flow(setup_test_workspace, setup_test_airlock_i # Create a review VM admin_token = await get_admin_token(verify) import_workspace_owner_token, _ = await get_workspace_auth_details(admin_token=admin_token, workspace_id=import_review_workspace_id, verify=verify) + + # The request belongs to the research workspace, so it must not be reachable by id through + # another workspace the caller happens to have access to. + await get_request(f'/api/workspaces/{import_review_workspace_id}/requests/{request_id}', import_workspace_owner_token, verify, 404) + user_resource_path, user_resource_id = await post_resource( payload={}, endpoint=f"/api{workspace_path}/requests/{request_id}/review-user-resource", @@ -156,70 +315,3 @@ async def test_airlock_review_vm_flow(setup_test_workspace, setup_test_airlock_i LOGGER.info("Review VM has started deletion successfully") # EXPORT FLOW - # We can't test teh export flow as we can't fully create an export request without special networking setup - - -@pytest.mark.airlock -@pytest.mark.extended -@pytest.mark.timeout(35 * 60) -async def test_airlock_flow(setup_test_workspace, verify) -> None: - # 1. Get the workspace set up - workspace_path, workspace_id = setup_test_workspace - workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) - - # 2. create and submit airlock request - request_id, container_url = await submit_airlock_import_request(workspace_path, workspace_owner_token, verify) - - # 3. approve request - LOGGER.info("Approving airlock request") - payload = { - "approval": "True", - "decisionExplanation": "the reason why this request was approved/rejected" - } - request_result = await post_request(payload, f'/api{workspace_path}/requests/{request_id}/review', workspace_owner_token, verify, 200) - assert request_result["airlockRequest"]["reviews"][0]["decisionExplanation"] == "the reason why this request was approved/rejected" - - await wait_for_status(airlock_strings.APPROVED_STATUS, workspace_owner_token, workspace_path, request_id, verify) - - # 4. check the file has been deleted from the source - # NOTE: We should really be checking that the file is deleted from in progress location too, - # but doing that will require setting up network access to in-progress storage account - try: - container_client = ContainerClient.from_container_url(container_url=container_url) - # We expect the container to eventually be deleted too, but sometimes this async operation takes some time. - # Checking that at least there are no blobs within the container - for _ in container_client.list_blobs(): - container_url_without_sas = container_url.split("?")[0] - assert False, f"The source blob in container {container_url_without_sas} should be deleted" - except ResourceNotFoundError: - # Expecting this exception - pass - - # 5. get a link to the blob in the approved location. - # For a full E2E we should try to download it, but can't without special networking setup. - # So at the very least we check that we get the link for it. - request_result = await get_request(f'/api{workspace_path}/requests/{request_id}/link', workspace_owner_token, verify, 200) - container_url = request_result["containerUrl"] - - # 6. create airlock export request - LOGGER.info("Creating airlock export request") - justification = "another business justification" - payload = { - "type": airlock_strings.EXPORT, - "businessJustification": justification - } - - request_result = await post_request(payload, f'/api{workspace_path}/requests', workspace_owner_token, verify, 201) - - assert request_result["airlockRequest"]["type"] == airlock_strings.EXPORT - assert request_result["airlockRequest"]["businessJustification"] == justification - assert request_result["airlockRequest"]["status"] == airlock_strings.DRAFT_STATUS - - request_id = request_result["airlockRequest"]["id"] - - # 7. get container link - LOGGER.info("Getting airlock request container URL") - request_result = await get_request(f'/api{workspace_path}/requests/{request_id}/link', workspace_owner_token, verify, 200) - container_url = request_result["containerUrl"] - # we can't test any more the export flow since we don't have the network - # access to upload the file from within the workspace. diff --git a/e2e_tests/test_airlock_consolidated.py b/e2e_tests/test_airlock_consolidated.py new file mode 100644 index 0000000000..4b80c20237 --- /dev/null +++ b/e2e_tests/test_airlock_consolidated.py @@ -0,0 +1,170 @@ +import re +import pytest +import asyncio +import logging + +from airlock.request import post_request, get_request, upload_blob_using_sas, wait_for_status +from airlock import strings as airlock_strings +from e2e_tests.conftest import get_workspace_owner_token + + +pytestmark = pytest.mark.asyncio(loop_scope="session") +LOGGER = logging.getLogger(__name__) +BLOB_FILE_PATH = "./test_airlock_sample.txt" + + +async def create_and_submit_import(workspace_path, workspace_owner_token, verify): + payload = { + "type": airlock_strings.IMPORT, + "businessJustification": "E2E test import" + } + result = await post_request(payload, f'/api{workspace_path}/requests', workspace_owner_token, verify, 201) + request_id = result["airlockRequest"]["id"] + assert result["airlockRequest"]["status"] == airlock_strings.DRAFT_STATUS + + link_result = await get_request( + f'/api{workspace_path}/requests/{request_id}/link', + workspace_owner_token, verify, 200 + ) + container_url = link_result["containerUrl"] + assert "stalairlock" in container_url and "stalairlockg" not in container_url + + blob_uploaded = False + for attempt in range(5): + try: + await asyncio.sleep(5) + upload_response = await upload_blob_using_sas(BLOB_FILE_PATH, container_url) + if "etag" in upload_response: + blob_uploaded = True + break + except Exception: + LOGGER.info(f"Upload attempt {attempt + 1} failed, retrying...") + await asyncio.sleep(10) + assert blob_uploaded + + result = await post_request(None, f'/api{workspace_path}/requests/{request_id}/submit', workspace_owner_token, verify, 200) + assert result["airlockRequest"]["status"] == airlock_strings.SUBMITTED_STATUS + + await wait_for_status(airlock_strings.IN_REVIEW_STATUS, workspace_owner_token, workspace_path, request_id, verify) + + return request_id, container_url + + +@pytest.mark.timeout(35 * 60) +@pytest.mark.airlock +async def test_v2_import_approve_flow(setup_test_workspace, verify): + workspace_path, workspace_id = setup_test_workspace + workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) + + request_id, container_url = await create_and_submit_import(workspace_path, workspace_owner_token, verify) + LOGGER.info(f"Import request {request_id} is in_review") + + payload = { + "approval": "True", + "decisionExplanation": "Approved for E2E test" + } + result = await post_request(payload, f'/api{workspace_path}/requests/{request_id}/review', workspace_owner_token, verify, 200) + assert result["airlockRequest"]["reviews"][0]["decisionExplanation"] == "Approved for E2E test" + + await wait_for_status(airlock_strings.APPROVED_STATUS, workspace_owner_token, workspace_path, request_id, verify) + LOGGER.info(f"Import request {request_id} approved") + + def extract_container_name(url): + m = re.match(r'https://[^/]+/([^?]+)', url) + return m.group(1) if m else None + + # The link handed out in Draft points at the draft container, which submission seals away. + assert extract_container_name(container_url) == f"{request_id}-draft" + + approved_link = await get_request( + f'/api{workspace_path}/requests/{request_id}/link', + workspace_owner_token, verify, 200 + ) + assert extract_container_name(approved_link["containerUrl"]) == request_id + + +@pytest.mark.timeout(35 * 60) +@pytest.mark.airlock +async def test_v2_import_reject_flow(setup_test_workspace, verify): + workspace_path, workspace_id = setup_test_workspace + workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) + + request_id, _ = await create_and_submit_import(workspace_path, workspace_owner_token, verify) + LOGGER.info(f"Import request {request_id} is in_review, rejecting") + + payload = { + "approval": "False", + "decisionExplanation": "Rejected for E2E test" + } + result = await post_request(payload, f'/api{workspace_path}/requests/{request_id}/review', workspace_owner_token, verify, 200) + assert result["airlockRequest"]["reviews"][0]["decisionExplanation"] == "Rejected for E2E test" + + await wait_for_status(airlock_strings.REJECTED_STATUS, workspace_owner_token, workspace_path, request_id, verify) + LOGGER.info(f"Import request {request_id} rejected") + + +@pytest.mark.timeout(10 * 60) +@pytest.mark.airlock +async def test_v2_import_cancel(setup_test_workspace, verify): + workspace_path, workspace_id = setup_test_workspace + workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) + + payload = { + "type": airlock_strings.IMPORT, + "businessJustification": "E2E cancel test" + } + result = await post_request(payload, f'/api{workspace_path}/requests', workspace_owner_token, verify, 201) + request_id = result["airlockRequest"]["id"] + assert result["airlockRequest"]["status"] == airlock_strings.DRAFT_STATUS + + await asyncio.sleep(10) + + result = await post_request(None, f'/api{workspace_path}/requests/{request_id}/cancel', workspace_owner_token, verify, 200) + assert result["airlockRequest"]["status"] == airlock_strings.CANCELLED_STATUS + LOGGER.info(f"Import request {request_id} cancelled from draft") + + +@pytest.mark.timeout(10 * 60) +@pytest.mark.airlock +async def test_v2_export_uses_workspace_storage(setup_test_workspace, verify): + workspace_path, workspace_id = setup_test_workspace + workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) + + payload = { + "type": airlock_strings.EXPORT, + "businessJustification": "E2E export storage test" + } + result = await post_request(payload, f'/api{workspace_path}/requests', workspace_owner_token, verify, 201) + request_id = result["airlockRequest"]["id"] + + link_result = await get_request( + f'/api{workspace_path}/requests/{request_id}/link', + workspace_owner_token, verify, 200 + ) + container_url = link_result["containerUrl"] + + assert "stalairlockg" in container_url + LOGGER.info(f"Export request uses correct storage: {container_url}") + + +@pytest.mark.timeout(10 * 60) +@pytest.mark.airlock +async def test_v2_import_uses_core_storage(setup_test_workspace, verify): + workspace_path, workspace_id = setup_test_workspace + workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) + + payload = { + "type": airlock_strings.IMPORT, + "businessJustification": "E2E import storage test" + } + result = await post_request(payload, f'/api{workspace_path}/requests', workspace_owner_token, verify, 201) + request_id = result["airlockRequest"]["id"] + + link_result = await get_request( + f'/api{workspace_path}/requests/{request_id}/link', + workspace_owner_token, verify, 200 + ) + container_url = link_result["containerUrl"] + + assert "stalairlock" in container_url and "stalairlockg" not in container_url + LOGGER.info(f"Import request uses correct storage: {container_url}") diff --git a/mkdocs.yml b/mkdocs.yml index f99e74c13f..a9a8679198 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -35,7 +35,7 @@ markdown_extensions: custom_fences: - name: mermaid class: mermaid - format: !!python/name:pymdownx.superfences.fence_code_format + format: !!python/name:pymdownx.superfences.fence_code_format "" - meta - admonition - pymdownx.highlight @@ -44,25 +44,29 @@ markdown_extensions: - pymdownx.tabbed - pymdownx.tasklist - pymdownx.emoji: - emoji_index: !!python/name:material.extensions.emoji.twemoji - emoji_generator: !!python/name:material.extensions.emoji.to_svg + emoji_index: !!python/name:material.extensions.emoji.twemoji "" + emoji_generator: !!python/name:material.extensions.emoji.to_svg "" - attr_list nav: - - Overview: # Pages to explain what the Azure TRE + - Overview: + # Pages to explain what the Azure TRE - Introducing the AzureTRE: index.md # Introduction to the concept of a TRE, and the AzureTRE - User Roles: azure-tre-overview/user-roles.md # The users and roles within an Azure TRE - - Architecture: # Pages to help understand the components, infra, and networking + - Architecture: + # Pages to help understand the components, infra, and networking - System Architecture: azure-tre-overview/architecture.md - Network Architecture: azure-tre-overview/networking.md - Azure Resources: azure-tre-overview/tre-resources-breakdown.md - Airlock: azure-tre-overview/airlock.md + - Airlock (Legacy): azure-tre-overview/airlock-legacy.md - Cost Reporting: azure-tre-overview/cost-reporting.md - Terms and Definitions: using-tre/terms-definitions.md - Case Studies: azure-tre-overview/case-studies.md # - Compliance: azure-tre-overview/compliance-info.md - - QuickStart: # Setup steps for anyone performing an initial deployment of the AzureTRE for eval purposes + - QuickStart: + # Setup steps for anyone performing an initial deployment of the AzureTRE for eval purposes - tre-admins/setup-instructions/index.md - 1. Prerequisites: tre-admins/setup-instructions/prerequisites.md - 2. Deployment Repository: tre-admins/setup-instructions/deployment-repo.md @@ -79,7 +83,8 @@ nav: - 7. Install Base Workspace: tre-admins/setup-instructions/ui-install-base-workspace.md - 8. Install Workspace Service and User Resource: tre-admins/setup-instructions/ui-install-ws-and-ur.md - - Using the Azure TRE: # Documentation for users of the TRE + - Using the Azure TRE: + # Documentation for users of the TRE - Introduction: using-tre/index.md - Custom Templates: using-tre/templates/index.md - Using AzureTRE for Research: @@ -88,7 +93,8 @@ nav: - Importing/exporting data with Airlock: using-tre/tre-for-research/importing-exporting-data-airlock.md - Reviewing Airlock Requests: using-tre/tre-for-research/review-airlock-request.md - - Templates and Services: # Docs to highlight and illustrate workspaces, workspace services etc + - Templates and Services: + # Docs to highlight and illustrate workspaces, workspace services etc - Workspaces: - Base: tre-templates/workspaces/base.md - Unrestricted: tre-templates/workspaces/unrestricted.md @@ -114,8 +120,10 @@ nav: - Import Review VM: tre-templates/user-resources/import-reviewvm.md - Export Review VM: tre-templates/user-resources/export-reviewvm.md - - Technical Guide: # All Technical Documentation (Admin, Development) - - Administration: # Docs related to the deployment and operation of AzureTRE infrastructure + - Technical Guide: + # All Technical Documentation (Admin, Development) + - Administration: + # Docs related to the deployment and operation of AzureTRE infrastructure - Starting and Stopping Azure TRE Services: tre-admins/start-stop.md - Environment Variables: tre-admins/environment-variables.md - Tear-down: tre-admins/tear-down.md @@ -142,9 +150,11 @@ nav: - Firewall Force Tunneling: tre-admins/configure-firewall-force-tunneling.md - DNS Security Policy: tre-admins/dns-security-policy.md - - Development: # Docs related to the developing code for the AzureTRE + - Development: + # Docs related to the developing code for the AzureTRE - Local Development: using-tre/local-development/local-development.md - - Contributing to AzureTRE: # Docs aimed at OSS developers, committing code to the AzureTRE repo + - Contributing to AzureTRE: + # Docs aimed at OSS developers, committing code to the AzureTRE repo - Introduction: tre-developers/index.md - API: - API Overview: tre-developers/api.md @@ -158,7 +168,8 @@ nav: - GitHub Actions: tre-admins/setup-instructions/workflows.md - GitHub PR Bot Commands: tre-developers/github-pr-bot-commands.md - - Developing Workspace Templates: # Docs aimed at developers creating workspace templates + - Developing Workspace Templates: + # Docs aimed at developers creating workspace templates - Authoring Workspace Templates: tre-workspace-authors/authoring-workspace-templates.md - Firewall Rules: tre-workspace-authors/firewall-rules.md - Pipeline Templates: @@ -167,7 +178,8 @@ nav: - AzureTRE CLI: tre-developers/CLI.md - - Troubleshooting FAQ: # General Troubleshooting Section for Development + - Troubleshooting FAQ: + # General Troubleshooting Section for Development - troubleshooting-faq/index.md - Enabling DEBUG logs: troubleshooting-faq/debug-logs.md - API logs using deployment center: troubleshooting-faq/api-logs-deployment-center.md diff --git a/templates/workspaces/airlock-import-review/porter.yaml b/templates/workspaces/airlock-import-review/porter.yaml index 6acc2e9d7c..71b893bfbb 100644 --- a/templates/workspaces/airlock-import-review/porter.yaml +++ b/templates/workspaces/airlock-import-review/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-airlock-import-review -version: 0.16.1 +version: 0.17.0 description: "A workspace to do Airlock Data Import Reviews for Azure TRE" dockerfile: Dockerfile.tmpl registry: azuretre @@ -126,6 +126,10 @@ parameters: type: boolean default: true description: "Enable backups for the workspace, including VMs and shared storage." + - name: enable_legacy_airlock + type: boolean + default: true + description: "Provision connectivity to the legacy (v1) stalimip import-in-progress storage account. Automatically propagated from the core enable_legacy_airlock setting by the resource processor; the default only applies to manual/local deployments." - name: delete_backups_on_uninstall type: boolean default: false @@ -222,6 +226,7 @@ install: aad_redirect_uris_b64: ${ bundle.parameters.aad_redirect_uris } app_service_plan_sku: ${ bundle.parameters.app_service_plan_sku } enable_backup: ${ bundle.parameters.enable_backup } + enable_legacy_airlock: ${ bundle.parameters.enable_legacy_airlock } enable_airlock: false arm_environment: ${ bundle.parameters.arm_environment } enable_cmk_encryption: ${ bundle.parameters.enable_cmk_encryption } @@ -272,6 +277,7 @@ upgrade: aad_redirect_uris_b64: ${ bundle.parameters.aad_redirect_uris } app_service_plan_sku: ${ bundle.parameters.app_service_plan_sku } enable_backup: ${ bundle.parameters.enable_backup } + enable_legacy_airlock: ${ bundle.parameters.enable_legacy_airlock } enable_airlock: false arm_environment: ${ bundle.parameters.arm_environment } enable_cmk_encryption: ${ bundle.parameters.enable_cmk_encryption } @@ -359,6 +365,7 @@ uninstall: aad_redirect_uris_b64: ${ bundle.parameters.aad_redirect_uris } app_service_plan_sku: ${ bundle.parameters.app_service_plan_sku } enable_backup: ${ bundle.parameters.enable_backup } + enable_legacy_airlock: ${ bundle.parameters.enable_legacy_airlock } enable_airlock: false arm_environment: ${ bundle.parameters.arm_environment } enable_cmk_encryption: ${ bundle.parameters.enable_cmk_encryption } diff --git a/templates/workspaces/airlock-import-review/terraform/import_review_resources.terraform b/templates/workspaces/airlock-import-review/terraform/import_review_resources.terraform index 3fbfb6e2bc..502616b4d0 100644 --- a/templates/workspaces/airlock-import-review/terraform/import_review_resources.terraform +++ b/templates/workspaces/airlock-import-review/terraform/import_review_resources.terraform @@ -2,23 +2,88 @@ # The Dockerfile includes a RUN command to change the extension from .terraform to .tf after the files from the base workspace are copied to this directory. locals { - core_resource_group_name = "rg-${var.tre_id}" - # STorage AirLock IMport InProgress + core_resource_group_name = "rg-${var.tre_id}" + airlock_core_storage_name = lower(replace("stalairlock${var.tre_id}", "-", "")) import_in_progress_storage_name = lower(replace("stalimip${var.tre_id}", "-", "")) } +variable "enable_legacy_airlock" { + type = bool + default = true + description = "Provision connectivity to the legacy (v1) stalimip import-in-progress storage account so reviewers can access data for airlock_version=1 requests. Should match the core enable_legacy_airlock setting." +} + module "terraform_azurerm_environment_configuration" { source = "git::https://github.com/microsoft/terraform-azurerm-environment-configuration.git?ref=0.7.0" arm_environment = var.arm_environment } +data "azurerm_storage_account" "sa_airlock_core" { + provider = azurerm.core + name = local.airlock_core_storage_name + resource_group_name = local.core_resource_group_name +} + +resource "azurerm_private_endpoint" "sa_airlock_core_pe" { + name = "pe-airlock-import-review-${local.workspace_resource_name_suffix}" + location = var.location + resource_group_name = azurerm_resource_group.ws.name + subnet_id = module.network.services_subnet_id + + lifecycle { ignore_changes = [tags] } + + private_service_connection { + name = "psc-airlock-import-review-${local.workspace_resource_name_suffix}" + private_connection_resource_id = data.azurerm_storage_account.sa_airlock_core.id + is_manual_connection = false + subresource_names = ["Blob"] + } + + tags = local.tre_workspace_tags +} + +resource "azurerm_private_dns_zone" "stg_airlock_core_blob" { + name = "${data.azurerm_storage_account.sa_airlock_core.name}.${module.terraform_azurerm_environment_configuration.private_links["privatelink.blob.core.windows.net"]}" + resource_group_name = azurerm_resource_group.ws.name + + tags = local.tre_workspace_tags + + depends_on = [azurerm_private_endpoint.sa_airlock_core_pe] +} + +resource "azurerm_private_dns_a_record" "stg_airlock_core_blob" { + name = "@" # Root record + zone_name = azurerm_private_dns_zone.stg_airlock_core_blob.name + resource_group_name = azurerm_resource_group.ws.name + ttl = 300 + records = [azurerm_private_endpoint.sa_airlock_core_pe.private_service_connection[0].private_ip_address] + + tags = local.tre_workspace_tags + +} + +resource "azurerm_private_dns_zone_virtual_network_link" "stg_airlock_core_blob" { + name = "vnl-airlock-import-review-${local.workspace_resource_name_suffix}" + resource_group_name = azurerm_resource_group.ws.name + private_dns_zone_name = azurerm_private_dns_zone.stg_airlock_core_blob.name + virtual_network_id = module.network.vnet_id + + tags = local.tre_workspace_tags + + depends_on = [azurerm_private_dns_a_record.stg_airlock_core_blob] +} + + +# Legacy v1 reviews require private connectivity to the per-stage storage account. data "azurerm_storage_account" "sa_import_inprogress" { + count = var.enable_legacy_airlock ? 1 : 0 provider = azurerm.core name = local.import_in_progress_storage_name resource_group_name = local.core_resource_group_name } resource "azurerm_private_endpoint" "sa_import_inprogress_pe" { + count = var.enable_legacy_airlock ? 1 : 0 name = "stg-ip-import-blob-${local.workspace_resource_name_suffix}" location = var.location resource_group_name = azurerm_resource_group.ws.name @@ -28,7 +93,7 @@ resource "azurerm_private_endpoint" "sa_import_inprogress_pe" { private_service_connection { name = "psc-stg-ip-import-blob-${local.workspace_resource_name_suffix}" - private_connection_resource_id = data.azurerm_storage_account.sa_import_inprogress.id + private_connection_resource_id = data.azurerm_storage_account.sa_import_inprogress[0].id is_manual_connection = false subresource_names = ["Blob"] } @@ -37,7 +102,8 @@ resource "azurerm_private_endpoint" "sa_import_inprogress_pe" { } resource "azurerm_private_dns_zone" "stg_import_inprogress_blob" { - name = "${data.azurerm_storage_account.sa_import_inprogress.name}.${module.terraform_azurerm_environment_configuration.private_links["privatelink.blob.core.windows.net"]}" + count = var.enable_legacy_airlock ? 1 : 0 + name = "${data.azurerm_storage_account.sa_import_inprogress[0].name}.${module.terraform_azurerm_environment_configuration.private_links["privatelink.blob.core.windows.net"]}" resource_group_name = azurerm_resource_group.ws.name tags = local.tre_workspace_tags @@ -46,20 +112,21 @@ resource "azurerm_private_dns_zone" "stg_import_inprogress_blob" { } resource "azurerm_private_dns_a_record" "stg_import_inprogress_blob" { + count = var.enable_legacy_airlock ? 1 : 0 name = "@" # Root record - zone_name = azurerm_private_dns_zone.stg_import_inprogress_blob.name + zone_name = azurerm_private_dns_zone.stg_import_inprogress_blob[0].name resource_group_name = azurerm_resource_group.ws.name ttl = 300 - records = [azurerm_private_endpoint.sa_import_inprogress_pe.private_service_connection[0].private_ip_address] + records = [azurerm_private_endpoint.sa_import_inprogress_pe[0].private_service_connection[0].private_ip_address] tags = local.tre_workspace_tags - } resource "azurerm_private_dns_zone_virtual_network_link" "stg_import_inprogress_blob" { + count = var.enable_legacy_airlock ? 1 : 0 name = "vnl-stg-ip-import-blob-${local.workspace_resource_name_suffix}" resource_group_name = azurerm_resource_group.ws.name - private_dns_zone_name = azurerm_private_dns_zone.stg_import_inprogress_blob.name + private_dns_zone_name = azurerm_private_dns_zone.stg_import_inprogress_blob[0].name virtual_network_id = module.network.vnet_id tags = local.tre_workspace_tags diff --git a/templates/workspaces/base/porter.yaml b/templates/workspaces/base/porter.yaml index 67770d9975..2047006094 100644 --- a/templates/workspaces/base/porter.yaml +++ b/templates/workspaces/base/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-base -version: 2.10.1 +version: 2.11.0 description: "A base Azure TRE workspace" dockerfile: Dockerfile.tmpl registry: azuretre @@ -73,19 +73,20 @@ parameters: - name: create_aad_groups type: boolean default: true - description: "Whether this bundle should create AAD groups for the workspace app roles (required for User Management)" + description: "Whether this bundle should create AAD groups for the workspace app + roles (required for User Management)" - name: core_api_client_id type: string description: "The client id of the core API" - name: workspace_owner_object_id type: string - description: "The object id of the user that will be granted WorkspaceOwner after it is created." + description: "The object id of the user that will be granted WorkspaceOwner + after it is created." - name: client_id type: string default: "" - description: - "The client id of the workspace in the identity provider. This value is typically provided to you - when you create the ws application" + description: "The client id of the workspace in the identity provider. This + value is typically provided to you when you create the ws application" - name: client_secret type: string sensitive: true @@ -112,7 +113,8 @@ parameters: - name: app_role_id_workspace_researcher type: string default: "" - description: "The id of the application role WorkspaceResearcher in the identity provider" + description: "The id of the application role WorkspaceResearcher in the identity + provider" - name: app_role_id_workspace_airlock_manager type: string default: "" @@ -128,6 +130,13 @@ parameters: - name: enable_airlock type: boolean default: true + - name: airlock_version + type: integer + default: 2 + description: "Airlock storage version: 1 = legacy per-stage storage accounts, 2 + = consolidated metadata-based storage. New workspaces default to 2. Pre-v2 workspaces + are stamped with an explicit 1 by the POST /migrations step, so this default never + migrates an existing workspace." - name: arm_environment type: string - name: enable_cmk_encryption @@ -139,7 +148,8 @@ parameters: - name: storage_account_redundancy type: string default: "GRS" - description: "The redundancy option for the storage account in the workspace: GRS (Geo-Redundant Storage) or ZRS (Zone-Redundant Storage)." + description: "The redundancy option for the storage account in the workspace: + GRS (Geo-Redundant Storage) or ZRS (Zone-Redundant Storage)." - name: enable_backup type: boolean default: true @@ -155,7 +165,8 @@ parameters: - name: auto_grant_workspace_consent type: boolean default: true - description: "Setting this to `true` will prevent the need for users to manually grant consent to new workspaces" + description: "Setting this to `true` will prevent the need for users to manually + grant consent to new workspaces" - name: enable_airlock_malware_scanning type: boolean default: false @@ -231,6 +242,11 @@ outputs: applyTo: - install - upgrade + - name: airlock_signer_client_id + type: string + applyTo: + - install + - upgrade mixins: - exec @@ -268,6 +284,7 @@ install: aad_redirect_uris_b64: ${ bundle.parameters.aad_redirect_uris } app_service_plan_sku: ${ bundle.parameters.app_service_plan_sku } enable_airlock: ${ bundle.parameters.enable_airlock } + airlock_version: ${ bundle.parameters.airlock_version } arm_environment: ${ bundle.parameters.arm_environment } enable_cmk_encryption: ${ bundle.parameters.enable_cmk_encryption } key_store_id: ${ bundle.parameters.key_store_id } @@ -298,6 +315,7 @@ install: - name: workspace_owners_group_id - name: workspace_researchers_group_id - name: workspace_airlock_managers_group_id + - name: airlock_signer_client_id upgrade: - terraform: @@ -328,6 +346,7 @@ upgrade: aad_redirect_uris_b64: ${ bundle.parameters.aad_redirect_uris } app_service_plan_sku: ${ bundle.parameters.app_service_plan_sku } enable_airlock: ${ bundle.parameters.enable_airlock } + airlock_version: ${ bundle.parameters.airlock_version } arm_environment: ${ bundle.parameters.arm_environment } enable_cmk_encryption: ${ bundle.parameters.enable_cmk_encryption } key_store_id: ${ bundle.parameters.key_store_id } @@ -358,6 +377,7 @@ upgrade: - name: workspace_owners_group_id - name: workspace_researchers_group_id - name: workspace_airlock_managers_group_id + - name: airlock_signer_client_id - az: description: "Set Azure Cloud Environment" arguments: @@ -426,6 +446,7 @@ uninstall: aad_redirect_uris_b64: ${ bundle.parameters.aad_redirect_uris } app_service_plan_sku: ${ bundle.parameters.app_service_plan_sku } enable_airlock: ${ bundle.parameters.enable_airlock } + airlock_version: ${ bundle.parameters.airlock_version } arm_environment: ${ bundle.parameters.arm_environment } enable_cmk_encryption: ${ bundle.parameters.enable_cmk_encryption } key_store_id: ${ bundle.parameters.key_store_id } diff --git a/templates/workspaces/base/template_schema.json b/templates/workspaces/base/template_schema.json index 2414ed4901..13039965ef 100644 --- a/templates/workspaces/base/template_schema.json +++ b/templates/workspaces/base/template_schema.json @@ -23,6 +23,16 @@ "default": true, "updateable": true }, + "airlock_version": { + "type": "integer", + "title": "Airlock Version", + "description": "Airlock storage version: 1 = legacy per-stage storage accounts, 2 = consolidated metadata-based storage. Leave unset to use the default (2). Version 1 requires legacy airlock to be enabled in core.", + "enum": [ + 1, + 2 + ], + "updateable": true + }, "app_service_plan_sku": { "type": "string", "title": "App Service Plan SKU", @@ -381,4 +391,4 @@ "*" ] } -} \ No newline at end of file +} diff --git a/templates/workspaces/base/terraform/airlock_v2/data.tf b/templates/workspaces/base/terraform/airlock_v2/data.tf new file mode 100644 index 0000000000..43dec00b0a --- /dev/null +++ b/templates/workspaces/base/terraform/airlock_v2/data.tf @@ -0,0 +1,13 @@ +data "azuread_client_config" "current" {} + +data "azurerm_user_assigned_identity" "api_id" { + provider = azurerm.core + name = "id-api-${var.tre_id}" + resource_group_name = "rg-${var.tre_id}" +} + +data "azurerm_private_dns_zone" "blobcore" { + provider = azurerm.core + name = module.terraform_azurerm_environment_configuration.private_links["privatelink.blob.core.windows.net"] + resource_group_name = local.core_resource_group_name +} diff --git a/templates/workspaces/base/terraform/airlock_v2/locals.tf b/templates/workspaces/base/terraform/airlock_v2/locals.tf new file mode 100644 index 0000000000..a82e4ff5d8 --- /dev/null +++ b/templates/workspaces/base/terraform/airlock_v2/locals.tf @@ -0,0 +1,5 @@ +locals { + core_resource_group_name = "rg-${var.tre_id}" + + airlock_workspace_global_storage_name = lower(replace("stalairlockg${var.tre_id}", "-", "")) +} diff --git a/templates/workspaces/base/terraform/airlock_v2/outputs.tf b/templates/workspaces/base/terraform/airlock_v2/outputs.tf new file mode 100644 index 0000000000..7b0921f6ce --- /dev/null +++ b/templates/workspaces/base/terraform/airlock_v2/outputs.tf @@ -0,0 +1,3 @@ +output "airlock_signer_client_id" { + value = azuread_application.airlock_signer.client_id +} diff --git a/templates/workspaces/base/terraform/airlock_v2/providers.tf b/templates/workspaces/base/terraform/airlock_v2/providers.tf new file mode 100644 index 0000000000..a82458ad50 --- /dev/null +++ b/templates/workspaces/base/terraform/airlock_v2/providers.tf @@ -0,0 +1,22 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 4.27.0" + configuration_aliases = [ + azurerm, + azurerm.core + ] + } + azuread = { + source = "hashicorp/azuread" + version = ">= 3.0.0" + } + } +} + + +module "terraform_azurerm_environment_configuration" { + source = "git::https://github.com/microsoft/terraform-azurerm-environment-configuration.git?ref=0.2.0" + arm_environment = var.arm_environment +} diff --git a/templates/workspaces/base/terraform/airlock_v2/signer.tf b/templates/workspaces/base/terraform/airlock_v2/signer.tf new file mode 100644 index 0000000000..0c3051e232 --- /dev/null +++ b/templates/workspaces/base/terraform/airlock_v2/signer.tf @@ -0,0 +1,37 @@ +# Per-workspace signers prevent role collisions and bind SAS access to each private endpoint. + +locals { + aad_issuer = "${module.terraform_azurerm_environment_configuration.active_directory_endpoint}/${data.azuread_client_config.current.tenant_id}/v2.0" + + # Must match the scope the API requests in core/credentials.py. + token_exchange_audience = var.arm_environment == "usgovernment" ? "api://AzureADTokenExchangeUSGov" : ( + var.arm_environment == "china" ? "api://AzureADTokenExchangeChina" : "api://AzureADTokenExchange" + ) +} + +resource "azuread_application" "airlock_signer" { + display_name = "airlock-signer-${var.short_workspace_id}" + owners = [data.azuread_client_config.current.object_id] + + lifecycle { ignore_changes = [owners] } +} + +resource "azuread_service_principal" "airlock_signer" { + client_id = azuread_application.airlock_signer.client_id + owners = [data.azuread_client_config.current.object_id] + + feature_tags { + enterprise = true + } + + lifecycle { ignore_changes = [owners] } +} + +resource "azuread_application_federated_identity_credential" "api" { + application_id = azuread_application.airlock_signer.id + display_name = "api-mi" + description = "Allows the core API managed identity to mint airlock SAS as this workspace's signer" + audiences = [local.token_exchange_audience] + issuer = local.aad_issuer + subject = data.azurerm_user_assigned_identity.api_id.principal_id +} diff --git a/templates/workspaces/base/terraform/airlock_v2/storage_accounts.tf b/templates/workspaces/base/terraform/airlock_v2/storage_accounts.tf new file mode 100644 index 0000000000..ce2dbc197f --- /dev/null +++ b/templates/workspaces/base/terraform/airlock_v2/storage_accounts.tf @@ -0,0 +1,111 @@ +data "azurerm_storage_account" "sa_airlock_workspace_global" { + provider = azurerm.core + name = local.airlock_workspace_global_storage_name + resource_group_name = local.core_resource_group_name +} + +# ABAC restricts each private endpoint to containers for its workspace. +resource "azurerm_private_endpoint" "airlock_workspace_pe" { + name = "pe-sa-airlock-ws-global-${var.short_workspace_id}" + location = var.location + resource_group_name = var.ws_resource_group_name + subnet_id = var.services_subnet_id + tags = var.tre_workspace_tags + + lifecycle { ignore_changes = [tags] } + + private_service_connection { + name = "psc-sa-airlock-ws-global-${var.short_workspace_id}" + private_connection_resource_id = data.azurerm_storage_account.sa_airlock_workspace_global.id + is_manual_connection = false + subresource_names = ["Blob"] + } +} + +# Per-workspace qualified zones prevent shared-hostname DNS collisions. +resource "azurerm_private_dns_zone" "airlock_workspace_global" { + name = "${local.airlock_workspace_global_storage_name}.${data.azurerm_private_dns_zone.blobcore.name}" + resource_group_name = var.ws_resource_group_name + tags = var.tre_workspace_tags + + lifecycle { ignore_changes = [tags] } +} + +resource "azurerm_private_dns_zone_virtual_network_link" "airlock_workspace_global" { + name = "vnl-airlock-ws-global-${var.short_workspace_id}" + resource_group_name = var.ws_resource_group_name + private_dns_zone_name = azurerm_private_dns_zone.airlock_workspace_global.name + virtual_network_id = var.workspace_vnet_id + registration_enabled = false + tags = var.tre_workspace_tags + + lifecycle { ignore_changes = [tags] } +} + +resource "azurerm_private_dns_a_record" "airlock_workspace_global" { + name = "@" + zone_name = azurerm_private_dns_zone.airlock_workspace_global.name + resource_group_name = var.ws_resource_group_name + ttl = 10 + records = [azurerm_private_endpoint.airlock_workspace_pe.private_service_connection[0].private_ip_address] +} + +resource "azurerm_role_assignment" "api_workspace_global_blob_data_contributor" { + provider = azurerm.core + + # Deterministic IDs prevent role-assignment collisions on the shared account. + name = uuidv5("url", "${data.azurerm_storage_account.sa_airlock_workspace_global.id}-${var.workspace_id}-blob-data-contributor") + scope = data.azurerm_storage_account.sa_airlock_workspace_global.id + role_definition_name = "Storage Blob Data Contributor" + principal_id = azuread_service_principal.airlock_signer.object_id + principal_type = "ServicePrincipal" + + condition_version = "2.0" + condition = <<-EOT + ( + ( + !(ActionMatches{'Microsoft.Storage/storageAccounts/blobServices/containers/blobs/read'}) + AND !(ActionMatches{'Microsoft.Storage/storageAccounts/blobServices/containers/blobs/write'}) + AND !(ActionMatches{'Microsoft.Storage/storageAccounts/blobServices/containers/blobs/add/action'}) + AND !(ActionMatches{'Microsoft.Storage/storageAccounts/blobServices/containers/blobs/delete'}) + AND !(ActionMatches{'Microsoft.Storage/storageAccounts/blobServices/containers/read'}) + AND !(ActionMatches{'Microsoft.Storage/storageAccounts/blobServices/containers/write'}) + AND !(ActionMatches{'Microsoft.Storage/storageAccounts/blobServices/containers/delete'}) + ) + OR + ( + ( + ActionMatches{'Microsoft.Storage/storageAccounts/blobServices/containers/read'} + OR ActionMatches{'Microsoft.Storage/storageAccounts/blobServices/containers/write'} + OR ActionMatches{'Microsoft.Storage/storageAccounts/blobServices/containers/delete'} + ) + AND + @Resource[Microsoft.Storage/storageAccounts/blobServices/containers/metadata:workspace_id] + StringEquals '${var.workspace_id}' + ) + OR + ( + @Environment[Microsoft.Network/privateEndpoints] StringEqualsIgnoreCase + '${azurerm_private_endpoint.airlock_workspace_pe.id}' + AND + @Resource[Microsoft.Storage/storageAccounts/blobServices/containers/metadata:workspace_id] + StringEquals '${var.workspace_id}' + AND + ( + @Resource[Microsoft.Storage/storageAccounts/blobServices/containers/metadata:stage] + StringEquals 'import-approved' + OR + @Resource[Microsoft.Storage/storageAccounts/blobServices/containers/metadata:stage] + StringEquals 'export-internal' + OR + ( + ActionMatches{'Microsoft.Storage/storageAccounts/blobServices/containers/blobs/read'} + AND + @Resource[Microsoft.Storage/storageAccounts/blobServices/containers/metadata:stage] + StringEquals 'export-in-progress' + ) + ) + ) + ) + EOT +} diff --git a/templates/workspaces/base/terraform/airlock_v2/variables.tf b/templates/workspaces/base/terraform/airlock_v2/variables.tf new file mode 100644 index 0000000000..2b228f35b3 --- /dev/null +++ b/templates/workspaces/base/terraform/airlock_v2/variables.tf @@ -0,0 +1,29 @@ +variable "location" { + type = string +} +variable "tre_id" { + type = string +} +variable "ws_resource_group_name" { + type = string +} +variable "services_subnet_id" { + type = string +} +variable "workspace_vnet_id" { + type = string + description = "The workspace virtual network ID, linked to a per-workspace private DNS zone so this workspace resolves the shared global airlock storage account to its own private endpoint" +} +variable "short_workspace_id" { + type = string +} +variable "tre_workspace_tags" { + type = map(string) +} +variable "arm_environment" { + type = string +} +variable "workspace_id" { + type = string + description = "The workspace ID used for ABAC conditions on global workspace storage" +} diff --git a/templates/workspaces/base/terraform/outputs.tf b/templates/workspaces/base/terraform/outputs.tf index 2bc0c2c716..e9fc89e75b 100644 --- a/templates/workspaces/base/terraform/outputs.tf +++ b/templates/workspaces/base/terraform/outputs.tf @@ -56,3 +56,7 @@ output "workspace_researchers_group_id" { output "workspace_airlock_managers_group_id" { value = var.register_aad_application ? module.aad[0].workspace_airlock_managers_group_id : "" } + +output "airlock_signer_client_id" { + value = var.enable_airlock && var.airlock_version >= 2 ? module.airlock_v2[0].airlock_signer_client_id : "" +} diff --git a/templates/workspaces/base/terraform/variables.tf b/templates/workspaces/base/terraform/variables.tf index b475c0135c..3ddf8395bb 100644 --- a/templates/workspaces/base/terraform/variables.tf +++ b/templates/workspaces/base/terraform/variables.tf @@ -69,6 +69,18 @@ variable "enable_airlock" { description = "Controls the deployment of Airlock resources in the workspace." } +variable "airlock_version" { + type = number + # Defaults to legacy so a direct terraform run never destroys v1 storage. + default = 1 + description = "Airlock storage version: 1 = legacy per-stage storage accounts, 2 = consolidated metadata-based storage." + + validation { + condition = contains([1, 2], var.airlock_version) + error_message = "airlock_version must be 1 (legacy per-stage storage accounts) or 2 (consolidated metadata-based storage)." + } +} + variable "aad_redirect_uris_b64" { type = string # B64 encoded list of objects like [{"name": "my uri 1", "value": "https://..."}, {}] default = "W10=" #b64 for [] @@ -175,11 +187,11 @@ variable "enable_dns_policy" { variable "enable_airlock_malware_scanning" { type = bool default = false - description = "Enable Airlock malware scanning for the workspace" + description = "Enable Airlock malware scanning for the workspace. Only used by the legacy (v1) airlock module; v2 scanning is configured on the consolidated core accounts." } variable "airlock_malware_scan_result_topic_name" { type = string - description = "The name of the topic to publish scan results to" + description = "The name of the topic to publish scan results to. Only used by the legacy (v1) airlock module." default = null } diff --git a/templates/workspaces/base/terraform/workspace.tf b/templates/workspaces/base/terraform/workspace.tf index 8008c545bd..9f8572793f 100644 --- a/templates/workspaces/base/terraform/workspace.tf +++ b/templates/workspaces/base/terraform/workspace.tf @@ -53,7 +53,7 @@ module "aad" { } module "airlock" { - count = var.enable_airlock ? 1 : 0 + count = var.enable_airlock && var.airlock_version == 1 ? 1 : 0 source = "./airlock" location = var.location tre_id = var.tre_id @@ -80,6 +80,30 @@ module "airlock" { ] } +module "airlock_v2" { + count = var.enable_airlock && var.airlock_version >= 2 ? 1 : 0 + source = "./airlock_v2" + location = var.location + tre_id = var.tre_id + tre_workspace_tags = local.tre_workspace_tags + ws_resource_group_name = azurerm_resource_group.ws.name + services_subnet_id = module.network.services_subnet_id + workspace_vnet_id = module.network.vnet_id + short_workspace_id = local.short_workspace_id + workspace_id = var.tre_resource_id + arm_environment = var.arm_environment + + providers = { + azurerm = azurerm + azurerm.core = azurerm.core + azuread = azuread + } + + depends_on = [ + module.network, + ] +} + module "azure_monitor" { source = "./azure-monitor" @@ -102,7 +126,8 @@ module "azure_monitor" { depends_on = [ module.network, - module.airlock + module.airlock, + module.airlock_v2 ] }