diff --git a/eng/pipelines/templates/jobs/live.tests.yml b/eng/pipelines/templates/jobs/live.tests.yml index 47d4ca408aa..67ea1ca62f3 100644 --- a/eng/pipelines/templates/jobs/live.tests.yml +++ b/eng/pipelines/templates/jobs/live.tests.yml @@ -28,6 +28,16 @@ parameters: type: boolean - name: PersistOidcToken type: boolean +# When true, skips build-test-resource-config.yml/deploy-test-resources.yml (and thus any +# per-run ARM deployment / Azure tenant auth) and runs PreTestRunSteps in their place. Used +# by legs that test against fixed, self-owned accounts instead of resources deployed fresh +# each run (see sdk/cosmos/pipeline/README.md for the motivating example). +- name: DisableAzureResourceCreation + type: boolean + default: false +- name: PreTestRunSteps + type: object + default: [] jobs: - job: @@ -72,29 +82,34 @@ jobs: - ${{ parameters.PreSteps }} - - template: /eng/common/TestResources/build-test-resource-config.yml - parameters: - SubscriptionConfiguration: ${{ parameters.CloudConfig.SubscriptionConfiguration }} - SubscriptionConfigurations: ${{ parameters.CloudConfig.SubscriptionConfigurations }} - SubscriptionConfigurationFilePaths: ${{ parameters.CloudConfig.SubscriptionConfigurationFilePaths }} - EnvVars: - Pool: $(Pool) - ${{insert}}: ${{ parameters.EnvVars }} - - - template: /eng/common/TestResources/deploy-test-resources.yml - parameters: - ${{ if or(parameters.Location, parameters.CloudConfig.Location) }}: - Location: ${{ coalesce(parameters.Location, parameters.CloudConfig.Location) }} - ServiceDirectory: ${{ parameters.ServiceDirectory }} - SubscriptionConfiguration: $(SubscriptionConfiguration) - ArmTemplateParameters: $(ArmTemplateParameters) - UseFederatedAuth: ${{ parameters.UseFederatedAuth }} - ServiceConnection: ${{ parameters.CloudConfig.ServiceConnection }} - SubscriptionConfigurationFilePaths: ${{ parameters.CloudConfig.SubscriptionConfigurationFilePaths}} - PersistOidcToken: ${{ parameters.PersistOidcToken }} - EnvVars: - Pool: $(Pool) - ${{insert}}: ${{ parameters.EnvVars }} + - ${{ if not(parameters.DisableAzureResourceCreation) }}: + - template: /eng/common/TestResources/build-test-resource-config.yml + parameters: + SubscriptionConfiguration: ${{ parameters.CloudConfig.SubscriptionConfiguration }} + SubscriptionConfigurations: ${{ parameters.CloudConfig.SubscriptionConfigurations }} + SubscriptionConfigurationFilePaths: ${{ parameters.CloudConfig.SubscriptionConfigurationFilePaths }} + EnvVars: + Pool: $(Pool) + ${{insert}}: ${{ parameters.EnvVars }} + + - template: /eng/common/TestResources/deploy-test-resources.yml + parameters: + ${{ if or(parameters.Location, parameters.CloudConfig.Location) }}: + Location: ${{ coalesce(parameters.Location, parameters.CloudConfig.Location) }} + ServiceDirectory: ${{ parameters.ServiceDirectory }} + SubscriptionConfiguration: $(SubscriptionConfiguration) + ArmTemplateParameters: $(ArmTemplateParameters) + UseFederatedAuth: ${{ parameters.UseFederatedAuth }} + ServiceConnection: ${{ parameters.CloudConfig.ServiceConnection }} + SubscriptionConfigurationFilePaths: ${{ parameters.CloudConfig.SubscriptionConfigurationFilePaths}} + PersistOidcToken: ${{ parameters.PersistOidcToken }} + EnvVars: + Pool: $(Pool) + ${{insert}}: ${{ parameters.EnvVars }} + - ${{ else }}: + # No ARM deployment and no Azure tenant auth for this leg - resolve a fixed, + # self-owned account instead (see sdk/cosmos/pipeline/README.md). + - ${{ parameters.PreTestRunSteps }} # To allow tests to run in parallel to Pack, we don't reuse the PackageInfo directory # from the Pack job. We just let each test leg create its own PackageInfo directory. @@ -111,7 +126,7 @@ jobs: - ${{ if parameters.UseFederatedAuth }}: - task: AzurePowerShell@5 displayName: "Test Packages - Federated Auth" - condition: and(succeeded(), eq(variables['CI_HAS_DEPLOYED_RESOURCES'], 'true')) + condition: and(succeeded(), or(eq(variables['CI_HAS_DEPLOYED_RESOURCES'], 'true'), eq(${{ parameters.DisableAzureResourceCreation }}, true))) timeoutInMinutes: ${{ parameters.TimeoutInMinutes }} env: CIBW_BUILD_VERBOSITY: 3 @@ -139,7 +154,7 @@ jobs: - ${{ else }}: - task: Powershell@2 displayName: "Test Packages" - condition: and(succeeded(), eq(variables['CI_HAS_DEPLOYED_RESOURCES'], 'true')) + condition: and(succeeded(), or(eq(variables['CI_HAS_DEPLOYED_RESOURCES'], 'true'), eq(${{ parameters.DisableAzureResourceCreation }}, true))) timeoutInMinutes: ${{ parameters.TimeoutInMinutes }} env: CIBW_BUILD_VERBOSITY: 3 @@ -154,14 +169,14 @@ jobs: - task: Powershell@2 displayName: Convert Test Results to JUnit XML - condition: and(succeededOrFailed(), eq(variables['CI_HAS_DEPLOYED_RESOURCES'], 'true')) + condition: and(succeededOrFailed(), or(eq(variables['CI_HAS_DEPLOYED_RESOURCES'], 'true'), eq(${{ parameters.DisableAzureResourceCreation }}, true))) inputs: pwsh: true filePath: $(Build.SourcesDirectory)/eng/scripts/Convert-TestResultsToJUnit.ps1 - task: PublishTestResults@2 displayName: Publish Test Results - condition: and(succeededOrFailed(), eq(variables['CI_HAS_DEPLOYED_RESOURCES'], 'true'), eq(variables['HasJUnitTestResults'], 'true')) + condition: and(succeededOrFailed(), or(eq(variables['CI_HAS_DEPLOYED_RESOURCES'], 'true'), eq(${{ parameters.DisableAzureResourceCreation }}, true)), eq(variables['HasJUnitTestResults'], 'true')) inputs: testResultsFormat: JUnit testResultsFiles: '**/test-results/junit/*.xml' diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 8167c3d2cbf..c185394ccf0 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -52,6 +52,21 @@ parameters: Selection: sparse NonSparseParameters: RustToolchainName GenerateVMJobs: true +# A second, independent set of live-test matrix configs for legs that test against fixed, +# self-owned resources instead of resources deployed fresh each run. Unlike +# LiveTestMatrixConfigs, jobs generated from these configs skip ARM deployment entirely +# (DisableAzureResourceCreation: true) and run PreTestRunSteps in its place, with +# UseFederatedAuth forced off since no Azure tenant auth is needed. See +# sdk/cosmos/pipeline/README.md for the motivating example. +- name: FixedAccountMatrixConfigs + type: object + default: [] +# Steps to run instead of ARM deployment for jobs generated from FixedAccountMatrixConfigs +# (e.g. resolving a fixed account's endpoint/key from a secret). Ignored if +# FixedAccountMatrixConfigs is empty. +- name: PreTestRunSteps + type: object + default: [] - name: AdditionalMatrixConfigs type: object default: [] @@ -171,6 +186,39 @@ extends: SubscriptionConfigurations: ${{ cloud.value.SubscriptionConfigurations }} SubscriptionConfigurationFilePaths: ${{ cloud.value.SubscriptionConfigurationFilePaths }} + - ${{ if ne(length(parameters.FixedAccountMatrixConfigs), 0) }}: + - template: /eng/common/pipelines/templates/jobs/generate-job-matrix.yml + parameters: + GenerateJobName: generate_job_matrix_fixed_account + JobTemplatePath: /eng/pipelines/templates/jobs/live.tests.yml + AdditionalParameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + TimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }} + EnvVars: ${{ parameters.EnvVars }} + PreSteps: ${{ parameters.PreSteps }} + PostSteps: ${{ parameters.PostSteps }} + Location: ${{ parameters.Location }} + # Fixed, self-owned accounts need no Azure tenant auth at all. + UseFederatedAuth: false + PersistOidcToken: false + DisableAzureResourceCreation: true + PreTestRunSteps: ${{ parameters.PreTestRunSteps }} + MatrixConfigs: + - ${{ parameters.FixedAccountMatrixConfigs }} + MatrixFilters: + - ${{ cloud.value.MatrixFilters }} + - ${{ parameters.MatrixFilters }} + MatrixReplace: + - ${{ cloud.value.MatrixReplace }} + - ${{ parameters.MatrixReplace }} + CloudConfig: + Cloud: ${{ cloud.key }} + Location: ${{ coalesce(cloud.value.Location, parameters.Location) }} + ServiceConnection: ${{ coalesce(cloud.value.ServiceConnection, lower(format('azure-sdk-tests-{0}', cloud.key))) }} + SubscriptionConfiguration: ${{ cloud.value.SubscriptionConfiguration }} + SubscriptionConfigurations: ${{ cloud.value.SubscriptionConfigurations }} + SubscriptionConfigurationFilePaths: ${{ cloud.value.SubscriptionConfigurationFilePaths }} + # The Prerelease and Release stages are conditioned on: # 1. Internal trigger, not Pull Request trigger # 2. Not weekly build diff --git a/sdk/cosmos/.cspell.json b/sdk/cosmos/.cspell.json index 6b703ad295f..8ceba51d00a 100644 --- a/sdk/cosmos/.cspell.json +++ b/sdk/cosmos/.cspell.json @@ -442,7 +442,14 @@ "vout", "Xpert", "xpert", - "yxxx" + "yxxx", + "issecret", + "multiwrite", + "rustflags", + "sdkci", + "setvariable", + "singlewrite", + "whatif" ], "ignorePaths": [ "live-platform-matrix.json", diff --git a/sdk/cosmos/azure_data_cosmos/build.rs b/sdk/cosmos/azure_data_cosmos/build.rs index 7245cc3cbd5..b856eeb0baa 100644 --- a/sdk/cosmos/azure_data_cosmos/build.rs +++ b/sdk/cosmos/azure_data_cosmos/build.rs @@ -1,5 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. + +// cspell:ignore enableaadauthentication + // Registers custom cfgs used by this crate's integration tests. // // Some CI/build setups enable `-W unexpected-cfgs`, and in newer Rust toolchains @@ -9,4 +12,11 @@ fn main() { println!( "cargo:rustc-check-cfg=cfg(test_category, values(\"emulator\", \"emulator_vnext\", \"emulator_inmemory\", \"emulator_inmemory_gateway_v2\", \"multi_write\", \"split\", \"binary_encoding\", \"gateway_v2\", \"gateway_v2_multi_region\"))" ); + // Marker cfg set by test setups where the target Cosmos account is provisioned + // for AAD data-plane access (local emulator started with /enableaadauthentication, + // or a live account whose bicep deployment created the data-plane role assignment). + // Fixed self-owned live accounts do not currently have that role assignment and + // therefore do not set this cfg, which causes AAD data-plane tests to be ignored + // on those legs. + println!("cargo:rustc-check-cfg=cfg(cosmos_aad_supported)"); } diff --git a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_aad.rs b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_aad.rs index bede8fd05da..94a6fb4f23e 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_aad.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_aad.rs @@ -12,11 +12,12 @@ //! mirrors the data-plane RBAC role provisioned in `test-resources.bicep`, which //! grants item/metadata data actions but **not** management-plane permissions. //! -//! Because the standard `test_category="emulator"` gate is used, the same tests -//! run against the local emulator (Build stage, with the emulator started using -//! `/enableaadauthentication`) and against live accounts (LiveTest stage), where -//! the framework selects a real Entra ID credential via -//! `azure_core_test::credentials::from_env`. +//! Because these tests need AAD data-plane access, they gate on the +//! `cosmos_aad_supported` cfg (set by the local emulator setup when started +//! with `/enableaadauthentication`, and by bicep-provisioned live accounts +//! that include the Cosmos data-plane role assignment). Fixed self-owned live +//! accounts without that role assignment do not set this cfg, so these tests +//! are skipped on those legs. use super::framework; @@ -48,16 +49,8 @@ struct AadTestItem { /// invoked for the Cosmos scope, guarding against silently exercising key auth. #[tokio::test] #[cfg_attr( - not(any( - test_category = "emulator", - test_category = "emulator_vnext", - test_category = "emulator_inmemory" - )), - ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" -)] -#[cfg_attr( - test_category = "emulator_inmemory", - ignore = "hosted in-memory emulator authentication is deferred to PR3" + any(not(cosmos_aad_supported), test_category = "emulator_inmemory"), + ignore = "requires an AAD-enabled Cosmos target (emulator with /enableaadauthentication, or a live account with the Cosmos data-plane role assignment); hosted in-memory emulator authentication is deferred to a follow-up PR" )] pub async fn aad_item_crud_roundtrip() -> Result<(), Box> { TestClient::run_with_unique_db( @@ -157,16 +150,8 @@ pub async fn aad_item_crud_roundtrip() -> Result<(), Box> { /// the `readMetadata` data action the SDK requires on its first request. #[tokio::test] #[cfg_attr( - not(any( - test_category = "emulator", - test_category = "emulator_vnext", - test_category = "emulator_inmemory" - )), - ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" -)] -#[cfg_attr( - test_category = "emulator_inmemory", - ignore = "hosted in-memory emulator authentication is deferred to PR3" + any(not(cosmos_aad_supported), test_category = "emulator_inmemory"), + ignore = "requires an AAD-enabled Cosmos target (emulator with /enableaadauthentication, or a live account with the Cosmos data-plane role assignment); hosted in-memory emulator authentication is deferred to a follow-up PR" )] pub async fn aad_read_container_metadata() -> Result<(), Box> { TestClient::run_with_unique_db( diff --git a/sdk/cosmos/ci.yml b/sdk/cosmos/ci.yml index 81beac5ccbb..ca96c67e501 100644 --- a/sdk/cosmos/ci.yml +++ b/sdk/cosmos/ci.yml @@ -51,7 +51,7 @@ extends: RUST_BACKTRACE: '1' # Live budget for the binary-encoding round-trip fuzzer # (azure_data_cosmos/tests/binary_roundtrip_fuzzer.rs). Only consumed on - # the `binary_encoding` live leg (live-platform-matrix.json); a no-op + # the `binary_encoding` live leg (live-legacy-arm-matrix.json); a no-op # elsewhere. Each iteration exercises 3 encoding configs x 4 point ops, so # 200 => ~2400 round-trips — bounded to fit the live-test time cap. Bump it # for a deeper soak. @@ -87,11 +87,35 @@ extends: Path: sdk/cosmos/inmemory-emulator-matrix.json Selection: all GenerateVMJobs: true - LiveTestMatrixConfigs: + # Cosmos_live_test runs against fixed, self-owned accounts (see + # sdk/cosmos/pipeline/README.md) instead of resources deployed fresh each run, + # so it lives in FixedAccountMatrixConfigs rather than LiveTestMatrixConfigs. + # PreTestRunSteps resolves the account selected by each matrix leg's + # AccountSelector from the rust-ci secret, in place of + # ARM deployment (DisableAzureResourceCreation: true is set automatically for + # this leg by archetype-sdk-client.yml). + FixedAccountMatrixConfigs: - Name: Cosmos_live_test Path: sdk/cosmos/live-platform-matrix.json Selection: sparse GenerateVMJobs: true + PreTestRunSteps: + - template: /sdk/cosmos/pipeline/resolve-test-account-steps.yml + parameters: + AccountSelector: $(AccountSelector) + LiveTestMatrixConfigs: + # Legs that still use ARM-deployed per-run Cosmos accounts (not migrated + # to fixed self-owned accounts): + # * Thin-client (GatewayV2) legs use $(thinclient-test-endpoint)/ + # $(thinclient-test-key) for the thin-client itself, but their primary + # client still needs an ARM-provisioned account. + # * Binary-encoding round-trip fuzz leg exercises encoding boundaries + # against a fresh account and is short-lived. + # Left on the legacy per-run path intentionally. + - Name: Cosmos_live_arm + Path: sdk/cosmos/live-legacy-arm-matrix.json + Selection: all + GenerateVMJobs: true # Dedicated live leg that runs the full integration suite with the primary # (data-plane) client authenticated via Entra ID (AAD) instead of the # account key. `AZURE_COSMOS_AUTH_MODE=aad` surfaces as a job env var and diff --git a/sdk/cosmos/eng/scripts/Invoke-CosmosTestCleanup.ps1 b/sdk/cosmos/eng/scripts/Invoke-CosmosTestCleanup.ps1 index 267a2ebbf0a..912cebabf6b 100644 --- a/sdk/cosmos/eng/scripts/Invoke-CosmosTestCleanup.ps1 +++ b/sdk/cosmos/eng/scripts/Invoke-CosmosTestCleanup.ps1 @@ -102,4 +102,7 @@ $env:AZURE_COSMOS_INMEMORY_ACCOUNT_ENDPOINT = $null # The next package's setup will re-add the correct flag from COSMOS_RUSTFLAGS # (or from AZURE_COSMOS_EMULATOR_FLAVOR=vnext when running the vnext stage). $env:RUSTFLAGS = $env:RUSTFLAGS -replace '\s*--cfg=test_category="[^"]*"', '' +# Also strip the cosmos_aad_supported marker cfg so the next package's setup +# starts from a clean slate. +$env:RUSTFLAGS = $env:RUSTFLAGS -replace '\s*--cfg=cosmos_aad_supported', '' Write-Host "RUSTFLAGS after cleanup: $env:RUSTFLAGS" diff --git a/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 b/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 index 296276565fb..5bcd0518500 100644 --- a/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 +++ b/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 @@ -211,7 +211,7 @@ if ($env:AZURE_COSMOS_EMULATOR_FLAVOR -eq 'vnext') { $env:AZURE_COSMOS_CONNECTION_STRING = "AccountEndpoint=http://localhost:8081;AccountKey=$vnextKey;" Write-Host "Set AZURE_COSMOS_CONNECTION_STRING to vnext emulator endpoint." } - $env:RUSTFLAGS = "$($env:RUSTFLAGS) --cfg=test_category=`"emulator_vnext`"" + $env:RUSTFLAGS = "$($env:RUSTFLAGS) --cfg=test_category=`"emulator_vnext`" --cfg=cosmos_aad_supported" Write-Host "RUSTFLAGS set to: $env:RUSTFLAGS" $env:RUST_TEST_THREADS = "1" return @@ -299,7 +299,7 @@ if ($IsWindows) { # Set environment variables for the tests $env:AZURE_COSMOS_CONNECTION_STRING = "emulator" - $env:RUSTFLAGS = "$($env:RUSTFLAGS) --cfg=test_category=`"emulator`"" + $env:RUSTFLAGS = "$($env:RUSTFLAGS) --cfg=test_category=`"emulator`" --cfg=cosmos_aad_supported" Write-Host "RUSTFLAGS set to: $env:RUSTFLAGS" # Run tests single-threaded to avoid env var contamination from proxy tests. @@ -352,7 +352,7 @@ elseif (Get-Command "docker" -ErrorAction SilentlyContinue) { # Set environment variables for the tests $env:AZURE_COSMOS_CONNECTION_STRING = "emulator" - $env:RUSTFLAGS = "$($env:RUSTFLAGS) --cfg=test_category=`"emulator`"" + $env:RUSTFLAGS = "$($env:RUSTFLAGS) --cfg=test_category=`"emulator`" --cfg=cosmos_aad_supported" Write-Host "RUSTFLAGS set to: $env:RUSTFLAGS" # Run tests single-threaded to avoid env var contamination from proxy tests. diff --git a/sdk/cosmos/live-legacy-arm-matrix.json b/sdk/cosmos/live-legacy-arm-matrix.json new file mode 100644 index 00000000000..c06f9b7bdd5 --- /dev/null +++ b/sdk/cosmos/live-legacy-arm-matrix.json @@ -0,0 +1,23 @@ +{ + "displayNames": {}, + "matrix": { + "Agent": { + "ubuntu": { + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" + } + }, + "RustToolchainName": ["stable"], + "Account Settings": { + "Session SingleRegion GatewayV2": { + "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Session'; testCategory = 'gateway_v2' }" + }, + "Session MultiRegion GatewayV2": { + "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Session'; enableMultipleRegions = $true; testCategory = 'gateway_v2_multi_region' }" + }, + "Session SingleWrite BinaryEncodingRoundtripFuzz": { + "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Session'; enableAutomaticFailover = $false; testCategory = 'binary_encoding' }" + } + } + } +} diff --git a/sdk/cosmos/live-platform-matrix.json b/sdk/cosmos/live-platform-matrix.json index 9c547a385eb..3228267b32d 100644 --- a/sdk/cosmos/live-platform-matrix.json +++ b/sdk/cosmos/live-platform-matrix.json @@ -18,56 +18,29 @@ "RustToolchainName": ["stable"], "Account Settings": { "Eventual SingleWrite": { - "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Eventual'; enableAutomaticFailover = $false; enableContinuousBackup = $true }" + "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Eventual'; enableAutomaticFailover = $false; enableContinuousBackup = $true }", + "AccountSelector": "eventual-singlewrite" }, "Session SingleWrite": { - "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Session'; enableAutomaticFailover = $false; enableContinuousBackup = $true }" + "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Session'; enableAutomaticFailover = $false; enableContinuousBackup = $true }", + "AccountSelector": "session-singlewrite" }, "Strong SingleWrite": { - "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Strong'; enableAutomaticFailover = $false; enableContinuousBackup = $true }" + "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Strong'; enableAutomaticFailover = $false; enableContinuousBackup = $true }", + "AccountSelector": "strong-singlewrite" }, "Session MultiWrite": { - "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Session'; enableMultipleWriteLocations = $true; enableAutomaticFailover = $false; testCategory = 'multi_write'; enableMultipleRegions = $true }" + "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Session'; enableMultipleWriteLocations = $true; enableAutomaticFailover = $false; testCategory = 'multi_write'; enableMultipleRegions = $true }", + "AccountSelector": "session-multiwrite" }, "Session Split": { - "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Session'; enableAutomaticFailover = $false; testCategory = 'split'; enableContinuousBackup = $true }" + "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Session'; enableAutomaticFailover = $false; testCategory = 'split'; enableContinuousBackup = $true }", + "AccountSelector": "session-split" }, "Session SingleWrite MultiRegion PartitionFailover": { - "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Session'; enableAutomaticFailover = $true; enableMultipleRegions = $true; testCategory = 'multi_region' }" + "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Session'; enableAutomaticFailover = $true; enableMultipleRegions = $true; testCategory = 'multi_region' }", + "AccountSelector": "session-mr-failover" } } - }, - "include": [ - { - "Agent": { - "ubuntu": { - "OSVmImage": "env:LINUXVMIMAGE", - "Pool": "env:LINUXPOOL" - } - }, - "RustToolchainName": ["stable"], - "Account Settings": { - "Session SingleRegion GatewayV2": { - "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Session'; testCategory = 'gateway_v2' }" - }, - "Session MultiRegion GatewayV2": { - "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Session'; enableMultipleRegions = $true; testCategory = 'gateway_v2_multi_region' }" - } - } - }, - { - "Agent": { - "ubuntu": { - "OSVmImage": "env:LINUXVMIMAGE", - "Pool": "env:LINUXPOOL" - } - }, - "RustToolchainName": ["stable"], - "Account Settings": { - "Session SingleWrite BinaryEncodingRoundtripFuzz": { - "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Session'; enableAutomaticFailover = $false; testCategory = 'binary_encoding' }" - } - } - } - ] + } } diff --git a/sdk/cosmos/pipeline/README.md b/sdk/cosmos/pipeline/README.md new file mode 100644 index 00000000000..51ddf358700 --- /dev/null +++ b/sdk/cosmos/pipeline/README.md @@ -0,0 +1,112 @@ +# Cosmos live-test fixed accounts + +This directory implements fixed, self-owned Cosmos DB accounts for the +`Cosmos_live_test` CI leg. + +## Why + +`Cosmos_live_test` previously deployed a fresh Cosmos account per CI run (via +`sdk/cosmos/test-resources.bicep`) into whatever Azure tenant the +`azure-sdk-tests-cosmos` service connection happened to be federated to. That +tenant is ephemeral and rotates periodically; rotating it requires recreating +the ADO service connection, which is an administrative action outside of what +a pipeline change can do on its own. + +Since these are key-based tests (they don't need Entra ID / AAD), they don't +need to run against that tenant at all. Instead, `Cosmos_live_test` now runs +against a small set of **fixed accounts** that live permanently in a +separate, self-owned subscription/resource group (`sdk-ci`) that the SDK team +controls directly. Because that subscription never expires or rotates, the +account endpoints and keys - stored once in an ADO secret - keep working +indefinitely, with no per-run Azure authentication and no service-connection +dependency at all for this leg. + +`Cosmos_live_test_aad` (the dedicated AAD/Entra ID leg) is **unaffected** by +this change: it continues to deploy fresh accounts per run via +`test-resources.bicep` against the existing service connection, exactly as +before. Thin-client/GatewayV2 legs are also unaffected/out of scope. + +## How it fits together + +1. `live-test-accounts.schema.json` defines + the shape of a single JSON blob describing every fixed account + (endpoint, key, consistency, region info, etc.), keyed by a logical + **account selector**. +2. `live-test-accounts.sample.json` is a + placeholder-filled example matching the current + `sdk/cosmos/live-platform-matrix.json` selectors, used by the local + resolver tests. +3. `account-provisioning/` contains the one-time / + per-rotation script (`New-CosmosLiveTestAccounts.ps1`) that creates the + accounts in the `sdk-ci` resource group and prints the JSON to store in + the ADO secret. See its README for the full runbook. +4. That JSON is stored as the `rust-ci` **Key Vault secret**, which is linked + (read-through) into the `Test Secrets for Cosmos Live Tests - user + administered` ADO variable group (already wired into `sdk/cosmos/ci.yml`). + Because the variable group is a KV-backed mapping, rotation happens on the + Key Vault secret itself - editing the variable-group value in ADO has no + effect. +5. `resolve-cosmos-test-account.ps1` is a + cross-platform (pwsh) script that, given a selector and the JSON secret, + resolves and exports (via Azure DevOps `##vso[task.setvariable]` logging + commands): + - `AZURE_COSMOS_CONNECTION_STRING` (secret - masked in logs) + - `AZURE_COSMOS_SECONDARY_KEY` (secret, only if the account defines one) + - `ACCOUNT_HOST`, `DATABASE_NAME`, `AZURE_COSMOS_DEFAULT_CONSISTENCY`, + `COSMOS_RUSTFLAGS` + + These are exactly the environment variables that + `sdk/cosmos/test-resources.bicep` used to produce as ARM deployment + outputs, so the test frameworks + (`azure_data_cosmos`/`azure_data_cosmos_driver`) need no changes. +6. `resolve-test-account-steps.yml` is a + reusable ADO step template that invokes the resolver script with the + current job's `$(AccountSelector)` matrix variable. +7. `sdk/cosmos/live-platform-matrix.json` adds an `AccountSelector` string + next to each leg's existing `ArmTemplateParameters` (left in place for + documentation/reference, but no longer used to deploy anything for this + leg). +8. `eng/pipelines/templates/jobs/live.tests.yml` gained a + `DisableAzureResourceCreation` parameter that skips + `build-test-resource-config.yml`/`deploy-test-resources.yml` (no ARM + deployment, no tenant/service-connection auth at all) and a + `PreTestRunSteps` hook that runs in their place. + `eng/pipelines/templates/stages/archetype-sdk-client.yml` gained a + `FixedAccountMatrixConfigs` parameter (a second, independent + `LiveTestMatrixConfigs`-like list) so this only applies to the matrix + configs that opt in - `Cosmos_live_test_aad` keeps deploying real ARM + resources, unaffected. +9. `sdk/cosmos/ci.yml` wires `Cosmos_live_test` through + `FixedAccountMatrixConfigs` with `PreTestRunSteps` pointing at + `resolve-test-account-steps.yml`, while `Cosmos_live_test_aad` remains on + `LiveTestMatrixConfigs` exactly as before. + +## Local testing + +Run the resolver's local test suite (no ADO, no Azure access required): + +```powershell +pwsh sdk/cosmos/pipeline/resolve-cosmos-test-account.tests.ps1 +``` + +You can also invoke the resolver directly against the sample JSON: + +```powershell +$env:COSMOS_ACCOUNTS_LOCAL = 'true' +$env:COSMOS_ACCOUNT_SELECTOR = 'session-singlewrite' +$env:COSMOS_TEST_ACCOUNTS_JSON = Get-Content -Raw sdk/cosmos/pipeline/live-test-accounts.sample.json +./sdk/cosmos/pipeline/resolve-cosmos-test-account.ps1 +``` + +## Adding or rotating an account + +See `account-provisioning/README.md`. + +## What this does *not* solve + +This mechanism only covers **key-based** tests. AAD-specific behavior +requires a real Entra ID identity/tenant to authenticate against; fixed +accounts don't remove that requirement, they only remove it for the tests +that don't need AAD in the first place. `Cosmos_live_test_aad` still depends +on the ephemeral tenant and the `azure-sdk-tests-cosmos` service connection, +exactly as it did before this change. diff --git a/sdk/cosmos/pipeline/account-provisioning/New-CosmosLiveTestAccounts.ps1 b/sdk/cosmos/pipeline/account-provisioning/New-CosmosLiveTestAccounts.ps1 new file mode 100644 index 00000000000..2f425b17611 --- /dev/null +++ b/sdk/cosmos/pipeline/account-provisioning/New-CosmosLiveTestAccounts.ps1 @@ -0,0 +1,255 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +<# +.SYNOPSIS + (Re)creates the fixed Cosmos DB accounts used by the azure-sdk-for-rust Cosmos live + tests and outputs the fixed-accounts JSON (endpoints + keys) for the ADO secret. + +.DESCRIPTION + The Rust Cosmos live tests run against fixed, self-owned accounts in a permanent + subscription/resource group (default: sdk-ci) instead of accounts deployed per-run + into the (frequently rotating) ephemeral test tenant. This script is re-run + whenever an account needs to be created or a key rotated, and: + 1. Ensures the resource group (default: sdk-ci) exists (created if missing). + 2. Creates (idempotently) one Cosmos account per entry in the definition file, + with the requested consistency / multi-write / multi-region / automatic-failover + configuration. + 3. Ensures the shared test database (definition.database, default: shared-test-db) + exists on each account, matching what sdk/cosmos/test-resources.bicep used to + create per-run. + 4. Reads each account's endpoint + primary (and optional secondary) key. + 5. Assembles the versioned account JSON (matching + sdk/cosmos/pipeline/live-test-accounts.schema.json) and emits it (to stdout, + and to -OutputPath if provided). + + This script does NOT touch any ADO variable group. Update the fixed-accounts secret + (see sdk/cosmos/pipeline/README.md) manually with the JSON it outputs. + + Uses the Az PowerShell modules (Az.Accounts, Az.Resources, Az.CosmosDB). + +.PARAMETER SubscriptionId + Subscription hosting the resource group and the Cosmos accounts. Must be a + permanent, self-owned subscription - NOT the ephemeral test tenant. + +.PARAMETER ResourceGroupName + Resource group for the accounts. Created if it does not exist. Defaults to 'sdk-ci'. + +.PARAMETER Location + Optional override for the primary/write region. When omitted, the primary region + comes from the definition's regionDefaults (single source of truth; matches the + regions sdk/cosmos/test-resources.bicep used to deploy). + +.PARAMETER SecondaryLocation + Optional override for the secondary region of multi-region accounts. When omitted, + it comes from the definition's regionDefaults.multiRegion. + +.PARAMETER DefinitionPath + Path to the account definition JSON. Defaults to the file next to this script. + +.PARAMETER AccountNamePrefix + Prefix for the globally-unique Cosmos account names. Defaults to 'sdkci'. + +.PARAMETER OutputPath + Optional path to write the assembled JSON to. The JSON is always also written to + stdout. NOTE: the JSON contains account keys - treat any file you write as a secret. + +.EXAMPLE + # Create/refresh accounts and write the JSON to a file, then update the secret manually + ./New-CosmosLiveTestAccounts.ps1 -SubscriptionId -OutputPath ./accounts.json + +.EXAMPLE + # Dry run - creates nothing, prints the assembled JSON with keys stubbed + ./New-CosmosLiveTestAccounts.ps1 -SubscriptionId -WhatIf + +.NOTES + Requires: PowerShell 7+, Az modules, and Contributor on the subscription. + Idempotent: safe to re-run. +#> +[CmdletBinding(SupportsShouldProcess = $true)] +param( + [Parameter(Mandatory = $true)] + [string] $SubscriptionId, + + [string] $ResourceGroupName = 'sdk-ci', + + [string] $Location, + + [string] $SecondaryLocation, + + [string] $DefinitionPath = ([System.IO.Path]::Combine($PSScriptRoot, 'cosmos-live-test-accounts.definition.json')), + + [ValidatePattern('^[a-z0-9]{1,10}$')] + [string] $AccountNamePrefix = 'sdkci', + + [string] $OutputPath +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +function Write-Info([string]$msg) { Write-Host "==> $msg" -ForegroundColor Cyan } + +# --- Prerequisites ----------------------------------------------------------- +foreach ($m in @('Az.Accounts', 'Az.Resources', 'Az.CosmosDB')) { + if (-not (Get-Module -ListAvailable -Name $m)) { + throw "Required module '$m' is not installed. Install with: Install-Module $m -Scope CurrentUser" + } +} + +if (-not (Test-Path $DefinitionPath)) { throw "Definition file not found: $DefinitionPath" } +$definition = Get-Content -Raw -Path $DefinitionPath | ConvertFrom-Json + +# Regions are defined once in the definition's regionDefaults so the provisioned topology +# matches sdk/cosmos/test-resources.bicep (the ARM template the old per-run flow used). The +# optional -Location / -SecondaryLocation params override the primary/secondary region for +# ad-hoc runs. +if (-not ($definition.PSObject.Properties.Name -contains 'regionDefaults')) { + throw "Definition '$DefinitionPath' is missing 'regionDefaults' (singleRegion / multiRegion)." +} +$singleRegionList = @($definition.regionDefaults.singleRegion) +$multiRegionList = @($definition.regionDefaults.multiRegion) +if ($singleRegionList.Count -lt 1 -or $multiRegionList.Count -lt 2) { + throw "regionDefaults must provide singleRegion (>=1) and multiRegion (>=2) entries." +} +if ($Location) { + $singleRegionList[0] = $Location + $multiRegionList[0] = $Location +} +if ($SecondaryLocation) { + $multiRegionList[1] = $SecondaryLocation +} +$primaryRegion = $multiRegionList[0] + +$databaseName = if ($definition.PSObject.Properties.Name -contains 'database' -and $definition.database) { + [string]$definition.database +} +else { + 'shared-test-db' +} + +Write-Info "Selecting subscription $SubscriptionId" +$null = Set-AzContext -Subscription $SubscriptionId + +# --- Resource group (create if missing) -------------------------------------- +if (-not (Get-AzResourceGroup -Name $ResourceGroupName -ErrorAction SilentlyContinue)) { + if ($PSCmdlet.ShouldProcess($ResourceGroupName, 'Create resource group')) { + Write-Info "Creating resource group $ResourceGroupName in $primaryRegion" + $null = New-AzResourceGroup -Name $ResourceGroupName -Location $primaryRegion + } +} +else { + Write-Info "Resource group $ResourceGroupName already exists" +} + +# --- Helper: build the -LocationObject list from a region list -------------- +# First region is the write region (failoverPriority 0); the rest are read regions. +function New-LocationObjects([string[]] $regionList) { + $locations = @() + for ($i = 0; $i -lt $regionList.Count; $i++) { + $locations += New-AzCosmosDBLocationObject -LocationName $regionList[$i] -FailoverPriority $i -IsZoneRedundant $false + } + return , $locations +} + +# --- Create / update each account, then collect endpoint + keys -------------- +$secret = [ordered]@{ + version = 1 + accounts = [ordered]@{} +} + +foreach ($acct in $definition.accounts) { + $selector = $acct.name + $accountName = ("{0}-{1}" -f $AccountNamePrefix, $selector).ToLower() + if ($accountName.Length -gt 44) { + throw "Generated account name '$accountName' exceeds 44 chars. Shorten AccountNamePrefix or the selector '$selector'." + } + + $multiRegion = [bool]$acct.enableMultipleRegions + $multiWrite = [bool]$acct.enableMultipleWriteLocations + $autoFailover = [bool]$acct.enableAutomaticFailover + $regionList = if ($multiRegion) { $multiRegionList } else { $singleRegionList } + $locations = New-LocationObjects $regionList + + $existing = Get-AzCosmosDBAccount -ResourceGroupName $ResourceGroupName -Name $accountName -ErrorAction SilentlyContinue + if (-not $existing) { + if ($PSCmdlet.ShouldProcess($accountName, "Create Cosmos account [$selector]")) { + Write-Info "Creating Cosmos account '$accountName' (selector=$selector, consistency=$($acct.defaultConsistencyLevel), multiWrite=$multiWrite, multiRegion=$multiRegion, autoFailover=$autoFailover)" + $params = @{ + ResourceGroupName = $ResourceGroupName + Name = $accountName + LocationObject = $locations + DefaultConsistencyLevel = $acct.defaultConsistencyLevel + EnableAutomaticFailover = $autoFailover + EnableMultipleWriteLocations = $multiWrite + ApiKind = 'GlobalDocumentDB' + Capabilities = @('EnableNoSQLVectorSearch', 'EnableNoSQLFullTextSearch') + } + if ($acct.PSObject.Properties.Name -contains 'enableContinuousBackup' -and $acct.enableContinuousBackup) { + $params['BackupPolicyType'] = 'Continuous' + $params['ContinuousTier'] = 'Continuous7Days' + } + $null = New-AzCosmosDBAccount @params + } + } + else { + Write-Info "Cosmos account '$accountName' already exists (selector=$selector); leaving configuration as-is" + } + + # Ensure the shared test database exists (mirrors the `database` resource that + # sdk/cosmos/test-resources.bicep used to create per-run). + if (-not $WhatIfPreference) { + $existingDb = Get-AzCosmosDBSqlDatabase -ResourceGroupName $ResourceGroupName -AccountName $accountName -Name $databaseName -ErrorAction SilentlyContinue + if (-not $existingDb) { + if ($PSCmdlet.ShouldProcess("$accountName/$databaseName", 'Create shared test database')) { + Write-Info "Creating database '$databaseName' on account '$accountName'" + $null = New-AzCosmosDBSqlDatabase -ResourceGroupName $ResourceGroupName -AccountName $accountName -Name $databaseName + } + } + } + + # Read endpoint + keys. Under -WhatIf (dry run) never read real keys - stub them so a + # preview never emits secrets, even for already-provisioned accounts. + if ($WhatIfPreference) { + $endpoint = "https://$accountName.documents.azure.com:443/" + $primary = 'WHATIF_KEY' + $secondary = 'WHATIF_SECONDARY_KEY' + } + else { + $account = Get-AzCosmosDBAccount -ResourceGroupName $ResourceGroupName -Name $accountName + $endpoint = $account.DocumentEndpoint + $keys = Get-AzCosmosDBAccountKey -ResourceGroupName $ResourceGroupName -Name $accountName -Type 'Keys' + $primary = $keys.PrimaryMasterKey + $secondary = $keys.SecondaryMasterKey + } + + $entry = [ordered]@{ + endpoint = $endpoint + key = $primary + database = $databaseName + consistency = $acct.defaultConsistencyLevel + testCategory = $acct.testCategory + multiWrite = $multiWrite + multiRegion = $multiRegion + } + if ($acct.PSObject.Properties.Name -contains 'includeSecondaryKey' -and $acct.includeSecondaryKey) { + $entry['secondaryKey'] = $secondary + } + $entry['regions'] = [string[]]$regionList + + $secret.accounts[$selector] = $entry +} + +# --- Emit the assembled JSON ------------------------------------------------- +$secretJson = $secret | ConvertTo-Json -Depth 8 + +if ($OutputPath) { + if ($PSCmdlet.ShouldProcess($OutputPath, 'Write accounts JSON to file')) { + Set-Content -Path $OutputPath -Value $secretJson -NoNewline + Write-Info "Wrote accounts JSON to '$OutputPath' ($($secret.accounts.Count) accounts). Contains keys - treat as secret." + } +} + +Write-Info "Assembled $($secret.accounts.Count) accounts. Update the fixed-accounts ADO secret manually with this JSON (see sdk/cosmos/pipeline/README.md)." +# Emit the JSON to stdout so it can be captured/redirected. +Write-Output $secretJson diff --git a/sdk/cosmos/pipeline/account-provisioning/README.md b/sdk/cosmos/pipeline/account-provisioning/README.md new file mode 100644 index 00000000000..f96a77bac1a --- /dev/null +++ b/sdk/cosmos/pipeline/account-provisioning/README.md @@ -0,0 +1,71 @@ +# Cosmos live-test account provisioning + +Creates and rotates the fixed, self-owned Cosmos DB accounts used by the +`Cosmos_live_test` leg (see `../README.md` for the overall +mechanism this supports). + +## Prerequisites + +- PowerShell 7+ +- `Az.Accounts`, `Az.Resources`, `Az.CosmosDB` modules + (`Install-Module Az.Accounts, Az.Resources, Az.CosmosDB -Scope CurrentUser`) +- Contributor access on a **permanent, self-owned subscription** - not the + ephemeral test tenant used by `Cosmos_live_test_aad`. Using a permanent + subscription is the entire point: it means these accounts, and the secret + that references them, never need to be touched when the ephemeral tenant + rotates. + +## Usage + +```powershell +# Dry run first - creates nothing, shows what would happen, keys are stubbed +./New-CosmosLiveTestAccounts.ps1 -SubscriptionId -WhatIf + +# Create/refresh accounts and write the JSON to a local file +./New-CosmosLiveTestAccounts.ps1 -SubscriptionId -OutputPath ./accounts.json +``` + +The script is idempotent: accounts that already exist are left alone (their +configuration is not reconciled), but the shared test database is created if +missing, and endpoint/keys are always re-read so the emitted JSON is current. + +## Updating the secret + +The script does **not** write to any secret store itself - copy the JSON it +prints (or the contents of `-OutputPath`) into the `rust-ci` **Key Vault +secret** that backs the `Test Secrets for Cosmos Live Tests - user +administered` variable group. + +The ADO variable group only *links* to the Key Vault secret (read-through +mapping): editing the value in the variable-group UI has no effect. Set a new +version on the KV secret itself (portal, `az keyvault secret set`, or the KV +REST API) - the next pipeline run picks it up automatically. + +Treat the JSON as a secret at every step - it contains account keys. + +## Adding a new account / rotating a key + +1. Add (or edit) an entry in + `cosmos-live-test-accounts.definition.json`. + The `name` becomes the `AccountSelector` that + `sdk/cosmos/live-platform-matrix.json` must reference. +2. Re-run the script (see Usage above). +3. Update the `rust-ci` Key Vault secret with the new JSON (see "Updating the + secret" above). +4. If you added a new selector, add a matching `AccountSelector` entry to the + relevant leg in `sdk/cosmos/live-platform-matrix.json`. + +To rotate a compromised or expiring key, use the Azure Portal or +`New-AzCosmosDBAccountKey` to regenerate it, then re-run this script (which +re-reads the current keys) and set a new version on the `rust-ci` Key Vault +secret. + +## Why this script exists + +The previous live-test flow deployed a fresh Cosmos account per CI run via +`sdk/cosmos/test-resources.bicep`, against whatever tenant the +`azure-sdk-tests-cosmos` service connection was federated to at the time. +That tenant is ephemeral and rotates periodically, and rotating it requires +recreating the ADO service connection - an admin operation, not something +that can be done from a pipeline. Fixed, self-owned accounts in a permanent +subscription sidestep that dependency entirely for key-based tests. diff --git a/sdk/cosmos/pipeline/account-provisioning/cosmos-live-test-accounts.definition.json b/sdk/cosmos/pipeline/account-provisioning/cosmos-live-test-accounts.definition.json new file mode 100644 index 00000000000..61cdc6eb676 --- /dev/null +++ b/sdk/cosmos/pipeline/account-provisioning/cosmos-live-test-accounts.definition.json @@ -0,0 +1,70 @@ +{ + "comment": "Desired fixed Cosmos accounts for the Rust SDK Cosmos live tests. Consumed by New-CosmosLiveTestAccounts.ps1 to (re)create accounts in the sdk-ci resource group and to emit JSON for the fixed-accounts ADO secret (see sdk/cosmos/pipeline/README.md). Account names here are the logical selectors used by live-platform-matrix.json's AccountSelector. accountNamePrefix + name form the actual Cosmos account name (must be globally unique, <=44 chars, lowercase). regionDefaults mirrors sdk/cosmos/test-resources.bicep's region choices (singleRegion: East US 2; multiRegion: East US 2 write + West US 3) so behavior matches the previous per-run ARM-provisioned accounts as closely as possible. testCategory/consistency/database on each entry are copied verbatim into the emitted account JSON and drive COSMOS_RUSTFLAGS / AZURE_COSMOS_DEFAULT_CONSISTENCY / DATABASE_NAME at test time.", + "regionDefaults": { + "singleRegion": ["East US 2"], + "multiRegion": ["East US 2", "West US 3"] + }, + "database": "shared-test-db", + "accounts": [ + { + "name": "eventual-singlewrite", + "defaultConsistencyLevel": "Eventual", + "enableAutomaticFailover": false, + "enableMultipleWriteLocations": false, + "enableMultipleRegions": false, + "enableContinuousBackup": true, + "testCategory": "emulator", + "includeSecondaryKey": true + }, + { + "name": "session-singlewrite", + "defaultConsistencyLevel": "Session", + "enableAutomaticFailover": false, + "enableMultipleWriteLocations": false, + "enableMultipleRegions": false, + "enableContinuousBackup": true, + "testCategory": "emulator", + "includeSecondaryKey": true + }, + { + "name": "strong-singlewrite", + "defaultConsistencyLevel": "Strong", + "enableAutomaticFailover": false, + "enableMultipleWriteLocations": false, + "enableMultipleRegions": false, + "enableContinuousBackup": true, + "testCategory": "emulator", + "includeSecondaryKey": true + }, + { + "name": "session-multiwrite", + "defaultConsistencyLevel": "Session", + "enableAutomaticFailover": false, + "enableMultipleWriteLocations": true, + "enableMultipleRegions": true, + "enableContinuousBackup": false, + "testCategory": "multi_write", + "includeSecondaryKey": true + }, + { + "name": "session-split", + "defaultConsistencyLevel": "Session", + "enableAutomaticFailover": false, + "enableMultipleWriteLocations": false, + "enableMultipleRegions": false, + "enableContinuousBackup": true, + "testCategory": "split", + "includeSecondaryKey": true + }, + { + "name": "session-mr-failover", + "defaultConsistencyLevel": "Session", + "enableAutomaticFailover": true, + "enableMultipleWriteLocations": false, + "enableMultipleRegions": true, + "enableContinuousBackup": false, + "testCategory": "multi_region", + "includeSecondaryKey": true + } + ] +} diff --git a/sdk/cosmos/pipeline/live-test-accounts.sample.json b/sdk/cosmos/pipeline/live-test-accounts.sample.json new file mode 100644 index 00000000000..af05ead7211 --- /dev/null +++ b/sdk/cosmos/pipeline/live-test-accounts.sample.json @@ -0,0 +1,71 @@ +{ + "version": 1, + "accounts": { + "eventual-singlewrite": { + "endpoint": "https://REPLACE-eventual-singlewrite.documents.azure.com:443/", + "key": "REPLACE_KEY", + "secondaryKey": "REPLACE_SECONDARY_KEY", + "database": "shared-test-db", + "consistency": "Eventual", + "testCategory": "emulator", + "multiWrite": false, + "multiRegion": false, + "regions": ["East US 2"] + }, + "session-singlewrite": { + "endpoint": "https://REPLACE-session-singlewrite.documents.azure.com:443/", + "key": "REPLACE_KEY", + "secondaryKey": "REPLACE_SECONDARY_KEY", + "database": "shared-test-db", + "consistency": "Session", + "testCategory": "emulator", + "multiWrite": false, + "multiRegion": false, + "regions": ["East US 2"] + }, + "strong-singlewrite": { + "endpoint": "https://REPLACE-strong-singlewrite.documents.azure.com:443/", + "key": "REPLACE_KEY", + "secondaryKey": "REPLACE_SECONDARY_KEY", + "database": "shared-test-db", + "consistency": "Strong", + "testCategory": "emulator", + "multiWrite": false, + "multiRegion": false, + "regions": ["East US 2"] + }, + "session-multiwrite": { + "endpoint": "https://REPLACE-session-multiwrite.documents.azure.com:443/", + "key": "REPLACE_KEY", + "secondaryKey": "REPLACE_SECONDARY_KEY", + "database": "shared-test-db", + "consistency": "Session", + "testCategory": "multi_write", + "multiWrite": true, + "multiRegion": true, + "regions": ["East US 2", "West US 3"] + }, + "session-split": { + "endpoint": "https://REPLACE-session-split.documents.azure.com:443/", + "key": "REPLACE_KEY", + "secondaryKey": "REPLACE_SECONDARY_KEY", + "database": "shared-test-db", + "consistency": "Session", + "testCategory": "split", + "multiWrite": false, + "multiRegion": false, + "regions": ["East US 2"] + }, + "session-mr-failover": { + "endpoint": "https://REPLACE-session-mr-failover.documents.azure.com:443/", + "key": "REPLACE_KEY", + "secondaryKey": "REPLACE_SECONDARY_KEY", + "database": "shared-test-db", + "consistency": "Session", + "testCategory": "multi_region", + "multiWrite": false, + "multiRegion": true, + "regions": ["East US 2", "West US 3"] + } + } +} diff --git a/sdk/cosmos/pipeline/live-test-accounts.schema.json b/sdk/cosmos/pipeline/live-test-accounts.schema.json new file mode 100644 index 00000000000..0b961323d5e --- /dev/null +++ b/sdk/cosmos/pipeline/live-test-accounts.schema.json @@ -0,0 +1,71 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/Azure/azure-sdk-for-rust/sdk/cosmos/pipeline/live-test-accounts.schema.json", + "title": "Cosmos live-test fixed accounts config", + "description": "Single-secret JSON describing the fixed, self-owned Cosmos DB accounts used by the Rust Cosmos live tests (resource group 'sdk-ci' in a permanent, non-ephemeral subscription). Stored as an ADO variable-group secret and refreshed by account-provisioning/New-CosmosLiveTestAccounts.ps1 whenever an account is rotated. Each live-test matrix leg selects one account by its logical AccountSelector.", + "type": "object", + "required": ["version", "accounts"], + "additionalProperties": false, + "properties": { + "version": { + "type": "integer", + "description": "Schema version. resolve-cosmos-test-account.ps1 rejects versions it does not understand.", + "enum": [1] + }, + "accounts": { + "type": "object", + "description": "Map of logical account selector (matches AccountSelector in live-platform-matrix.json) -> account descriptor.", + "minProperties": 1, + "additionalProperties": { "$ref": "#/definitions/account" } + } + }, + "definitions": { + "account": { + "type": "object", + "required": ["endpoint", "key", "consistency", "testCategory"], + "additionalProperties": false, + "properties": { + "endpoint": { + "type": "string", + "description": "documentEndpoint of the Cosmos account (https://.documents.azure.com:443/).", + "pattern": "^https://" + }, + "key": { + "type": "string", + "description": "Primary master key. Treated as a secret by the resolver (masked in pipeline logs).", + "minLength": 1 + }, + "secondaryKey": { + "type": "string", + "description": "Optional secondary master key (exported as AZURE_COSMOS_SECONDARY_KEY when present). Not consumed by any test today; reserved for future use." + }, + "database": { + "type": "string", + "description": "Name of the pre-created shared test database on this account, exported as DATABASE_NAME. Defaults to 'shared-test-db' when omitted." + }, + "consistency": { + "type": "string", + "description": "Account default consistency level, exported as AZURE_COSMOS_DEFAULT_CONSISTENCY.", + "enum": ["Strong", "BoundedStaleness", "Session", "ConsistentPrefix", "Eventual"] + }, + "testCategory": { + "type": "string", + "description": "Value used to build the RUSTFLAGS cfg exported as COSMOS_RUSTFLAGS, i.e. --cfg=test_category=\"\". Must match the test_category gate the corresponding tests expect (e.g. 'emulator', 'multi_write', 'split', 'multi_region')." + }, + "multiWrite": { + "type": "boolean", + "description": "True if the account has multiple write locations (multi-master). Informational; not exported." + }, + "multiRegion": { + "type": "boolean", + "description": "True if the account is provisioned with more than one region. Informational; not exported." + }, + "regions": { + "type": "array", + "description": "Regions the account is provisioned in (informational / documentation only).", + "items": { "type": "string" } + } + } + } + } +} diff --git a/sdk/cosmos/pipeline/resolve-cosmos-test-account.ps1 b/sdk/cosmos/pipeline/resolve-cosmos-test-account.ps1 new file mode 100644 index 00000000000..8dc629ae4ea --- /dev/null +++ b/sdk/cosmos/pipeline/resolve-cosmos-test-account.ps1 @@ -0,0 +1,126 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# +# Resolves a single fixed Cosmos live-test account from the one JSON secret and +# exports AZURE_COSMOS_CONNECTION_STRING (+ ACCOUNT_HOST, DATABASE_NAME, +# AZURE_COSMOS_DEFAULT_CONSISTENCY, COSMOS_RUSTFLAGS, and optionally +# AZURE_COSMOS_SECONDARY_KEY) for the tests. +# +# The Rust Cosmos live-test matrix runs on ubuntu, windows, and macOS agents, so +# this resolver is PowerShell (pwsh), not bash+jq. +# +# Inputs (environment variables, unless passed as parameters): +# COSMOS_TEST_ACCOUNTS_JSON Raw JSON matching live-test-accounts.schema.json +# (the value of the fixed-accounts ADO secret). +# COSMOS_ACCOUNT_SELECTOR Logical account name to select (e.g. session-multiwrite). +# COSMOS_ACCOUNTS_LOCAL Optional. When "true", prints KEY=VALUE to stdout instead of +# emitting Azure DevOps ##vso logging commands (used for local tests). +# +# Exit codes: 0 on success; non-zero on any validation failure. +[CmdletBinding()] +param( + [string] $AccountSelector = $env:COSMOS_ACCOUNT_SELECTOR, + [string] $AccountsJson = $env:COSMOS_TEST_ACCOUNTS_JSON, + [switch] $Local = ($env:COSMOS_ACCOUNTS_LOCAL -eq 'true') +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +function Fail([string]$message) { + Write-Host "ERROR: $message" -ForegroundColor Red + exit 1 +} + +if ([string]::IsNullOrWhiteSpace($AccountsJson)) { + Fail "COSMOS_TEST_ACCOUNTS_JSON is empty. Wire the fixed-accounts secret to this variable." +} +if ([string]::IsNullOrWhiteSpace($AccountSelector)) { + Fail "COSMOS_ACCOUNT_SELECTOR is empty. Set it to a logical account name." +} + +try { + $config = $AccountsJson | ConvertFrom-Json -ErrorAction Stop +} +catch { + Fail "COSMOS_TEST_ACCOUNTS_JSON is not valid JSON: $($_.Exception.Message)" +} + +$schemaVersion = if ($config.PSObject.Properties.Name -contains 'version') { $config.version } else { $null } +if ($null -eq $schemaVersion -or $schemaVersion -ne 1) { + $displayVersion = if ($null -eq $schemaVersion) { '' } else { [string]$schemaVersion } + Fail "Unsupported or missing schema version '$displayVersion' (parser supports: 1)." +} + +if (-not ($config.PSObject.Properties.Name -contains 'accounts') -or + -not ($config.accounts.PSObject.Properties.Name -contains $AccountSelector)) { + $available = if ($config.PSObject.Properties.Name -contains 'accounts') { + ($config.accounts.PSObject.Properties.Name -join ', ') + } else { '' } + Fail "Account selector '$AccountSelector' not found. Available: $available" +} + +$account = $config.accounts.$AccountSelector + +# Under Set-StrictMode -Version Latest, reading a missing property throws before +# we can emit the targeted 'missing required ...' diagnostic below. Read each +# field conditionally so malformed secrets receive the actionable error message. +function Get-OptionalString($obj, [string]$name) { + if ($obj.PSObject.Properties.Name -contains $name) { [string]$obj.$name } else { '' } +} + +$endpoint = Get-OptionalString $account 'endpoint' +$key = Get-OptionalString $account 'key' +$secondaryKey = Get-OptionalString $account 'secondaryKey' +$databaseRaw = Get-OptionalString $account 'database' +$database = if ([string]::IsNullOrWhiteSpace($databaseRaw)) { 'shared-test-db' } else { $databaseRaw } +$consistency = Get-OptionalString $account 'consistency' +$testCategory = Get-OptionalString $account 'testCategory' + +if ([string]::IsNullOrWhiteSpace($endpoint)) { Fail "Account '$AccountSelector' is missing required 'endpoint'." } +if ([string]::IsNullOrWhiteSpace($key)) { Fail "Account '$AccountSelector' is missing required 'key'." } +if (-not $endpoint.StartsWith('https://')) { Fail "Account '$AccountSelector' endpoint must start with https:// (got '$endpoint')." } +if ([string]::IsNullOrWhiteSpace($consistency)) { Fail "Account '$AccountSelector' is missing required 'consistency'." } +if ([string]::IsNullOrWhiteSpace($testCategory)) { Fail "Account '$AccountSelector' is missing required 'testCategory'." } + +$connectionString = "AccountEndpoint=$endpoint;AccountKey=$key;" +$rustFlags = "--cfg=test_category=`"$testCategory`"" + +function Emit-Public([string]$name, [string]$value) { + if ($Local) { + Write-Output "$name=$value" + } + else { + Write-Host "##vso[task.setvariable variable=$name;issecret=false]$value" + } +} + +# Emit a secret using the azure-sdk double-set convention (see +# eng/common/TestResources/TestResources-Helpers.ps1): register the literal value +# as a secret (variable _NAME) so the log scrubber masks it everywhere, AND set a +# plain variable NAME so it still auto-exports as an environment variable to the +# cargo test task. Marking a variable issecret=true alone would prevent env +# propagation to subsequent tasks. +function Emit-Secret([string]$name, [string]$value) { + if ($Local) { + Write-Output "$name=$value" + } + else { + Write-Host "##vso[task.setvariable variable=_$name;issecret=true]$value" + Write-Host "##vso[task.setvariable variable=$name;issecret=false]$value" + } +} + +# AZURE_COSMOS_CONNECTION_STRING carries the account key, so it must go through +# Emit-Secret. The remaining values are non-sensitive test configuration. +Emit-Secret 'AZURE_COSMOS_CONNECTION_STRING' $connectionString +Emit-Public 'ACCOUNT_HOST' $endpoint +Emit-Public 'DATABASE_NAME' $database +Emit-Public 'AZURE_COSMOS_DEFAULT_CONSISTENCY' $consistency +Emit-Public 'COSMOS_RUSTFLAGS' $rustFlags +if (-not [string]::IsNullOrWhiteSpace($secondaryKey)) { + Emit-Secret 'AZURE_COSMOS_SECONDARY_KEY' $secondaryKey +} + +# Masked, secret-free summary for logs. +Write-Host "Resolved Cosmos test account '$AccountSelector': endpoint=$endpoint consistency=$consistency testCategory=$testCategory key=***" diff --git a/sdk/cosmos/pipeline/resolve-cosmos-test-account.tests.ps1 b/sdk/cosmos/pipeline/resolve-cosmos-test-account.tests.ps1 new file mode 100644 index 00000000000..542dbd447be --- /dev/null +++ b/sdk/cosmos/pipeline/resolve-cosmos-test-account.tests.ps1 @@ -0,0 +1,89 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# +# Local tests for resolve-cosmos-test-account.ps1 (no ADO required). +# Run: pwsh sdk/cosmos/pipeline/resolve-cosmos-test-account.tests.ps1 +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +$here = $PSScriptRoot +$script = [System.IO.Path]::Combine($here, 'resolve-cosmos-test-account.ps1') +$samplePath = [System.IO.Path]::Combine($here, 'live-test-accounts.sample.json') +$sampleJson = Get-Content -Raw -Path $samplePath + +$script:pass = 0 +$script:fail = 0 + +function Test-Ok([string]$name) { + Write-Host " PASS: $name" -ForegroundColor Green + $script:pass++ +} + +function Test-Fail([string]$name, [string]$detail = '') { + Write-Host " FAIL: $name $detail" -ForegroundColor Red + $script:fail++ +} + +function Invoke-Resolver([string]$selector, [string]$json) { + $env:COSMOS_ACCOUNTS_LOCAL = 'true' + $env:COSMOS_ACCOUNT_SELECTOR = $selector + $env:COSMOS_TEST_ACCOUNTS_JSON = $json + $output = & pwsh -NoProfile -File $script 2>$null + $rc = $LASTEXITCODE + Remove-Item Env:\COSMOS_ACCOUNTS_LOCAL, Env:\COSMOS_ACCOUNT_SELECTOR, Env:\COSMOS_TEST_ACCOUNTS_JSON -ErrorAction SilentlyContinue + return @{ Output = ($output -join "`n"); ExitCode = $rc } +} + +Write-Host "Test 1: resolves a valid selector and exports connection string + rustflags" +$result = Invoke-Resolver 'session-multiwrite' $sampleJson +if ($result.ExitCode -eq 0 -and + $result.Output -match 'AZURE_COSMOS_CONNECTION_STRING=AccountEndpoint=https://REPLACE-session-multiwrite' -and + $result.Output -match 'COSMOS_RUSTFLAGS=--cfg=test_category="multi_write"' -and + $result.Output -match 'AZURE_COSMOS_DEFAULT_CONSISTENCY=Session') { + Test-Ok "resolved connection string + rustflags + consistency" +} +else { + Test-Fail "resolve valid selector" "(rc=$($result.ExitCode)): $($result.Output)" +} + +Write-Host "Test 2: exports AZURE_COSMOS_SECONDARY_KEY when present" +$result = Invoke-Resolver 'session-split' $sampleJson +if ($result.ExitCode -eq 0 -and $result.Output -match 'AZURE_COSMOS_SECONDARY_KEY=REPLACE_SECONDARY_KEY') { + Test-Ok "secondary key exported" +} +else { + Test-Fail "secondary key" "(rc=$($result.ExitCode)): $($result.Output)" +} + +Write-Host "Test 3: unknown selector fails with non-zero" +$result = Invoke-Resolver 'does-not-exist' $sampleJson +if ($result.ExitCode -ne 0) { Test-Ok "unknown selector rejected" } else { Test-Fail "unknown selector should fail" $result.Output } + +Write-Host "Test 4: invalid JSON fails" +$result = Invoke-Resolver 'session-singlewrite' '{not json' +if ($result.ExitCode -ne 0) { Test-Ok "invalid json rejected" } else { Test-Fail "invalid json should fail" } + +Write-Host "Test 5: unsupported version fails" +$result = Invoke-Resolver 'session-singlewrite' '{"version":99,"accounts":{"session-singlewrite":{"endpoint":"https://x","key":"k","consistency":"Session","testCategory":"emulator"}}}' +if ($result.ExitCode -ne 0) { Test-Ok "bad version rejected" } else { Test-Fail "bad version should fail" } + +Write-Host "Test 6: missing key fails" +$result = Invoke-Resolver 'x' '{"version":1,"accounts":{"x":{"endpoint":"https://x","consistency":"Session","testCategory":"emulator"}}}' +if ($result.ExitCode -ne 0) { Test-Ok "missing key rejected" } else { Test-Fail "missing key should fail" } + +Write-Host "Test 7: non-https endpoint fails" +$result = Invoke-Resolver 'x' '{"version":1,"accounts":{"x":{"endpoint":"http://x","key":"k","consistency":"Session","testCategory":"emulator"}}}' +if ($result.ExitCode -ne 0) { Test-Ok "non-https rejected" } else { Test-Fail "non-https should fail" } + +Write-Host "Test 8: empty selector fails" +$result = Invoke-Resolver '' $sampleJson +if ($result.ExitCode -ne 0) { Test-Ok "empty selector rejected" } else { Test-Fail "empty selector should fail" } + +Write-Host "Test 9: missing testCategory fails" +$result = Invoke-Resolver 'x' '{"version":1,"accounts":{"x":{"endpoint":"https://x","key":"k","consistency":"Session"}}}' +if ($result.ExitCode -ne 0) { Test-Ok "missing testCategory rejected" } else { Test-Fail "missing testCategory should fail" } + +Write-Host "" +Write-Host "Results: $script:pass passed, $script:fail failed" +if ($script:fail -ne 0) { exit 1 } +exit 0 diff --git a/sdk/cosmos/pipeline/resolve-test-account-steps.yml b/sdk/cosmos/pipeline/resolve-test-account-steps.yml new file mode 100644 index 00000000000..b725f2ad1e7 --- /dev/null +++ b/sdk/cosmos/pipeline/resolve-test-account-steps.yml @@ -0,0 +1,33 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# +# Reusable pre-test step that resolves one fixed Cosmos live-test account from the +# single JSON secret and exports AZURE_COSMOS_CONNECTION_STRING (+ ACCOUNT_HOST, +# DATABASE_NAME, AZURE_COSMOS_DEFAULT_CONSISTENCY, COSMOS_RUSTFLAGS, and optionally +# AZURE_COSMOS_SECONDARY_KEY) for the test run. Drop this into a live-test stage's +# PreTestRunSteps (see eng/pipelines/templates/jobs/live.tests.yml), used together with +# DisableAzureResourceCreation: true so no per-run ARM deployment happens against the +# (frequently rotating) test tenant. +# +# The account JSON lives in the ADO variable group secret referenced by +# AccountsJsonVar (default: rust-ci, see +# sdk/cosmos/pipeline/README.md for how to populate it). +# +# AccountSelector selects which logical account to use. It is typically the macro +# $(AccountSelector) sourced from the live-platform-matrix.json leg. +parameters: + - name: AccountSelector + type: string + - name: AccountsJsonVar + type: string + default: 'rust-ci' + +steps: + - task: Powershell@2 + displayName: 'Resolve fixed Cosmos test account (${{ parameters.AccountSelector }})' + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/sdk/cosmos/pipeline/resolve-cosmos-test-account.ps1 + env: + COSMOS_TEST_ACCOUNTS_JSON: $(${{ parameters.AccountsJsonVar }}) + COSMOS_ACCOUNT_SELECTOR: ${{ parameters.AccountSelector }} diff --git a/sdk/cosmos/test-resources.bicep b/sdk/cosmos/test-resources.bicep index e84ce3fe17c..f4dd1073c4f 100644 --- a/sdk/cosmos/test-resources.bicep +++ b/sdk/cosmos/test-resources.bicep @@ -135,7 +135,7 @@ resource accountName_roleAssignmentId 'Microsoft.DocumentDB/databaseAccounts/sql } } -output COSMOS_RUSTFLAGS string = '--cfg=test_category="${testCategory}"' +output COSMOS_RUSTFLAGS string = '--cfg=test_category="${testCategory}" --cfg=cosmos_aad_supported' output DATABASE_NAME string = databaseName output AZURE_COSMOS_CONNECTION_STRING string = 'AccountEndpoint=${reference(resourceId, apiVersion).documentEndpoint};AccountKey=${listKeys(resourceId, apiVersion).primaryMasterKey};' output ACCOUNT_HOST string = reference(resourceId, apiVersion).documentEndpoint