diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..567aeae63 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +--- +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..b3321c81f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,95 @@ +--- +name: CI + +"on": + pull_request: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + lint: + name: Ruff + runs-on: ubuntu-24.04 + steps: + - name: Check out repository + # v7.0.1 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Set up Python + # v7.0.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: 3.x + - name: Set up uv + # v10.0.1 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d + with: + enable-cache: true + cache-suffix: lint-upgrade + - name: Sync lint environment + run: uv sync --upgrade --no-dev --group lint + - name: Run Ruff + run: uv run --no-sync ruff check . + + test: + name: Python ${{ matrix.python-version }} + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + python-version: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" + steps: + - name: Check out repository + # v7.0.1 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Set up Python ${{ matrix.python-version }} + # v7.0.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: ${{ matrix.python-version }} + - name: Set up uv + # v10.0.1 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d + with: + enable-cache: true + cache-suffix: test-${{ matrix.python-version }}-upgrade + - name: Sync test environment + run: >- + uv sync --upgrade --python "${{ matrix.python-version }}" + --no-dev --group test + - name: Run unit tests + run: >- + uv run --no-sync pytest tests + --junitxml=junit/azure-iot-device-test-results.xml + --cov=azure + --cov-report=xml:coverage.xml + --cov-report=html:coverage + --cov-append + - name: Upload test results and coverage + if: ${{ always() }} + # v7.0.1 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: test-results-python-${{ matrix.python-version }} + path: | + junit/ + coverage.xml + coverage/ + if-no-files-found: error diff --git a/README.md b/README.md index f33c65f1c..d25efc758 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@

azure-iot-device

-![Build Status](https://azure-iot-sdks.visualstudio.com/azure-iot-sdks/_apis/build/status/Azure.azure-iot-sdk-python) +[![Build Status](https://github.com/Azure/azure-iot-sdk-python/actions/workflows/ci.yml/badge.svg)](https://github.com/Azure/azure-iot-sdk-python/actions/workflows/ci.yml) The Azure IoT Device SDK for Python enables Python developers to easily create IoT device solutions that seamlessly connect to the Azure IoT Hub ecosystem. diff --git a/vsts/build.yaml b/vsts/build.yaml index fc0406816..f9a3b9d91 100644 --- a/vsts/build.yaml +++ b/vsts/build.yaml @@ -3,9 +3,9 @@ resources: variables: # CodeQL is auto-injected into every job by the 1ES policy decorator. Off by - # default so the python test matrix (5 parallel jobs) does not each build and - # upload a competing database -- S360 records arg_max(SnapshotDate) per - # (repo, language). The dedicated 'CodeQL' job below opts back in. + # default so other jobs do not build and upload competing databases -- S360 + # records arg_max(SnapshotDate) per (repo, language). The dedicated 'CodeQL' + # job below opts back in. # See https://aka.ms/codeql3000 and https://aka.ms/sdlfaqcodeql. Codeql.Enabled: false @@ -39,77 +39,18 @@ jobs: condition: always() - job: 'Static_Analysis' + displayName: 'Component Governance' pool: vmImage: 'ubuntu-24.04' steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.x' - architecture: 'x64' - task: ComponentGovernanceComponentDetection@0 inputs: scanType: 'Register' verbosity: 'Verbose' alertWarningLevel: 'High' - - - script: 'python -m pip install uv' - displayName: 'Install uv' - - - script: 'uv sync --upgrade --no-dev --group lint' - displayName: 'Sync lint environment' - - - script: 'uv run --no-sync ruff check .' - displayName: 'Ruff check' - -- job: 'Test' - pool: - vmImage: 'ubuntu-24.04' - strategy: - matrix: - Python310: - python.version: '3.10' - Python311: - python.version: '3.11' - Python312: - python.version: '3.12' - Python313: - python.version: '3.13' - Python314: - python.version: '3.14' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(python.version)' - inputs: - versionSpec: '$(python.version)' - architecture: 'x64' - - - script: 'python -m pip install uv' - displayName: 'Install uv' - - - script: 'uv sync --upgrade --python "$(python.version)" --no-dev --group test' - displayName: 'Sync test environment' - - - script: 'uv run --no-sync pytest tests --junitxml=junit/azure-iot-device-test-results.xml --cov=azure --cov-report=xml:coverage.xml --cov-report=html:coverage --cov-append' - displayName: pytest - - - task: PublishTestResults@2 - displayName: 'Publish Test Results **/*-test-results.xml' - inputs: - testResultsFiles: '**/*-test-results.xml' - testRunTitle: 'Python $(python.version)' - condition: always() - - - task: PublishCodeCoverageResults@2 - inputs: - summaryFileLocation: '$(Build.SourcesDirectory)/coverage.xml' - failIfCoverageEmpty: true - job: 'Publish' - dependsOn: - - 'Test' - - 'Static_Analysis' - pool: vmImage: 'ubuntu-24.04' @@ -122,8 +63,20 @@ jobs: - script: 'python -m pip install uv' displayName: 'Install uv' + # Cache downloads without reusing the job environment. + - task: Cache@2 + displayName: 'Cache uv' + inputs: + key: '"uv" | "$(Agent.OS)" | "build" | uv.lock' + restoreKeys: | + "uv" | "$(Agent.OS)" | "build" + "uv" | "$(Agent.OS)" + path: $(Pipeline.Workspace)/.uv-cache + - script: 'uv build' displayName: Build source and wheel distribution + env: + UV_CACHE_DIR: $(Pipeline.Workspace)/.uv-cache - task: PublishBuildArtifacts@1 displayName: 'Publish artifact dist' diff --git a/vsts/dps-e2e.yaml b/vsts/dps-e2e.yaml index 3678e075c..72987cca7 100644 --- a/vsts/dps-e2e.yaml +++ b/vsts/dps-e2e.yaml @@ -9,11 +9,12 @@ stages: pool: vmImage: 'ubuntu-24.04' steps: + - checkout: none - script: | - sudo apt-get install -y uuid - AZURE_RESOURCE_GROUP="iotpythondpse2e$(uuid | tr -d '-')" - AZURE_IOTHUB_NAME="iotpythondpse2e$(uuid | tr -d '-')" - AZURE_DPS_NAME="iotpythondpse2e$(uuid | tr -d '-')" + RESOURCE_SUFFIX=$(tr -d '-' < /proc/sys/kernel/random/uuid) + AZURE_RESOURCE_GROUP="iotpythondpse2e$RESOURCE_SUFFIX" + AZURE_IOTHUB_NAME="iotpythondpse2e$RESOURCE_SUFFIX" + AZURE_DPS_NAME="iotpythondpse2e$RESOURCE_SUFFIX" # Export variables to other stages/jobs/steps. echo "##vso[task.setvariable variable=AZURE_RESOURCE_GROUP;isOutput=true]$AZURE_RESOURCE_GROUP" @@ -178,8 +179,8 @@ stages: echo "##vso[task.setvariable variable=DPS_ROOT_CA_CERT_PATH;isOutput=true]$DPS_ROOT_CA_CERT_PATH" echo "##vso[task.setvariable variable=DPS_ROOT_CA_PK_PATH;isOutput=true]$DPS_ROOT_CA_PK_PATH" echo "##vso[task.setvariable variable=DPS_ROOT_CA_CERT_BASE64;isOutput=true]$DPS_ROOT_CA_CERT_BASE64" - echo "##vso[task.setvariable variable=DPS_ROOT_CA_PK_BASE64;isOutput=true]$DPS_ROOT_CA_PK_BASE64" - echo "##vso[task.setvariable variable=DPS_ROOT_CA_PASSWORD;isOutput=true]$DPS_ROOT_CA_PASSWORD" + echo "##vso[task.setvariable variable=DPS_ROOT_CA_PK_BASE64;isOutput=true;isSecret=true]$DPS_ROOT_CA_PK_BASE64" + echo "##vso[task.setvariable variable=DPS_ROOT_CA_PASSWORD;isOutput=true;isSecret=true]$DPS_ROOT_CA_PASSWORD" echo "##vso[task.setvariable variable=DPS_ROOT_CA_CONFIG_PATH;isOutput=true]$DPS_ROOT_CA_CONFIG_PATH" name: testCertificates displayName: Create Test Certificates @@ -202,19 +203,22 @@ stages: # Create Resource Group az group create -g $AZURE_RESOURCE_GROUP -l $(AZURE_REGION) - # Create Azure IoT Hub - time az iot hub create -g $AZURE_RESOURCE_GROUP -n $AZURE_IOTHUB_NAME --dla false --mintls 1.2 --partition-count 4 --sku S1 + # Create independent resources concurrently, then wait before linking them. + time az iot hub create -g $AZURE_RESOURCE_GROUP -n $AZURE_IOTHUB_NAME --dla false --mintls 1.2 --partition-count 4 --sku S1 & + iothub_create_pid=$! + time az iot dps create -g $AZURE_RESOURCE_GROUP --name $AZURE_DPS_NAME -l $(AZURE_REGION) & + dps_create_pid=$! + + create_failed=0 + wait "$iothub_create_pid" || create_failed=1 + wait "$dps_create_pid" || create_failed=1 + if (( create_failed )); then + exit 1 + fi export IOTHUB_CONNECTION_STRING=$(az iot hub connection-string show -g $AZURE_RESOURCE_GROUP --hub-name $AZURE_IOTHUB_NAME --pn iothubowner --query "connectionString" --output tsv) export EVENTHUB_CONNECTION_STRING=$(az iot hub connection-string show -g $AZURE_RESOURCE_GROUP --hub-name $AZURE_IOTHUB_NAME --eh --query "connectionString" --output tsv) - for i in $(seq 1 11); do - cg_name="cg$i" - az iot hub consumer-group create -g "$AZURE_RESOURCE_GROUP" --hub-name "$AZURE_IOTHUB_NAME" --name "$cg_name" - done - - # Create Device Provisioning service - time az iot dps create -g $AZURE_RESOURCE_GROUP --name $AZURE_DPS_NAME -l $(AZURE_REGION) time az iot dps linked-hub create -g $AZURE_RESOURCE_GROUP --dps-name $AZURE_DPS_NAME --connection-string "$IOTHUB_CONNECTION_STRING" export DPS_CONNECTION_STRING=$(az iot dps connection-string show -g $AZURE_RESOURCE_GROUP --dps-name $AZURE_DPS_NAME --policy-name provisioningserviceowner --query "connectionString" --output tsv) @@ -245,9 +249,9 @@ stages: time az iot dps certificate verify --dps-name $AZURE_DPS_NAME --resource-group $AZURE_RESOURCE_GROUP --name $DPS_ROOT_CA_CERT_NAME --path $verification_code_pem_path --etag $etag # Export variables to other stages/jobs/steps. - echo "##vso[task.setvariable variable=IOTHUB_CONNECTION_STRING;isOutput=true]$IOTHUB_CONNECTION_STRING" - echo "##vso[task.setvariable variable=EVENTHUB_CONNECTION_STRING;isOutput=true]$EVENTHUB_CONNECTION_STRING" - echo "##vso[task.setvariable variable=DPS_CONNECTION_STRING;isOutput=true]$DPS_CONNECTION_STRING" + echo "##vso[task.setvariable variable=IOTHUB_CONNECTION_STRING;isOutput=true;isSecret=true]$IOTHUB_CONNECTION_STRING" + echo "##vso[task.setvariable variable=EVENTHUB_CONNECTION_STRING;isOutput=true;isSecret=true]$EVENTHUB_CONNECTION_STRING" + echo "##vso[task.setvariable variable=DPS_CONNECTION_STRING;isOutput=true;isSecret=true]$DPS_CONNECTION_STRING" echo "##vso[task.setvariable variable=DPS_ID_SCOPE;isOutput=true]$DPS_ID_SCOPE" echo "##vso[task.setvariable variable=DPS_HOSTNAME;isOutput=true]$DPS_HOSTNAME" displayName: Create IoT Hub & DPS @@ -273,11 +277,23 @@ stages: versionSpec: '3.10' architecture: 'x64' + # Cache downloads without reusing the job environment. + - task: Cache@2 + displayName: 'Cache uv' + inputs: + key: '"uv" | "$(Agent.OS)" | "3.10" | uv.lock' + restoreKeys: | + "uv" | "$(Agent.OS)" | "3.10" + "uv" | "$(Agent.OS)" + path: $(Pipeline.Workspace)/.uv-cache + - script: 'python -m pip install uv' displayName: 'Install uv' - script: 'uv sync --upgrade --python 3.10 --no-dev --group test --group e2e' displayName: 'Sync E2E environment' + env: + UV_CACHE_DIR: $(Pipeline.Workspace)/.uv-cache - script: | cd $(Agent.WorkFolder) @@ -322,6 +338,7 @@ stages: variables: AZURE_RESOURCE_GROUP: $[stageDependencies.setup.create_azure_resources.outputs['setResourceNames.AZURE_RESOURCE_GROUP']] steps: + - checkout: none - task: AzureCLI@2 inputs: azureSubscription: 'iot hub sdk service connection' diff --git a/vsts/horton-e2e.yaml b/vsts/horton-e2e.yaml index b31b726fa..ea96a34b8 100644 --- a/vsts/horton-e2e.yaml +++ b/vsts/horton-e2e.yaml @@ -1,3 +1,13 @@ +trigger: + branches: + include: + - main + +pr: + branches: + include: + - main + variables: Horton.FrameworkRoot: $(Agent.BuildDirectory)/e2e-fx Horton.FrameworkRef: master diff --git a/vsts/python-canary.yaml b/vsts/python-canary.yaml index 26e3aa31d..0edc2cfae 100644 --- a/vsts/python-canary.yaml +++ b/vsts/python-canary.yaml @@ -4,7 +4,8 @@ jobs: - job: 'Test' strategy: - maxParallel: 4 + # Each leg has an isolated consumer group, so all legs can run together. + maxParallel: 7 matrix: py310_windows_mqtt: pv: '3.10' @@ -19,27 +20,27 @@ jobs: py310_linux_mqttws: pv: '3.10' transport: 'mqttws' - imageName: 'Ubuntu 20.04' + imageName: 'ubuntu-24.04' consumerGroup: 'cg4' py310_linux_mqtt: pv: '3.10' transport: 'mqtt' - imageName: 'Ubuntu 20.04' + imageName: 'ubuntu-24.04' consumerGroup: 'cg6' py311_linux_mqtt: pv: '3.11' transport: 'mqtt' - imageName: 'Ubuntu 20.04' + imageName: 'ubuntu-24.04' consumerGroup: 'cg7' py312_linux_mqtt: pv: '3.12' transport: 'mqtt' - imageName: 'Ubuntu 20.04' + imageName: 'ubuntu-24.04' consumerGroup: 'cg8' py312_linux_mqttws: pv: '3.12' transport: 'mqttws' - imageName: 'Ubuntu 20.04' + imageName: 'ubuntu-24.04' consumerGroup: 'cg9' pool: @@ -51,11 +52,23 @@ jobs: versionSpec: $(pv) architecture: 'x64' + # Cache downloads without reusing the job environment. + - task: Cache@2 + displayName: 'Cache uv' + inputs: + key: '"uv" | "$(Agent.OS)" | "$(pv)" | uv.lock' + restoreKeys: | + "uv" | "$(Agent.OS)" | "$(pv)" + "uv" | "$(Agent.OS)" + path: $(Pipeline.Workspace)/.uv-cache + - script: 'python -m pip install uv' displayName: 'Install uv' - script: 'uv sync --upgrade --python "$(pv)" --no-dev --group test --group e2e' displayName: 'Sync E2E environment' + env: + UV_CACHE_DIR: $(Pipeline.Workspace)/.uv-cache - script: | cd $(Build.SourcesDirectory)/tests/e2e/iothub_e2e diff --git a/vsts/python-e2e.yaml b/vsts/python-e2e.yaml index b782f17eb..7fb8309b3 100644 --- a/vsts/python-e2e.yaml +++ b/vsts/python-e2e.yaml @@ -15,10 +15,11 @@ stages: pool: vmImage: 'ubuntu-24.04' steps: + - checkout: none - script: | - sudo apt-get install -y uuid - AZURE_RESOURCE_GROUP="aziotsdkpythone2e$(uuid | tr -d '-')" - AZURE_IOTHUB_NAME="aziotsdkpythone2e$(uuid | tr -d '-')" + RESOURCE_SUFFIX=$(tr -d '-' < /proc/sys/kernel/random/uuid) + AZURE_RESOURCE_GROUP="aziotsdkpythone2e$RESOURCE_SUFFIX" + AZURE_IOTHUB_NAME="aziotsdkpythone2e$RESOURCE_SUFFIX" # Export variables to other stages/jobs/steps. echo "##vso[task.setvariable variable=AZURE_RESOURCE_GROUP;isOutput=true]$AZURE_RESOURCE_GROUP" @@ -43,10 +44,26 @@ stages: export IOTHUB_CONNECTION_STRING=$(az iot hub connection-string show -g $AZURE_RESOURCE_GROUP --hub-name $AZURE_IOTHUB_NAME --pn iothubowner --query "connectionString" --output tsv) export EVENTHUB_CONNECTION_STRING=$(az iot hub connection-string show -g $AZURE_RESOURCE_GROUP --hub-name $AZURE_IOTHUB_NAME --eh --query "connectionString" --output tsv) + # Limit Azure API concurrency while creating independent groups faster. + wait_for_consumer_groups() { + local failed=0 + for pid in "$@"; do + wait "$pid" || failed=1 + done + return "$failed" + } + + consumer_group_pids=() for i in $(seq 1 5); do cg_name="cg$i" - az iot hub consumer-group create -g "$AZURE_RESOURCE_GROUP" --hub-name "$AZURE_IOTHUB_NAME" --name "$cg_name" + az iot hub consumer-group create -g "$AZURE_RESOURCE_GROUP" --hub-name "$AZURE_IOTHUB_NAME" --name "$cg_name" & + consumer_group_pids+=("$!") + if (( ${#consumer_group_pids[@]} == 4 )); then + wait_for_consumer_groups "${consumer_group_pids[@]}" + consumer_group_pids=() + fi done + wait_for_consumer_groups "${consumer_group_pids[@]}" # Route twin change events to the built-in Event Hubs-compatible endpoint # so reported-properties E2E tests can observe service-side patch arrivals. @@ -63,8 +80,8 @@ stages: --route-name "twin-change-events" # Export variables to other stages/jobs/steps. - echo "##vso[task.setvariable variable=IOTHUB_CONNECTION_STRING;isOutput=true]$IOTHUB_CONNECTION_STRING" - echo "##vso[task.setvariable variable=EVENTHUB_CONNECTION_STRING;isOutput=true]$EVENTHUB_CONNECTION_STRING" + echo "##vso[task.setvariable variable=IOTHUB_CONNECTION_STRING;isOutput=true;isSecret=true]$IOTHUB_CONNECTION_STRING" + echo "##vso[task.setvariable variable=EVENTHUB_CONNECTION_STRING;isOutput=true;isSecret=true]$EVENTHUB_CONNECTION_STRING" displayName: Create IoT Hub & DPS name: createAzureResources - stage: build_and_tests @@ -87,11 +104,23 @@ stages: versionSpec: $(pv) architecture: 'x64' + # Cache downloads without reusing the job environment. + - task: Cache@2 + displayName: 'Cache uv' + inputs: + key: '"uv" | "$(Agent.OS)" | "$(pv)" | uv.lock' + restoreKeys: | + "uv" | "$(Agent.OS)" | "$(pv)" + "uv" | "$(Agent.OS)" + path: $(Pipeline.Workspace)/.uv-cache + - script: 'python -m pip install uv' displayName: 'Install uv' - script: 'uv sync --upgrade --python "$(pv)" --no-dev --group test --group e2e' displayName: 'Sync E2E environment' + env: + UV_CACHE_DIR: $(Pipeline.Workspace)/.uv-cache - script: | cd $(Build.SourcesDirectory)/tests/e2e/iothub_e2e @@ -126,11 +155,23 @@ stages: versionSpec: $(pv) architecture: 'x64' + # Cache downloads without reusing the job environment. + - task: Cache@2 + displayName: 'Cache uv' + inputs: + key: '"uv" | "$(Agent.OS)" | "$(pv)" | uv.lock' + restoreKeys: | + "uv" | "$(Agent.OS)" | "$(pv)" + "uv" | "$(Agent.OS)" + path: $(Pipeline.Workspace)/.uv-cache + - script: 'python -m pip install uv' displayName: 'Install uv' - script: 'uv sync --upgrade --python "$(pv)" --no-dev --group test --group e2e' displayName: 'Sync E2E environment' + env: + UV_CACHE_DIR: $(Pipeline.Workspace)/.uv-cache - script: | cd $(Build.SourcesDirectory)/tests/e2e/iothub_e2e @@ -156,6 +197,7 @@ stages: variables: AZURE_RESOURCE_GROUP: $[stageDependencies.setup.create_iot_hub_and_dps.outputs['setResourceNames.AZURE_RESOURCE_GROUP']] steps: + - checkout: none - task: AzureCLI@2 inputs: azureSubscription: 'iot hub sdk service connection' @@ -164,8 +206,6 @@ stages: inlineScript: | # set -e - printenv | sort | grep -i azure - time az group delete --name $(AZURE_RESOURCE_GROUP) --yes displayName: Destroy Azure Resource Group condition: always() # Run step even if the pipeline run is cancelled. diff --git a/vsts/python-nightly.yaml b/vsts/python-nightly.yaml index b576ebf72..870ef8ef8 100644 --- a/vsts/python-nightly.yaml +++ b/vsts/python-nightly.yaml @@ -9,10 +9,11 @@ stages: pool: vmImage: 'ubuntu-24.04' steps: + - checkout: none - script: | - sudo apt-get install -y uuid - AZURE_RESOURCE_GROUP="aziotsdkpython$(uuid)" - AZURE_IOTHUB_NAME="aziotsdkpython$(uuid)" + RESOURCE_SUFFIX=$(tr -d '-' < /proc/sys/kernel/random/uuid) + AZURE_RESOURCE_GROUP="aziotsdkpython$RESOURCE_SUFFIX" + AZURE_IOTHUB_NAME="aziotsdkpython$RESOURCE_SUFFIX" # Export variables to other stages/jobs/steps. echo "##vso[task.setvariable variable=AZURE_RESOURCE_GROUP;isOutput=true]$AZURE_RESOURCE_GROUP" @@ -37,10 +38,26 @@ stages: export IOTHUB_CONNECTION_STRING=$(az iot hub connection-string show -g $AZURE_RESOURCE_GROUP --hub-name $AZURE_IOTHUB_NAME --pn iothubowner --query "connectionString" --output tsv) export EVENTHUB_CONNECTION_STRING=$(az iot hub connection-string show -g $AZURE_RESOURCE_GROUP --hub-name $AZURE_IOTHUB_NAME --eh --query "connectionString" --output tsv) + # Limit Azure API concurrency while creating independent groups faster. + wait_for_consumer_groups() { + local failed=0 + for pid in "$@"; do + wait "$pid" || failed=1 + done + return "$failed" + } + + consumer_group_pids=() for i in $(seq 1 11); do cg_name="cg$i" - az iot hub consumer-group create -g "$AZURE_RESOURCE_GROUP" --hub-name "$AZURE_IOTHUB_NAME" --name "$cg_name" + az iot hub consumer-group create -g "$AZURE_RESOURCE_GROUP" --hub-name "$AZURE_IOTHUB_NAME" --name "$cg_name" & + consumer_group_pids+=("$!") + if (( ${#consumer_group_pids[@]} == 4 )); then + wait_for_consumer_groups "${consumer_group_pids[@]}" + consumer_group_pids=() + fi done + wait_for_consumer_groups "${consumer_group_pids[@]}" # Route twin change events to the built-in Event Hubs-compatible endpoint # so reported-properties E2E tests can observe service-side patch arrivals. @@ -57,8 +74,8 @@ stages: --route-name "twin-change-events" # Export variables to other stages/jobs/steps. - echo "##vso[task.setvariable variable=IOTHUB_CONNECTION_STRING;isOutput=true]$IOTHUB_CONNECTION_STRING" - echo "##vso[task.setvariable variable=EVENTHUB_CONNECTION_STRING;isOutput=true]$EVENTHUB_CONNECTION_STRING" + echo "##vso[task.setvariable variable=IOTHUB_CONNECTION_STRING;isOutput=true;isSecret=true]$IOTHUB_CONNECTION_STRING" + echo "##vso[task.setvariable variable=EVENTHUB_CONNECTION_STRING;isOutput=true;isSecret=true]$EVENTHUB_CONNECTION_STRING" displayName: Create IoT Hub & DPS name: createAzureResources - stage: build_and_tests @@ -79,7 +96,8 @@ stages: IOTHUB_CONNECTION_STRING: $[stageDependencies.setup.create_iot_hub_and_dps.outputs['createAzureResources.IOTHUB_CONNECTION_STRING']] EVENTHUB_CONNECTION_STRING: $[stageDependencies.setup.create_iot_hub_and_dps.outputs['createAzureResources.EVENTHUB_CONNECTION_STRING']] strategy: - maxParallel: 9 + # Each leg has an isolated consumer group, so all legs can run together. + maxParallel: 11 matrix: py310_windows_mqtt: pv: '3.10' @@ -147,11 +165,22 @@ stages: versionSpec: $(pv) architecture: 'x64' addToPath: true + # Cache downloads without reusing the job environment. + - task: Cache@2 + displayName: 'Cache uv' + inputs: + key: '"uv" | "$(Agent.OS)" | "$(pv)" | uv.lock' + restoreKeys: | + "uv" | "$(Agent.OS)" | "$(pv)" + "uv" | "$(Agent.OS)" + path: $(Pipeline.Workspace)/.uv-cache - script: 'python -m pip install uv' displayName: 'Install uv' - script: 'uv sync --upgrade --python "$(pv)" --no-dev --group test --group e2e' displayName: 'Sync E2E environment' + env: + UV_CACHE_DIR: $(Pipeline.Workspace)/.uv-cache - script: | cd $(Build.SourcesDirectory)/tests/e2e/iothub_e2e @@ -180,6 +209,7 @@ stages: variables: AZURE_RESOURCE_GROUP: $[stageDependencies.setup.create_iot_hub_and_dps.outputs['setResourceNames.AZURE_RESOURCE_GROUP']] steps: + - checkout: none - task: AzureCLI@2 inputs: azureSubscription: 'iot hub sdk service connection' @@ -188,8 +218,6 @@ stages: inlineScript: | # set -e - printenv | sort | grep -i azure - time az group delete --name $(AZURE_RESOURCE_GROUP) --yes displayName: Destroy Azure Resource Group condition: always() # Run step even if the pipeline run is cancelled. \ No newline at end of file