diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 000000000..567aeae63
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,7 @@
+---
+version: 2
+updates:
+ - package-ecosystem: github-actions
+ directory: /
+ schedule:
+ interval: weekly
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 000000000..b3321c81f
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,95 @@
+---
+name: CI
+
+"on":
+ pull_request:
+ push:
+ branches:
+ - main
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.number || github.run_id }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+ lint:
+ name: Ruff
+ runs-on: ubuntu-24.04
+ steps:
+ - name: Check out repository
+ # v7.0.1
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+ with:
+ persist-credentials: false
+ - name: Set up Python
+ # v7.0.0
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
+ with:
+ python-version: 3.x
+ - name: Set up uv
+ # v10.0.1
+ uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d
+ with:
+ enable-cache: true
+ cache-suffix: lint-upgrade
+ - name: Sync lint environment
+ run: uv sync --upgrade --no-dev --group lint
+ - name: Run Ruff
+ run: uv run --no-sync ruff check .
+
+ test:
+ name: Python ${{ matrix.python-version }}
+ runs-on: ubuntu-24.04
+ strategy:
+ fail-fast: false
+ matrix:
+ python-version:
+ - "3.10"
+ - "3.11"
+ - "3.12"
+ - "3.13"
+ - "3.14"
+ steps:
+ - name: Check out repository
+ # v7.0.1
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+ with:
+ persist-credentials: false
+ - name: Set up Python ${{ matrix.python-version }}
+ # v7.0.0
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Set up uv
+ # v10.0.1
+ uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d
+ with:
+ enable-cache: true
+ cache-suffix: test-${{ matrix.python-version }}-upgrade
+ - name: Sync test environment
+ run: >-
+ uv sync --upgrade --python "${{ matrix.python-version }}"
+ --no-dev --group test
+ - name: Run unit tests
+ run: >-
+ uv run --no-sync pytest tests
+ --junitxml=junit/azure-iot-device-test-results.xml
+ --cov=azure
+ --cov-report=xml:coverage.xml
+ --cov-report=html:coverage
+ --cov-append
+ - name: Upload test results and coverage
+ if: ${{ always() }}
+ # v7.0.1
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
+ with:
+ name: test-results-python-${{ matrix.python-version }}
+ path: |
+ junit/
+ coverage.xml
+ coverage/
+ if-no-files-found: error
diff --git a/README.md b/README.md
index f33c65f1c..d25efc758 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@
azure-iot-device
-
+[](https://github.com/Azure/azure-iot-sdk-python/actions/workflows/ci.yml)
The Azure IoT Device SDK for Python enables Python developers to easily create IoT device solutions that seamlessly connect to the Azure IoT Hub ecosystem.
diff --git a/vsts/build.yaml b/vsts/build.yaml
index fc0406816..f9a3b9d91 100644
--- a/vsts/build.yaml
+++ b/vsts/build.yaml
@@ -3,9 +3,9 @@ resources:
variables:
# CodeQL is auto-injected into every job by the 1ES policy decorator. Off by
- # default so the python test matrix (5 parallel jobs) does not each build and
- # upload a competing database -- S360 records arg_max(SnapshotDate) per
- # (repo, language). The dedicated 'CodeQL' job below opts back in.
+ # default so other jobs do not build and upload competing databases -- S360
+ # records arg_max(SnapshotDate) per (repo, language). The dedicated 'CodeQL'
+ # job below opts back in.
# See https://aka.ms/codeql3000 and https://aka.ms/sdlfaqcodeql.
Codeql.Enabled: false
@@ -39,77 +39,18 @@ jobs:
condition: always()
- job: 'Static_Analysis'
+ displayName: 'Component Governance'
pool:
vmImage: 'ubuntu-24.04'
steps:
- - task: UsePythonVersion@0
- inputs:
- versionSpec: '3.x'
- architecture: 'x64'
- task: ComponentGovernanceComponentDetection@0
inputs:
scanType: 'Register'
verbosity: 'Verbose'
alertWarningLevel: 'High'
-
- - script: 'python -m pip install uv'
- displayName: 'Install uv'
-
- - script: 'uv sync --upgrade --no-dev --group lint'
- displayName: 'Sync lint environment'
-
- - script: 'uv run --no-sync ruff check .'
- displayName: 'Ruff check'
-
-- job: 'Test'
- pool:
- vmImage: 'ubuntu-24.04'
- strategy:
- matrix:
- Python310:
- python.version: '3.10'
- Python311:
- python.version: '3.11'
- Python312:
- python.version: '3.12'
- Python313:
- python.version: '3.13'
- Python314:
- python.version: '3.14'
- steps:
- - task: UsePythonVersion@0
- displayName: 'Use Python $(python.version)'
- inputs:
- versionSpec: '$(python.version)'
- architecture: 'x64'
-
- - script: 'python -m pip install uv'
- displayName: 'Install uv'
-
- - script: 'uv sync --upgrade --python "$(python.version)" --no-dev --group test'
- displayName: 'Sync test environment'
-
- - script: 'uv run --no-sync pytest tests --junitxml=junit/azure-iot-device-test-results.xml --cov=azure --cov-report=xml:coverage.xml --cov-report=html:coverage --cov-append'
- displayName: pytest
-
- - task: PublishTestResults@2
- displayName: 'Publish Test Results **/*-test-results.xml'
- inputs:
- testResultsFiles: '**/*-test-results.xml'
- testRunTitle: 'Python $(python.version)'
- condition: always()
-
- - task: PublishCodeCoverageResults@2
- inputs:
- summaryFileLocation: '$(Build.SourcesDirectory)/coverage.xml'
- failIfCoverageEmpty: true
- job: 'Publish'
- dependsOn:
- - 'Test'
- - 'Static_Analysis'
-
pool:
vmImage: 'ubuntu-24.04'
@@ -122,8 +63,20 @@ jobs:
- script: 'python -m pip install uv'
displayName: 'Install uv'
+ # Cache downloads without reusing the job environment.
+ - task: Cache@2
+ displayName: 'Cache uv'
+ inputs:
+ key: '"uv" | "$(Agent.OS)" | "build" | uv.lock'
+ restoreKeys: |
+ "uv" | "$(Agent.OS)" | "build"
+ "uv" | "$(Agent.OS)"
+ path: $(Pipeline.Workspace)/.uv-cache
+
- script: 'uv build'
displayName: Build source and wheel distribution
+ env:
+ UV_CACHE_DIR: $(Pipeline.Workspace)/.uv-cache
- task: PublishBuildArtifacts@1
displayName: 'Publish artifact dist'
diff --git a/vsts/dps-e2e.yaml b/vsts/dps-e2e.yaml
index 3678e075c..72987cca7 100644
--- a/vsts/dps-e2e.yaml
+++ b/vsts/dps-e2e.yaml
@@ -9,11 +9,12 @@ stages:
pool:
vmImage: 'ubuntu-24.04'
steps:
+ - checkout: none
- script: |
- sudo apt-get install -y uuid
- AZURE_RESOURCE_GROUP="iotpythondpse2e$(uuid | tr -d '-')"
- AZURE_IOTHUB_NAME="iotpythondpse2e$(uuid | tr -d '-')"
- AZURE_DPS_NAME="iotpythondpse2e$(uuid | tr -d '-')"
+ RESOURCE_SUFFIX=$(tr -d '-' < /proc/sys/kernel/random/uuid)
+ AZURE_RESOURCE_GROUP="iotpythondpse2e$RESOURCE_SUFFIX"
+ AZURE_IOTHUB_NAME="iotpythondpse2e$RESOURCE_SUFFIX"
+ AZURE_DPS_NAME="iotpythondpse2e$RESOURCE_SUFFIX"
# Export variables to other stages/jobs/steps.
echo "##vso[task.setvariable variable=AZURE_RESOURCE_GROUP;isOutput=true]$AZURE_RESOURCE_GROUP"
@@ -178,8 +179,8 @@ stages:
echo "##vso[task.setvariable variable=DPS_ROOT_CA_CERT_PATH;isOutput=true]$DPS_ROOT_CA_CERT_PATH"
echo "##vso[task.setvariable variable=DPS_ROOT_CA_PK_PATH;isOutput=true]$DPS_ROOT_CA_PK_PATH"
echo "##vso[task.setvariable variable=DPS_ROOT_CA_CERT_BASE64;isOutput=true]$DPS_ROOT_CA_CERT_BASE64"
- echo "##vso[task.setvariable variable=DPS_ROOT_CA_PK_BASE64;isOutput=true]$DPS_ROOT_CA_PK_BASE64"
- echo "##vso[task.setvariable variable=DPS_ROOT_CA_PASSWORD;isOutput=true]$DPS_ROOT_CA_PASSWORD"
+ echo "##vso[task.setvariable variable=DPS_ROOT_CA_PK_BASE64;isOutput=true;isSecret=true]$DPS_ROOT_CA_PK_BASE64"
+ echo "##vso[task.setvariable variable=DPS_ROOT_CA_PASSWORD;isOutput=true;isSecret=true]$DPS_ROOT_CA_PASSWORD"
echo "##vso[task.setvariable variable=DPS_ROOT_CA_CONFIG_PATH;isOutput=true]$DPS_ROOT_CA_CONFIG_PATH"
name: testCertificates
displayName: Create Test Certificates
@@ -202,19 +203,22 @@ stages:
# Create Resource Group
az group create -g $AZURE_RESOURCE_GROUP -l $(AZURE_REGION)
- # Create Azure IoT Hub
- time az iot hub create -g $AZURE_RESOURCE_GROUP -n $AZURE_IOTHUB_NAME --dla false --mintls 1.2 --partition-count 4 --sku S1
+ # Create independent resources concurrently, then wait before linking them.
+ time az iot hub create -g $AZURE_RESOURCE_GROUP -n $AZURE_IOTHUB_NAME --dla false --mintls 1.2 --partition-count 4 --sku S1 &
+ iothub_create_pid=$!
+ time az iot dps create -g $AZURE_RESOURCE_GROUP --name $AZURE_DPS_NAME -l $(AZURE_REGION) &
+ dps_create_pid=$!
+
+ create_failed=0
+ wait "$iothub_create_pid" || create_failed=1
+ wait "$dps_create_pid" || create_failed=1
+ if (( create_failed )); then
+ exit 1
+ fi
export IOTHUB_CONNECTION_STRING=$(az iot hub connection-string show -g $AZURE_RESOURCE_GROUP --hub-name $AZURE_IOTHUB_NAME --pn iothubowner --query "connectionString" --output tsv)
export EVENTHUB_CONNECTION_STRING=$(az iot hub connection-string show -g $AZURE_RESOURCE_GROUP --hub-name $AZURE_IOTHUB_NAME --eh --query "connectionString" --output tsv)
- for i in $(seq 1 11); do
- cg_name="cg$i"
- az iot hub consumer-group create -g "$AZURE_RESOURCE_GROUP" --hub-name "$AZURE_IOTHUB_NAME" --name "$cg_name"
- done
-
- # Create Device Provisioning service
- time az iot dps create -g $AZURE_RESOURCE_GROUP --name $AZURE_DPS_NAME -l $(AZURE_REGION)
time az iot dps linked-hub create -g $AZURE_RESOURCE_GROUP --dps-name $AZURE_DPS_NAME --connection-string "$IOTHUB_CONNECTION_STRING"
export DPS_CONNECTION_STRING=$(az iot dps connection-string show -g $AZURE_RESOURCE_GROUP --dps-name $AZURE_DPS_NAME --policy-name provisioningserviceowner --query "connectionString" --output tsv)
@@ -245,9 +249,9 @@ stages:
time az iot dps certificate verify --dps-name $AZURE_DPS_NAME --resource-group $AZURE_RESOURCE_GROUP --name $DPS_ROOT_CA_CERT_NAME --path $verification_code_pem_path --etag $etag
# Export variables to other stages/jobs/steps.
- echo "##vso[task.setvariable variable=IOTHUB_CONNECTION_STRING;isOutput=true]$IOTHUB_CONNECTION_STRING"
- echo "##vso[task.setvariable variable=EVENTHUB_CONNECTION_STRING;isOutput=true]$EVENTHUB_CONNECTION_STRING"
- echo "##vso[task.setvariable variable=DPS_CONNECTION_STRING;isOutput=true]$DPS_CONNECTION_STRING"
+ echo "##vso[task.setvariable variable=IOTHUB_CONNECTION_STRING;isOutput=true;isSecret=true]$IOTHUB_CONNECTION_STRING"
+ echo "##vso[task.setvariable variable=EVENTHUB_CONNECTION_STRING;isOutput=true;isSecret=true]$EVENTHUB_CONNECTION_STRING"
+ echo "##vso[task.setvariable variable=DPS_CONNECTION_STRING;isOutput=true;isSecret=true]$DPS_CONNECTION_STRING"
echo "##vso[task.setvariable variable=DPS_ID_SCOPE;isOutput=true]$DPS_ID_SCOPE"
echo "##vso[task.setvariable variable=DPS_HOSTNAME;isOutput=true]$DPS_HOSTNAME"
displayName: Create IoT Hub & DPS
@@ -273,11 +277,23 @@ stages:
versionSpec: '3.10'
architecture: 'x64'
+ # Cache downloads without reusing the job environment.
+ - task: Cache@2
+ displayName: 'Cache uv'
+ inputs:
+ key: '"uv" | "$(Agent.OS)" | "3.10" | uv.lock'
+ restoreKeys: |
+ "uv" | "$(Agent.OS)" | "3.10"
+ "uv" | "$(Agent.OS)"
+ path: $(Pipeline.Workspace)/.uv-cache
+
- script: 'python -m pip install uv'
displayName: 'Install uv'
- script: 'uv sync --upgrade --python 3.10 --no-dev --group test --group e2e'
displayName: 'Sync E2E environment'
+ env:
+ UV_CACHE_DIR: $(Pipeline.Workspace)/.uv-cache
- script: |
cd $(Agent.WorkFolder)
@@ -322,6 +338,7 @@ stages:
variables:
AZURE_RESOURCE_GROUP: $[stageDependencies.setup.create_azure_resources.outputs['setResourceNames.AZURE_RESOURCE_GROUP']]
steps:
+ - checkout: none
- task: AzureCLI@2
inputs:
azureSubscription: 'iot hub sdk service connection'
diff --git a/vsts/horton-e2e.yaml b/vsts/horton-e2e.yaml
index b31b726fa..ea96a34b8 100644
--- a/vsts/horton-e2e.yaml
+++ b/vsts/horton-e2e.yaml
@@ -1,3 +1,13 @@
+trigger:
+ branches:
+ include:
+ - main
+
+pr:
+ branches:
+ include:
+ - main
+
variables:
Horton.FrameworkRoot: $(Agent.BuildDirectory)/e2e-fx
Horton.FrameworkRef: master
diff --git a/vsts/python-canary.yaml b/vsts/python-canary.yaml
index 26e3aa31d..0edc2cfae 100644
--- a/vsts/python-canary.yaml
+++ b/vsts/python-canary.yaml
@@ -4,7 +4,8 @@ jobs:
- job: 'Test'
strategy:
- maxParallel: 4
+ # Each leg has an isolated consumer group, so all legs can run together.
+ maxParallel: 7
matrix:
py310_windows_mqtt:
pv: '3.10'
@@ -19,27 +20,27 @@ jobs:
py310_linux_mqttws:
pv: '3.10'
transport: 'mqttws'
- imageName: 'Ubuntu 20.04'
+ imageName: 'ubuntu-24.04'
consumerGroup: 'cg4'
py310_linux_mqtt:
pv: '3.10'
transport: 'mqtt'
- imageName: 'Ubuntu 20.04'
+ imageName: 'ubuntu-24.04'
consumerGroup: 'cg6'
py311_linux_mqtt:
pv: '3.11'
transport: 'mqtt'
- imageName: 'Ubuntu 20.04'
+ imageName: 'ubuntu-24.04'
consumerGroup: 'cg7'
py312_linux_mqtt:
pv: '3.12'
transport: 'mqtt'
- imageName: 'Ubuntu 20.04'
+ imageName: 'ubuntu-24.04'
consumerGroup: 'cg8'
py312_linux_mqttws:
pv: '3.12'
transport: 'mqttws'
- imageName: 'Ubuntu 20.04'
+ imageName: 'ubuntu-24.04'
consumerGroup: 'cg9'
pool:
@@ -51,11 +52,23 @@ jobs:
versionSpec: $(pv)
architecture: 'x64'
+ # Cache downloads without reusing the job environment.
+ - task: Cache@2
+ displayName: 'Cache uv'
+ inputs:
+ key: '"uv" | "$(Agent.OS)" | "$(pv)" | uv.lock'
+ restoreKeys: |
+ "uv" | "$(Agent.OS)" | "$(pv)"
+ "uv" | "$(Agent.OS)"
+ path: $(Pipeline.Workspace)/.uv-cache
+
- script: 'python -m pip install uv'
displayName: 'Install uv'
- script: 'uv sync --upgrade --python "$(pv)" --no-dev --group test --group e2e'
displayName: 'Sync E2E environment'
+ env:
+ UV_CACHE_DIR: $(Pipeline.Workspace)/.uv-cache
- script: |
cd $(Build.SourcesDirectory)/tests/e2e/iothub_e2e
diff --git a/vsts/python-e2e.yaml b/vsts/python-e2e.yaml
index b782f17eb..7fb8309b3 100644
--- a/vsts/python-e2e.yaml
+++ b/vsts/python-e2e.yaml
@@ -15,10 +15,11 @@ stages:
pool:
vmImage: 'ubuntu-24.04'
steps:
+ - checkout: none
- script: |
- sudo apt-get install -y uuid
- AZURE_RESOURCE_GROUP="aziotsdkpythone2e$(uuid | tr -d '-')"
- AZURE_IOTHUB_NAME="aziotsdkpythone2e$(uuid | tr -d '-')"
+ RESOURCE_SUFFIX=$(tr -d '-' < /proc/sys/kernel/random/uuid)
+ AZURE_RESOURCE_GROUP="aziotsdkpythone2e$RESOURCE_SUFFIX"
+ AZURE_IOTHUB_NAME="aziotsdkpythone2e$RESOURCE_SUFFIX"
# Export variables to other stages/jobs/steps.
echo "##vso[task.setvariable variable=AZURE_RESOURCE_GROUP;isOutput=true]$AZURE_RESOURCE_GROUP"
@@ -43,10 +44,26 @@ stages:
export IOTHUB_CONNECTION_STRING=$(az iot hub connection-string show -g $AZURE_RESOURCE_GROUP --hub-name $AZURE_IOTHUB_NAME --pn iothubowner --query "connectionString" --output tsv)
export EVENTHUB_CONNECTION_STRING=$(az iot hub connection-string show -g $AZURE_RESOURCE_GROUP --hub-name $AZURE_IOTHUB_NAME --eh --query "connectionString" --output tsv)
+ # Limit Azure API concurrency while creating independent groups faster.
+ wait_for_consumer_groups() {
+ local failed=0
+ for pid in "$@"; do
+ wait "$pid" || failed=1
+ done
+ return "$failed"
+ }
+
+ consumer_group_pids=()
for i in $(seq 1 5); do
cg_name="cg$i"
- az iot hub consumer-group create -g "$AZURE_RESOURCE_GROUP" --hub-name "$AZURE_IOTHUB_NAME" --name "$cg_name"
+ az iot hub consumer-group create -g "$AZURE_RESOURCE_GROUP" --hub-name "$AZURE_IOTHUB_NAME" --name "$cg_name" &
+ consumer_group_pids+=("$!")
+ if (( ${#consumer_group_pids[@]} == 4 )); then
+ wait_for_consumer_groups "${consumer_group_pids[@]}"
+ consumer_group_pids=()
+ fi
done
+ wait_for_consumer_groups "${consumer_group_pids[@]}"
# Route twin change events to the built-in Event Hubs-compatible endpoint
# so reported-properties E2E tests can observe service-side patch arrivals.
@@ -63,8 +80,8 @@ stages:
--route-name "twin-change-events"
# Export variables to other stages/jobs/steps.
- echo "##vso[task.setvariable variable=IOTHUB_CONNECTION_STRING;isOutput=true]$IOTHUB_CONNECTION_STRING"
- echo "##vso[task.setvariable variable=EVENTHUB_CONNECTION_STRING;isOutput=true]$EVENTHUB_CONNECTION_STRING"
+ echo "##vso[task.setvariable variable=IOTHUB_CONNECTION_STRING;isOutput=true;isSecret=true]$IOTHUB_CONNECTION_STRING"
+ echo "##vso[task.setvariable variable=EVENTHUB_CONNECTION_STRING;isOutput=true;isSecret=true]$EVENTHUB_CONNECTION_STRING"
displayName: Create IoT Hub & DPS
name: createAzureResources
- stage: build_and_tests
@@ -87,11 +104,23 @@ stages:
versionSpec: $(pv)
architecture: 'x64'
+ # Cache downloads without reusing the job environment.
+ - task: Cache@2
+ displayName: 'Cache uv'
+ inputs:
+ key: '"uv" | "$(Agent.OS)" | "$(pv)" | uv.lock'
+ restoreKeys: |
+ "uv" | "$(Agent.OS)" | "$(pv)"
+ "uv" | "$(Agent.OS)"
+ path: $(Pipeline.Workspace)/.uv-cache
+
- script: 'python -m pip install uv'
displayName: 'Install uv'
- script: 'uv sync --upgrade --python "$(pv)" --no-dev --group test --group e2e'
displayName: 'Sync E2E environment'
+ env:
+ UV_CACHE_DIR: $(Pipeline.Workspace)/.uv-cache
- script: |
cd $(Build.SourcesDirectory)/tests/e2e/iothub_e2e
@@ -126,11 +155,23 @@ stages:
versionSpec: $(pv)
architecture: 'x64'
+ # Cache downloads without reusing the job environment.
+ - task: Cache@2
+ displayName: 'Cache uv'
+ inputs:
+ key: '"uv" | "$(Agent.OS)" | "$(pv)" | uv.lock'
+ restoreKeys: |
+ "uv" | "$(Agent.OS)" | "$(pv)"
+ "uv" | "$(Agent.OS)"
+ path: $(Pipeline.Workspace)/.uv-cache
+
- script: 'python -m pip install uv'
displayName: 'Install uv'
- script: 'uv sync --upgrade --python "$(pv)" --no-dev --group test --group e2e'
displayName: 'Sync E2E environment'
+ env:
+ UV_CACHE_DIR: $(Pipeline.Workspace)/.uv-cache
- script: |
cd $(Build.SourcesDirectory)/tests/e2e/iothub_e2e
@@ -156,6 +197,7 @@ stages:
variables:
AZURE_RESOURCE_GROUP: $[stageDependencies.setup.create_iot_hub_and_dps.outputs['setResourceNames.AZURE_RESOURCE_GROUP']]
steps:
+ - checkout: none
- task: AzureCLI@2
inputs:
azureSubscription: 'iot hub sdk service connection'
@@ -164,8 +206,6 @@ stages:
inlineScript: |
# set -e
- printenv | sort | grep -i azure
-
time az group delete --name $(AZURE_RESOURCE_GROUP) --yes
displayName: Destroy Azure Resource Group
condition: always() # Run step even if the pipeline run is cancelled.
diff --git a/vsts/python-nightly.yaml b/vsts/python-nightly.yaml
index b576ebf72..870ef8ef8 100644
--- a/vsts/python-nightly.yaml
+++ b/vsts/python-nightly.yaml
@@ -9,10 +9,11 @@ stages:
pool:
vmImage: 'ubuntu-24.04'
steps:
+ - checkout: none
- script: |
- sudo apt-get install -y uuid
- AZURE_RESOURCE_GROUP="aziotsdkpython$(uuid)"
- AZURE_IOTHUB_NAME="aziotsdkpython$(uuid)"
+ RESOURCE_SUFFIX=$(tr -d '-' < /proc/sys/kernel/random/uuid)
+ AZURE_RESOURCE_GROUP="aziotsdkpython$RESOURCE_SUFFIX"
+ AZURE_IOTHUB_NAME="aziotsdkpython$RESOURCE_SUFFIX"
# Export variables to other stages/jobs/steps.
echo "##vso[task.setvariable variable=AZURE_RESOURCE_GROUP;isOutput=true]$AZURE_RESOURCE_GROUP"
@@ -37,10 +38,26 @@ stages:
export IOTHUB_CONNECTION_STRING=$(az iot hub connection-string show -g $AZURE_RESOURCE_GROUP --hub-name $AZURE_IOTHUB_NAME --pn iothubowner --query "connectionString" --output tsv)
export EVENTHUB_CONNECTION_STRING=$(az iot hub connection-string show -g $AZURE_RESOURCE_GROUP --hub-name $AZURE_IOTHUB_NAME --eh --query "connectionString" --output tsv)
+ # Limit Azure API concurrency while creating independent groups faster.
+ wait_for_consumer_groups() {
+ local failed=0
+ for pid in "$@"; do
+ wait "$pid" || failed=1
+ done
+ return "$failed"
+ }
+
+ consumer_group_pids=()
for i in $(seq 1 11); do
cg_name="cg$i"
- az iot hub consumer-group create -g "$AZURE_RESOURCE_GROUP" --hub-name "$AZURE_IOTHUB_NAME" --name "$cg_name"
+ az iot hub consumer-group create -g "$AZURE_RESOURCE_GROUP" --hub-name "$AZURE_IOTHUB_NAME" --name "$cg_name" &
+ consumer_group_pids+=("$!")
+ if (( ${#consumer_group_pids[@]} == 4 )); then
+ wait_for_consumer_groups "${consumer_group_pids[@]}"
+ consumer_group_pids=()
+ fi
done
+ wait_for_consumer_groups "${consumer_group_pids[@]}"
# Route twin change events to the built-in Event Hubs-compatible endpoint
# so reported-properties E2E tests can observe service-side patch arrivals.
@@ -57,8 +74,8 @@ stages:
--route-name "twin-change-events"
# Export variables to other stages/jobs/steps.
- echo "##vso[task.setvariable variable=IOTHUB_CONNECTION_STRING;isOutput=true]$IOTHUB_CONNECTION_STRING"
- echo "##vso[task.setvariable variable=EVENTHUB_CONNECTION_STRING;isOutput=true]$EVENTHUB_CONNECTION_STRING"
+ echo "##vso[task.setvariable variable=IOTHUB_CONNECTION_STRING;isOutput=true;isSecret=true]$IOTHUB_CONNECTION_STRING"
+ echo "##vso[task.setvariable variable=EVENTHUB_CONNECTION_STRING;isOutput=true;isSecret=true]$EVENTHUB_CONNECTION_STRING"
displayName: Create IoT Hub & DPS
name: createAzureResources
- stage: build_and_tests
@@ -79,7 +96,8 @@ stages:
IOTHUB_CONNECTION_STRING: $[stageDependencies.setup.create_iot_hub_and_dps.outputs['createAzureResources.IOTHUB_CONNECTION_STRING']]
EVENTHUB_CONNECTION_STRING: $[stageDependencies.setup.create_iot_hub_and_dps.outputs['createAzureResources.EVENTHUB_CONNECTION_STRING']]
strategy:
- maxParallel: 9
+ # Each leg has an isolated consumer group, so all legs can run together.
+ maxParallel: 11
matrix:
py310_windows_mqtt:
pv: '3.10'
@@ -147,11 +165,22 @@ stages:
versionSpec: $(pv)
architecture: 'x64'
addToPath: true
+ # Cache downloads without reusing the job environment.
+ - task: Cache@2
+ displayName: 'Cache uv'
+ inputs:
+ key: '"uv" | "$(Agent.OS)" | "$(pv)" | uv.lock'
+ restoreKeys: |
+ "uv" | "$(Agent.OS)" | "$(pv)"
+ "uv" | "$(Agent.OS)"
+ path: $(Pipeline.Workspace)/.uv-cache
- script: 'python -m pip install uv'
displayName: 'Install uv'
- script: 'uv sync --upgrade --python "$(pv)" --no-dev --group test --group e2e'
displayName: 'Sync E2E environment'
+ env:
+ UV_CACHE_DIR: $(Pipeline.Workspace)/.uv-cache
- script: |
cd $(Build.SourcesDirectory)/tests/e2e/iothub_e2e
@@ -180,6 +209,7 @@ stages:
variables:
AZURE_RESOURCE_GROUP: $[stageDependencies.setup.create_iot_hub_and_dps.outputs['setResourceNames.AZURE_RESOURCE_GROUP']]
steps:
+ - checkout: none
- task: AzureCLI@2
inputs:
azureSubscription: 'iot hub sdk service connection'
@@ -188,8 +218,6 @@ stages:
inlineScript: |
# set -e
- printenv | sort | grep -i azure
-
time az group delete --name $(AZURE_RESOURCE_GROUP) --yes
displayName: Destroy Azure Resource Group
condition: always() # Run step even if the pipeline run is cancelled.
\ No newline at end of file