Release 3.9.0

.github/workflows/hil_testing_nightly.yml

@@ -0,0 +1,28 @@
+name: DepthAI Core HIL Testing Nightly
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+# Only allow latest nightly run on same branch to be tested
+concurrency:
+  group: ci-nightly-tests-${{ github.ref }}-1
+  cancel-in-progress: true
+
+jobs:
+  run_ptp_fsync_tests:
+    uses: ./.github/workflows/test_child.yml
+    with:
+      flavor: "vanilla"
+      job_prefix: "vanilla-nightly"
+      run_standard_tests: false
+      luxonis_os_versions_to_test: "[]"
+      luxonis_os_versions_to_test_fsync: "['1.33.0']"
+      luxonis_os_versions_to_test_ptp: "['1.33.0']"
+    secrets:
+      CONTAINER_REGISTRY: ${{ secrets.CONTAINER_REGISTRY }}
+      HIL_PAT_TOKEN: ${{ secrets.HIL_PAT_TOKEN }}
+      CI_TELEMETRY_URL: ${{ secrets.CI_TELEMETRY_URL }}
+      CI_TELEMETRY_API_KEY: ${{ secrets.CI_TELEMETRY_API_KEY }}

.github/workflows/hil_testing_pull_request.yml

@@ -0,0 +1,62 @@
+name: DepthAI Core HIL Testing PR
+
+# Trigger on submitted reviews or when manually marked as testable
+on:
+  pull_request_review:
+    types: [submitted]
+  pull_request:
+    types: [labeled]
+
+permissions:
+  contents: read
+
+# Cancel older runs for the same PR and trigger.
+concurrency:
+  group: depthai-core-pr-testing-${{ github.event.pull_request.number }}-${{ github.event.review.state || github.event.label.name }}
+  cancel-in-progress: true
+
+jobs:
+  run_pr_tests:
+    name: Run Linux PR HIL tests
+    if: >
+      (
+        github.event.review.state == 'approved' ||
+        (github.event_name == 'pull_request' && github.event.label.name == 'testable')
+      ) &&
+      (
+        github.event.pull_request.base.ref == 'main' ||
+        github.event.pull_request.base.ref == 'develop'
+      )
+    uses: ./.github/workflows/test_child.yml
+    with:
+      flavor: "vanilla"
+      job_prefix: "pr-approved"
+      luxonis_os_versions_to_test: "['1.33.0']"
+    secrets:
+      CONTAINER_REGISTRY: ${{ secrets.CONTAINER_REGISTRY }}
+      HIL_PAT_TOKEN: ${{ secrets.HIL_PAT_TOKEN }}
+      CI_TELEMETRY_URL: ${{ secrets.CI_TELEMETRY_URL }}
+      CI_TELEMETRY_API_KEY: ${{ secrets.CI_TELEMETRY_API_KEY }}
+
+  required_hil_test_report:
+    name: Required HIL test report
+    needs: [run_pr_tests]
+    if: >
+      always() &&
+      (
+        github.event.review.state == 'approved' ||
+        (github.event_name == 'pull_request' && github.event.label.name == 'testable')
+      ) &&
+      (
+        github.event.pull_request.base.ref == 'main' ||
+        github.event.pull_request.base.ref == 'develop'
+      )
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check required test results
+      run: |
+        if [[ "${{ needs.run_pr_tests.result }}" != "success" ]]; then
+          echo "Required HIL tests failed or did not complete."
+          echo "run_pr_tests: ${{ needs.run_pr_tests.result }}"
+          exit 1
+        fi

.github/workflows/hil_testing_schedule_develop.yml

@@ -0,0 +1,45 @@
+name: DepthAI Core HIL Testing Develop Schedule
+
+on:
+  schedule:
+    - cron: '0 1 * * *' # Every day at 01:00 UTC
+    - cron: '0 3 * * 6' # Saturdays at 03:00 UTC
+
+permissions:
+  actions: write
+
+jobs:
+  dispatch:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger develop HIL testing
+        if: github.event.schedule == '0 3 * * 6'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: gh workflow run hil_testing_weekly.yml --repo "$GITHUB_REPOSITORY" --ref develop
+
+      - name: Trigger nightly develop HIL testing
+        if: github.event.schedule == '0 1 * * *'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: gh workflow run hil_testing_nightly.yml --repo "$GITHUB_REPOSITORY" --ref develop
+
+      - name: Notify Slack
+        if: github.event.schedule == '0 3 * * 6'
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          SLACK_BOT_CHANNEL_ID: ${{ secrets.SLACK_BOT_CHANNEL_ID_TEST }}
+          SCHEDULER_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          MESSAGE="*Scheduled DepthAI Core HIL tests triggered*
+          Branch: develop
+          Workflow: DepthAI Core HIL Testing
+          Scheduler Run: $SCHEDULER_RUN_URL"
+
+          jq -n --arg user_id "$SLACK_BOT_CHANNEL_ID" --arg message "$MESSAGE" \
+            '{"channel": $user_id, "text": $message}' > payload.json
+
+          curl -X POST https://slack.com/api/chat.postMessage \
+            -H "Authorization: Bearer $SLACK_BOT_TOKEN" \
+            -H "Content-type: application/json" \
+            --data @payload.json

.github/workflows/hil_testing_weekly.yml

@@ -0,0 +1,227 @@
+name: DepthAI Core HIL Testing Weekly
+
+on:
+  workflow_dispatch:
+    inputs:
+      test_os:
+        description: Run Linux, Windows, and macOS tests
+        type: boolean
+        default: true
+      test_replay:
+        description: Run replay tests
+        type: boolean
+        default: true
+      test_sanitizers:
+        description: Run sanitizer tests
+        type: boolean
+        default: true
+
+permissions:
+  actions: read
+  contents: read
+
+# Only allow latest run on same branch to be tested
+concurrency:
+  group: ci-tests-${{ github.ref }}-1
+  cancel-in-progress: true
+
+jobs:
+  select_tests:
+    runs-on: ubuntu-latest
+    outputs:
+      test_all: ${{ steps.selection.outputs.test_all }}
+    steps:
+      - id: selection
+        run: echo "test_all=${{ !inputs.test_os && !inputs.test_replay && !inputs.test_sanitizers }}" >> "$GITHUB_OUTPUT"
+
+  run_vanilla_tests:
+    needs: select_tests
+    if: inputs.test_os || needs.select_tests.outputs.test_all == 'true'
+    uses: ./.github/workflows/test_child.yml
+    with:
+      flavor: "vanilla"
+      luxonis_os_versions_to_test: "['1.20.5','1.27.1','1.33.0']"
+      luxonis_os_versions_to_test_rgb: "['1.33.0']"
+      luxonis_os_versions_to_test_usb: "['1.33.0']"
+    secrets:
+      CONTAINER_REGISTRY: ${{ secrets.CONTAINER_REGISTRY }}
+      HIL_PAT_TOKEN: ${{ secrets.HIL_PAT_TOKEN }}
+      CI_TELEMETRY_URL: ${{ secrets.CI_TELEMETRY_URL }}
+      CI_TELEMETRY_API_KEY: ${{ secrets.CI_TELEMETRY_API_KEY }}
+
+  run_tsan_tests:
+    needs: select_tests
+    if: inputs.test_sanitizers || needs.select_tests.outputs.test_all == 'true'
+    uses: ./.github/workflows/test_child.yml
+    with:
+      flavor: "tsan"
+      luxonis_os_versions_to_test: "['1.33.0']"
+    secrets:
+      CONTAINER_REGISTRY: ${{ secrets.CONTAINER_REGISTRY }}
+      HIL_PAT_TOKEN: ${{ secrets.HIL_PAT_TOKEN }}
+      CI_TELEMETRY_URL: ${{ secrets.CI_TELEMETRY_URL }}
+      CI_TELEMETRY_API_KEY: ${{ secrets.CI_TELEMETRY_API_KEY }}
+
+  run_asan-ubsan_tests:
+    needs: select_tests
+    if: inputs.test_sanitizers || needs.select_tests.outputs.test_all == 'true'
+    uses: ./.github/workflows/test_child.yml
+    with:
+      flavor: "asan-ubsan"
+      luxonis_os_versions_to_test: "['1.33.0']"
+    secrets:
+      CONTAINER_REGISTRY: ${{ secrets.CONTAINER_REGISTRY }}
+      HIL_PAT_TOKEN: ${{ secrets.HIL_PAT_TOKEN }}
+      CI_TELEMETRY_URL: ${{ secrets.CI_TELEMETRY_URL }}
+      CI_TELEMETRY_API_KEY: ${{ secrets.CI_TELEMETRY_API_KEY }}
+
+  run_windows_tests:
+    needs: select_tests
+    if: inputs.test_os || needs.select_tests.outputs.test_all == 'true'
+    uses: ./.github/workflows/test_child_windows.yml
+    with:
+      luxonis_os_versions_to_test: "['1.33.0']"
+      luxonis_os_versions_to_test_usb: '["1.33.0"]'
+    secrets:
+      CONTAINER_REGISTRY: ${{ secrets.CONTAINER_REGISTRY }}
+      HIL_PAT_TOKEN: ${{ secrets.HIL_PAT_TOKEN }}
+      CI_TELEMETRY_URL: ${{ secrets.CI_TELEMETRY_URL }}
+      CI_TELEMETRY_API_KEY: ${{ secrets.CI_TELEMETRY_API_KEY }}
+
+  run_replay_tests:
+    needs: select_tests
+    if: inputs.test_replay || needs.select_tests.outputs.test_all == 'true'
+    uses: ./.github/workflows/test_child.yml
+    with:
+      flavor: "vanilla"
+      job_prefix: "vanilla-replay"
+      enable_replay_tests: true
+      rvc2_timeout_minutes: 1440
+      luxonis_os_versions_to_test: "['1.33.0']"
+    secrets:
+      CONTAINER_REGISTRY: ${{ secrets.CONTAINER_REGISTRY }}
+      HIL_PAT_TOKEN: ${{ secrets.HIL_PAT_TOKEN }}
+      CI_TELEMETRY_URL: ${{ secrets.CI_TELEMETRY_URL }}
+      CI_TELEMETRY_API_KEY: ${{ secrets.CI_TELEMETRY_API_KEY }}
+
+  run_vanilla_mac_tests:
+    needs: select_tests
+    if: inputs.test_os || needs.select_tests.outputs.test_all == 'true'
+    uses: ./.github/workflows/test_child_mac.yml
+    with:
+      flavor: "vanilla"
+      luxonis_os_versions_to_test: "['1.33.0']"
+      luxonis_os_versions_to_test_usb: '["1.33.0"]'
+    secrets:
+      HIL_PAT_TOKEN: ${{ secrets.HIL_PAT_TOKEN }}
+      CI_TELEMETRY_URL: ${{ secrets.CI_TELEMETRY_URL }}
+      CI_TELEMETRY_API_KEY: ${{ secrets.CI_TELEMETRY_API_KEY }}
+
+  notify_slack:
+    needs:
+      - select_tests
+      - run_vanilla_tests
+      - run_tsan_tests
+      - run_asan-ubsan_tests
+      - run_windows_tests
+      - run_replay_tests
+      - run_vanilla_mac_tests
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download JUnit reports
+        uses: actions/download-artifact@v4
+        continue-on-error: true
+        with:
+          pattern: junit-reports-*
+          path: junit-reports
+
+      - name: Send Slack test summary
+        continue-on-error: true
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          SLACK_BOT_CHANNEL_ID: ${{ secrets.SLACK_BOT_CHANNEL_ID_TEST }}
+          NEEDS_JSON: ${{ toJson(needs) }}
+          RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          SUMMARY=$(jq -r '
+            def job_label:
+              {
+                run_vanilla_tests: "Linux",
+                run_tsan_tests: "TSan",
+                "run_asan-ubsan_tests": "ASan/UBSan",
+                run_windows_tests: "Windows",
+                run_replay_tests: "Replay",
+                run_vanilla_mac_tests: "macOS"
+              }[.] // .;
+
+            to_entries
+            | map(select(.key != "select_tests"))
+            | map("- " + (.key | job_label) + ": `" + .value.result + "`")
+            | join("\n")
+          ' <<< "$NEEDS_JSON")
+
+          FAILED_COUNT=$(jq '[to_entries[].value.result | select(. == "failure" or . == "cancelled")] | length' <<< "$NEEDS_JSON")
+          CTEST_SUMMARY=$(python3 - <<'PY'
+          from pathlib import Path
+          from xml.etree import ElementTree as ET
+          import re
+
+          passed = failed = total = 0
+          failed_tests = []
+          summary_re = re.compile(r"Passed=(\d+), Failed=(\d+), Total=(\d+)")
+
+          for report in Path("junit-reports").rglob("*.xml"):
+              root = ET.parse(report).getroot()
+              for suite in root.findall(".//testsuite"):
+                  for prop in suite.findall("./properties/property"):
+                      if prop.get("name") != "ctest.summary":
+                          continue
+                      match = summary_re.fullmatch(prop.get("value", ""))
+                      if match:
+                          suite_passed, suite_failed, suite_total = map(int, match.groups())
+                          passed += suite_passed
+                          failed += suite_failed
+                          total += suite_total
+                  for case in suite.findall("./testcase"):
+                      if case.find("./failure") is not None:
+                          failed_tests.append(f"{suite.get('name', report.stem)} - {case.get('name', 'unknown')}")
+
+          if total == 0:
+              print("CTest reports: no summaries found")
+          else:
+              print(f"CTest totals: {passed} passed, {failed} failed, {total} total")
+              if failed_tests:
+                  print("Failed tests:")
+                  for test in failed_tests[:10]:
+                      print(f"- {test}")
+                  if len(failed_tests) > 10:
+                      print(f"- ...and {len(failed_tests) - 10} more")
+          PY
+          )
+
+          if [ "$FAILED_COUNT" -gt 0 ]; then
+            RESULT="failed"
+          else
+            RESULT="passed"
+          fi
+
+          MESSAGE="*DepthAI Core HIL tests finished*
+          Branch: $GITHUB_REF_NAME
+          Result: $RESULT
+          Run: $RUN_URL
+
+          $SUMMARY"
+          if [ -n "$CTEST_SUMMARY" ]; then
+            MESSAGE="$MESSAGE
+
+          $CTEST_SUMMARY"
+          fi
+
+          jq -n --arg channel "$SLACK_BOT_CHANNEL_ID" --arg message "$MESSAGE" \
+            '{"channel": $channel, "text": $message}' > payload.json
+
+          curl -X POST https://slack.com/api/chat.postMessage \
+            -H "Authorization: Bearer $SLACK_BOT_TOKEN" \
+            -H "Content-type: application/json" \
+            --data @payload.json