Repository navigation
DepthAI Core HIL Testing Weekly #9
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: DepthAI Core HIL Testing Weekly | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| test_os: | |
| description: Run Linux, Windows, and macOS tests | |
| type: boolean | |
| default: true | |
| test_replay: | |
| description: Run replay tests | |
| type: boolean | |
| default: true | |
| test_sanitizers: | |
| description: Run sanitizer tests | |
| type: boolean | |
| default: true | |
| permissions: | |
| actions: read | |
| contents: read | |
| # Only allow latest run on same branch to be tested | |
| concurrency: | |
| group: ci-tests-${{ github.ref }}-1 | |
| cancel-in-progress: true | |
| jobs: | |
| select_tests: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| test_all: ${{ steps.selection.outputs.test_all }} | |
| steps: | |
| - id: selection | |
| run: echo "test_all=${{ !inputs.test_os && !inputs.test_replay && !inputs.test_sanitizers }}" >> "$GITHUB_OUTPUT" | |
| run_vanilla_tests: | |
| needs: select_tests | |
| if: inputs.test_os || needs.select_tests.outputs.test_all == 'true' | |
| uses: ./.github/workflows/test_child.yml | |
| with: | |
| flavor: "vanilla" | |
| luxonis_os_versions_to_test: "['1.20.5','1.27.1','1.33.0']" | |
| luxonis_os_versions_to_test_rgb: "['1.33.0']" | |
| luxonis_os_versions_to_test_usb: "['1.33.0']" | |
| secrets: | |
| CONTAINER_REGISTRY: ${{ secrets.CONTAINER_REGISTRY }} | |
| HIL_PAT_TOKEN: ${{ secrets.HIL_PAT_TOKEN }} | |
| CI_TELEMETRY_URL: ${{ secrets.CI_TELEMETRY_URL }} | |
| CI_TELEMETRY_API_KEY: ${{ secrets.CI_TELEMETRY_API_KEY }} | |
| run_tsan_tests: | |
| needs: select_tests | |
| if: inputs.test_sanitizers || needs.select_tests.outputs.test_all == 'true' | |
| uses: ./.github/workflows/test_child.yml | |
| with: | |
| flavor: "tsan" | |
| luxonis_os_versions_to_test: "['1.33.0']" | |
| secrets: | |
| CONTAINER_REGISTRY: ${{ secrets.CONTAINER_REGISTRY }} | |
| HIL_PAT_TOKEN: ${{ secrets.HIL_PAT_TOKEN }} | |
| CI_TELEMETRY_URL: ${{ secrets.CI_TELEMETRY_URL }} | |
| CI_TELEMETRY_API_KEY: ${{ secrets.CI_TELEMETRY_API_KEY }} | |
| run_asan-ubsan_tests: | |
| needs: select_tests | |
| if: inputs.test_sanitizers || needs.select_tests.outputs.test_all == 'true' | |
| uses: ./.github/workflows/test_child.yml | |
| with: | |
| flavor: "asan-ubsan" | |
| luxonis_os_versions_to_test: "['1.33.0']" | |
| secrets: | |
| CONTAINER_REGISTRY: ${{ secrets.CONTAINER_REGISTRY }} | |
| HIL_PAT_TOKEN: ${{ secrets.HIL_PAT_TOKEN }} | |
| CI_TELEMETRY_URL: ${{ secrets.CI_TELEMETRY_URL }} | |
| CI_TELEMETRY_API_KEY: ${{ secrets.CI_TELEMETRY_API_KEY }} | |
| run_windows_tests: | |
| needs: select_tests | |
| if: inputs.test_os || needs.select_tests.outputs.test_all == 'true' | |
| uses: ./.github/workflows/test_child_windows.yml | |
| with: | |
| luxonis_os_versions_to_test: "['1.33.0']" | |
| luxonis_os_versions_to_test_usb: '["1.33.0"]' | |
| secrets: | |
| CONTAINER_REGISTRY: ${{ secrets.CONTAINER_REGISTRY }} | |
| HIL_PAT_TOKEN: ${{ secrets.HIL_PAT_TOKEN }} | |
| CI_TELEMETRY_URL: ${{ secrets.CI_TELEMETRY_URL }} | |
| CI_TELEMETRY_API_KEY: ${{ secrets.CI_TELEMETRY_API_KEY }} | |
| run_replay_tests: | |
| needs: select_tests | |
| if: inputs.test_replay || needs.select_tests.outputs.test_all == 'true' | |
| uses: ./.github/workflows/test_child.yml | |
| with: | |
| flavor: "vanilla" | |
| job_prefix: "vanilla-replay" | |
| enable_replay_tests: true | |
| rvc2_timeout_minutes: 1440 | |
| luxonis_os_versions_to_test: "['1.33.0']" | |
| secrets: | |
| CONTAINER_REGISTRY: ${{ secrets.CONTAINER_REGISTRY }} | |
| HIL_PAT_TOKEN: ${{ secrets.HIL_PAT_TOKEN }} | |
| CI_TELEMETRY_URL: ${{ secrets.CI_TELEMETRY_URL }} | |
| CI_TELEMETRY_API_KEY: ${{ secrets.CI_TELEMETRY_API_KEY }} | |
| run_vanilla_mac_tests: | |
| needs: select_tests | |
| if: inputs.test_os || needs.select_tests.outputs.test_all == 'true' | |
| uses: ./.github/workflows/test_child_mac.yml | |
| with: | |
| flavor: "vanilla" | |
| luxonis_os_versions_to_test: "['1.33.0']" | |
| luxonis_os_versions_to_test_usb: '["1.33.0"]' | |
| secrets: | |
| HIL_PAT_TOKEN: ${{ secrets.HIL_PAT_TOKEN }} | |
| CI_TELEMETRY_URL: ${{ secrets.CI_TELEMETRY_URL }} | |
| CI_TELEMETRY_API_KEY: ${{ secrets.CI_TELEMETRY_API_KEY }} | |
| notify_slack: | |
| needs: | |
| - select_tests | |
| - run_vanilla_tests | |
| - run_tsan_tests | |
| - run_asan-ubsan_tests | |
| - run_windows_tests | |
| - run_replay_tests | |
| - run_vanilla_mac_tests | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Download JUnit reports | |
| uses: actions/download-artifact@v4 | |
| continue-on-error: true | |
| with: | |
| pattern: junit-reports-* | |
| path: junit-reports | |
| - name: Send Slack test summary | |
| continue-on-error: true | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| SLACK_BOT_CHANNEL_ID: ${{ secrets.SLACK_BOT_CHANNEL_ID_TEST }} | |
| NEEDS_JSON: ${{ toJson(needs) }} | |
| RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| SUMMARY=$(jq -r ' | |
| def job_label: | |
| { | |
| run_vanilla_tests: "Linux", | |
| run_tsan_tests: "TSan", | |
| "run_asan-ubsan_tests": "ASan/UBSan", | |
| run_windows_tests: "Windows", | |
| run_replay_tests: "Replay", | |
| run_vanilla_mac_tests: "macOS" | |
| }[.] // .; | |
| to_entries | |
| | map(select(.key != "select_tests")) | |
| | map("- " + (.key | job_label) + ": `" + .value.result + "`") | |
| | join("\n") | |
| ' <<< "$NEEDS_JSON") | |
| FAILED_COUNT=$(jq '[to_entries[].value.result | select(. == "failure" or . == "cancelled")] | length' <<< "$NEEDS_JSON") | |
| CTEST_SUMMARY=$(python3 - <<'PY' | |
| from pathlib import Path | |
| from xml.etree import ElementTree as ET | |
| import re | |
| passed = failed = total = 0 | |
| failed_tests = [] | |
| summary_re = re.compile(r"Passed=(\d+), Failed=(\d+), Total=(\d+)") | |
| for report in Path("junit-reports").rglob("*.xml"): | |
| root = ET.parse(report).getroot() | |
| for suite in root.findall(".//testsuite"): | |
| for prop in suite.findall("./properties/property"): | |
| if prop.get("name") != "ctest.summary": | |
| continue | |
| match = summary_re.fullmatch(prop.get("value", "")) | |
| if match: | |
| suite_passed, suite_failed, suite_total = map(int, match.groups()) | |
| passed += suite_passed | |
| failed += suite_failed | |
| total += suite_total | |
| for case in suite.findall("./testcase"): | |
| if case.find("./failure") is not None: | |
| failed_tests.append(f"{suite.get('name', report.stem)} - {case.get('name', 'unknown')}") | |
| if total == 0: | |
| print("CTest reports: no summaries found") | |
| else: | |
| print(f"CTest totals: {passed} passed, {failed} failed, {total} total") | |
| if failed_tests: | |
| print("Failed tests:") | |
| for test in failed_tests[:10]: | |
| print(f"- {test}") | |
| if len(failed_tests) > 10: | |
| print(f"- ...and {len(failed_tests) - 10} more") | |
| PY | |
| ) | |
| if [ "$FAILED_COUNT" -gt 0 ]; then | |
| RESULT="failed" | |
| else | |
| RESULT="passed" | |
| fi | |
| MESSAGE="*DepthAI Core HIL tests finished* | |
| Branch: $GITHUB_REF_NAME | |
| Result: $RESULT | |
| Run: $RUN_URL | |
| $SUMMARY" | |
| if [ -n "$CTEST_SUMMARY" ]; then | |
| MESSAGE="$MESSAGE | |
| $CTEST_SUMMARY" | |
| fi | |
| jq -n --arg channel "$SLACK_BOT_CHANNEL_ID" --arg message "$MESSAGE" \ | |
| '{"channel": $channel, "text": $message}' > payload.json | |
| curl -X POST https://slack.com/api/chat.postMessage \ | |
| -H "Authorization: Bearer $SLACK_BOT_TOKEN" \ | |
| -H "Content-type: application/json" \ | |
| --data @payload.json |