|
1 | | -# Run all tests, including flaky test. |
2 | | -# |
3 | | -# The default CI workflow ignores flaky tests. This workflow will run |
4 | | -# all tests, including ignored ones. |
5 | | -# |
6 | | -# To use this workflow you can either: |
7 | | -# |
8 | | -# - Label a PR with "flaky-test", the normal CI workflow will not run |
9 | | -# any jobs but the jobs here will be run. Note that to merge the PR |
10 | | -# you'll need to remove the label eventually because the normal CI |
11 | | -# jobs are required by branch protection. |
12 | | -# |
13 | | -# - Manually trigger the workflow, you may choose a branch for this to |
14 | | -# run on. |
15 | | -# |
16 | | -# Additionally this jobs runs once a day on a schedule. |
17 | | -# |
18 | | -# Currently doctests are not run by this workflow. |
| 1 | +# # Run all tests, including flaky test. |
| 2 | +# # |
| 3 | +# # The default CI workflow ignores flaky tests. This workflow will run |
| 4 | +# # all tests, including ignored ones. |
| 5 | +# # |
| 6 | +# # To use this workflow you can either: |
| 7 | +# # |
| 8 | +# # - Label a PR with "flaky-test", the normal CI workflow will not run |
| 9 | +# # any jobs but the jobs here will be run. Note that to merge the PR |
| 10 | +# # you'll need to remove the label eventually because the normal CI |
| 11 | +# # jobs are required by branch protection. |
| 12 | +# # |
| 13 | +# # - Manually trigger the workflow, you may choose a branch for this to |
| 14 | +# # run on. |
| 15 | +# # |
| 16 | +# # Additionally this jobs runs once a day on a schedule. |
| 17 | +# # |
| 18 | +# # Currently doctests are not run by this workflow. |
19 | 19 |
|
20 | | -name: Flaky CI |
| 20 | +# name: Flaky CI |
21 | 21 |
|
22 | | -on: |
23 | | - pull_request: |
24 | | - types: [ 'labeled', 'unlabeled', 'opened', 'synchronize', 'reopened' ] |
25 | | - schedule: |
26 | | - # 06:30 UTC every day |
27 | | - - cron: '30 6 * * *' |
28 | | - workflow_dispatch: |
29 | | - inputs: |
30 | | - branch: |
31 | | - description: 'Branch to run on, defaults to main' |
32 | | - required: true |
33 | | - default: 'main' |
34 | | - type: string |
| 22 | +# on: |
| 23 | +# pull_request: |
| 24 | +# types: [ 'labeled', 'unlabeled', 'opened', 'synchronize', 'reopened' ] |
| 25 | +# schedule: |
| 26 | +# # 06:30 UTC every day |
| 27 | +# - cron: '30 6 * * *' |
| 28 | +# workflow_dispatch: |
| 29 | +# inputs: |
| 30 | +# branch: |
| 31 | +# description: 'Branch to run on, defaults to main' |
| 32 | +# required: true |
| 33 | +# default: 'main' |
| 34 | +# type: string |
35 | 35 |
|
36 | | -concurrency: |
37 | | - group: flaky-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} |
38 | | - cancel-in-progress: true |
| 36 | +# concurrency: |
| 37 | +# group: flaky-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} |
| 38 | +# cancel-in-progress: true |
39 | 39 |
|
40 | | -env: |
41 | | - IROH_FORCE_STAGING_RELAYS: "1" |
| 40 | +# env: |
| 41 | +# IROH_FORCE_STAGING_RELAYS: "1" |
42 | 42 |
|
43 | | -jobs: |
44 | | - tests: |
45 | | - if: "contains(github.event.pull_request.labels.*.name, 'flaky-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'" |
46 | | - uses: './.github/workflows/tests.yaml' |
47 | | - with: |
48 | | - flaky: true |
49 | | - git-ref: ${{ inputs.branch }} |
50 | | - notify: |
51 | | - needs: tests |
52 | | - if: ${{ always() }} |
53 | | - runs-on: ubuntu-latest |
54 | | - steps: |
55 | | - - name: Extract test results |
56 | | - run: | |
57 | | - printf '${{ toJSON(needs) }}\n' |
58 | | - result=$(echo '${{ toJSON(needs) }}' | jq -r .tests.result) |
59 | | - echo TESTS_RESULT=$result |
60 | | - echo "TESTS_RESULT=$result" >>"$GITHUB_ENV" |
61 | | - - name: download nextest reports |
62 | | - uses: actions/download-artifact@v4 |
63 | | - with: |
64 | | - pattern: libtest_run_${{ github.run_number }}-${{ github.run_attempt }}-* |
65 | | - merge-multiple: true |
66 | | - path: nextest-results |
67 | | - - name: create summary report |
68 | | - id: make_summary |
69 | | - run: | |
70 | | - # prevent the glob expression in the loop to match on itself when the dir is empty |
71 | | - shopt -s nullglob |
72 | | - # to deal with multiline outputs it's recommended to use a random EOF, the syntax is based on |
73 | | - # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings |
74 | | - EOF=aP51VriWCxNJ1JjvmO9i |
75 | | - echo "summary<<$EOF" >> $GITHUB_OUTPUT |
76 | | - echo "Flaky tests failure:" >> $GITHUB_OUTPUT |
77 | | - echo " " >> $GITHUB_OUTPUT |
78 | | - for report in nextest-results/*.json; do |
79 | | - # remove the name prefix and extension, and split the parts |
80 | | - name=$(echo ${report:16:-5} | tr _ ' ') |
81 | | - echo $name |
82 | | - echo "- **$name**" >> $GITHUB_OUTPUT |
83 | | - # select the failed tests |
84 | | - # the tests have this format "crate::module$test_name", the sed expressions remove the quotes and replace $ for :: |
85 | | - failure=$(jq --slurp '.[] | select(.["type"] == "test" and .["event"] == "failed" ) | .["name"]' $report | sed -e 's/^"//g' -e 's/\$/::/' -e 's/"//') |
86 | | - echo "$failure" |
87 | | - echo "$failure" >> $GITHUB_OUTPUT |
88 | | - done |
89 | | - echo "" >> $GITHUB_OUTPUT |
90 | | - echo "See https://github.com/${{ github.repository }}/actions/workflows/flaky.yaml" >> $GITHUB_OUTPUT |
91 | | - echo "$EOF" >> $GITHUB_OUTPUT |
92 | | - - name: Notify discord on failure |
93 | | - uses: n0-computer/discord-webhook-notify@v1 |
94 | | - if: ${{ env.TESTS_RESULT == 'failure' || env.TESTS_RESULT == 'success' }} |
95 | | - with: |
96 | | - text: "Flaky tests in **${{ github.repository }}**:" |
97 | | - severity: ${{ env.TESTS_RESULT == 'failure' && 'warn' || 'info' }} |
98 | | - details: ${{ env.TESTS_RESULT == 'failure' && steps.make_summary.outputs.summary || 'No flaky failures!' }} |
99 | | - webhookUrl: ${{ secrets.DISCORD_N0_GITHUB_CHANNEL_WEBHOOK_URL }} |
| 43 | +# jobs: |
| 44 | +# tests: |
| 45 | +# if: "contains(github.event.pull_request.labels.*.name, 'flaky-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'" |
| 46 | +# uses: './.github/workflows/tests.yaml' |
| 47 | +# with: |
| 48 | +# flaky: true |
| 49 | +# git-ref: ${{ inputs.branch }} |
| 50 | +# notify: |
| 51 | +# needs: tests |
| 52 | +# if: ${{ always() }} |
| 53 | +# runs-on: ubuntu-latest |
| 54 | +# steps: |
| 55 | +# - name: Extract test results |
| 56 | +# run: | |
| 57 | +# printf '${{ toJSON(needs) }}\n' |
| 58 | +# result=$(echo '${{ toJSON(needs) }}' | jq -r .tests.result) |
| 59 | +# echo TESTS_RESULT=$result |
| 60 | +# echo "TESTS_RESULT=$result" >>"$GITHUB_ENV" |
| 61 | +# - name: download nextest reports |
| 62 | +# uses: actions/download-artifact@v4 |
| 63 | +# with: |
| 64 | +# pattern: libtest_run_${{ github.run_number }}-${{ github.run_attempt }}-* |
| 65 | +# merge-multiple: true |
| 66 | +# path: nextest-results |
| 67 | +# - name: create summary report |
| 68 | +# id: make_summary |
| 69 | +# run: | |
| 70 | +# # prevent the glob expression in the loop to match on itself when the dir is empty |
| 71 | +# shopt -s nullglob |
| 72 | +# # to deal with multiline outputs it's recommended to use a random EOF, the syntax is based on |
| 73 | +# # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings |
| 74 | +# EOF=aP51VriWCxNJ1JjvmO9i |
| 75 | +# echo "summary<<$EOF" >> $GITHUB_OUTPUT |
| 76 | +# echo "Flaky tests failure:" >> $GITHUB_OUTPUT |
| 77 | +# echo " " >> $GITHUB_OUTPUT |
| 78 | +# for report in nextest-results/*.json; do |
| 79 | +# # remove the name prefix and extension, and split the parts |
| 80 | +# name=$(echo ${report:16:-5} | tr _ ' ') |
| 81 | +# echo $name |
| 82 | +# echo "- **$name**" >> $GITHUB_OUTPUT |
| 83 | +# # select the failed tests |
| 84 | +# # the tests have this format "crate::module$test_name", the sed expressions remove the quotes and replace $ for :: |
| 85 | +# failure=$(jq --slurp '.[] | select(.["type"] == "test" and .["event"] == "failed" ) | .["name"]' $report | sed -e 's/^"//g' -e 's/\$/::/' -e 's/"//') |
| 86 | +# echo "$failure" |
| 87 | +# echo "$failure" >> $GITHUB_OUTPUT |
| 88 | +# done |
| 89 | +# echo "" >> $GITHUB_OUTPUT |
| 90 | +# echo "See https://github.com/${{ github.repository }}/actions/workflows/flaky.yaml" >> $GITHUB_OUTPUT |
| 91 | +# echo "$EOF" >> $GITHUB_OUTPUT |
| 92 | +# - name: Notify discord on failure |
| 93 | +# uses: n0-computer/discord-webhook-notify@v1 |
| 94 | +# if: ${{ env.TESTS_RESULT == 'failure' || env.TESTS_RESULT == 'success' }} |
| 95 | +# with: |
| 96 | +# text: "Flaky tests in **${{ github.repository }}**:" |
| 97 | +# severity: ${{ env.TESTS_RESULT == 'failure' && 'warn' || 'info' }} |
| 98 | +# details: ${{ env.TESTS_RESULT == 'failure' && steps.make_summary.outputs.summary || 'No flaky failures!' }} |
| 99 | +# webhookUrl: ${{ secrets.DISCORD_N0_GITHUB_CHANNEL_WEBHOOK_URL }} |
0 commit comments