Skip to content

Commit aa6d3fc

Browse files
apostasieapostasie
committed
System logs collector
Signed-off-by: apostasie <[email protected]>
1 parent 8e77757 commit aa6d3fc

33 files changed

+1475
-30
lines changed

‎.github/workflows/job-test-in-container.yml

Lines changed: 45 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ on:
3535
required: false
3636
default: false
3737
type: boolean
38+
outputs:
39+
artifact:
40+
description: "Artifact generated by this job"
41+
value: ${{ jobs.test.outputs.artifact }}
3842

3943
env:
4044
GOTOOLCHAIN: local
@@ -55,6 +59,8 @@ jobs:
5559
defaults:
5660
run:
5761
shell: bash
62+
outputs:
63+
artifact: ${{ steps.artifact-upload.outputs.artifact-url }}
5864

5965
env:
6066
# https://github.com/containerd/nerdctl/issues/622
@@ -161,9 +167,9 @@ jobs:
161167
&& args=(test-integration ./hack/test-integration.sh -test.allow-modify-users=true) \
162168
|| args=(test-integration-${{ inputs.target }} /test-integration-rootless.sh ./hack/test-integration.sh)
163169
if [ "${{ inputs.ipv6 }}" == true ]; then
164-
docker run --network host -t --rm --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=false -test.only-ipv6 -test.target=${{ inputs.binary }}
170+
docker run --name test-runner --network host -t --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=false -test.only-ipv6 -test.target=${{ inputs.binary }}
165171
else
166-
docker run -t --rm --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=false -test.target=${{ inputs.binary }}
172+
docker run --name test-runner -t --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=false -test.target=${{ inputs.binary }}
167173
fi
168174
# FIXME: this NEEDS to go away
169175
- name: "Run: integration tests (flaky)"
@@ -175,7 +181,42 @@ jobs:
175181
&& args=(test-integration ./hack/test-integration.sh) \
176182
|| args=(test-integration-${{ inputs.target }} /test-integration-rootless.sh ./hack/test-integration.sh)
177183
if [ "${{ inputs.ipv6 }}" == true ]; then
178-
docker run --network host -t --rm --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=true -test.only-ipv6 -test.target=${{ inputs.binary }}
184+
docker run --name test-runner-flaky --network host -t --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=true -test.only-ipv6 -test.target=${{ inputs.binary }}
179185
else
180-
docker run -t --rm --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=true -test.target=${{ inputs.binary }}
186+
docker run --name test-runner-flaky -t --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=true -test.target=${{ inputs.binary }}
181187
fi
188+
189+
- name: "Wrap: collector"
190+
if: ${{ failure() || success() }}
191+
run: |
192+
# Get the reports from inside the containers
193+
[ "${{ inputs.target }}" == "rootful" ] && src=/root || src=/home/rootless
194+
mkdir -p ~/report
195+
docker cp test-runner:$src/nerdctl-test-report ~/report/main || true
196+
# Flaky may not have run
197+
docker cp test-runner-flaky:$src/nerdctl-test-report ~/report/flaky 2>/dev/null || true
198+
# Add metadata info to the runs
199+
. ./mod/wax/scripts/collector.sh
200+
collect::metadata \
201+
"runner=${{ inputs.runner }}" \
202+
"binary=${{ inputs.binary }}" \
203+
"canary=${{ inputs.canary }}" \
204+
"target=${{ inputs.target }}" \
205+
"ipv6=${{ inputs.ipv6 }}" \
206+
"containerd-version=${{ inputs.containerd-version }}" \
207+
"rootlesskit-version=${{ inputs.rootlesskit-version }}" \
208+
"attempt=$GITHUB_RUN_ATTEMPT" \
209+
"sha=$GITHUB_SHA" \
210+
"id=$GITHUB_RUN_ID" \
211+
"number=$GITHUB_RUN_NUMBER" \
212+
> ~/report/main/metadata.json || true
213+
[ ! -e ~/report/flaky ] || cp ~/report/main/metadata.json ~/report/flaky/metadata.json || true
214+
215+
- name: "Wrap: upload"
216+
id: artifact-upload
217+
if: ${{ failure() || success() }}
218+
uses: actions/upload-artifact@v4
219+
with:
220+
path: ~/report/*
221+
retention-days: 1
222+
name: wax-${{ inputs.runner }}-${{ inputs.binary }}-${{ inputs.canary }}-${{ inputs.target }}-${{ inputs.ipv6 }}-${{ inputs.containerd-version }}-${{ inputs.rootlesskit-version }}

‎.github/workflows/job-test-in-host.yml

Lines changed: 57 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@ on:
4343
linux-cni-sha:
4444
required: true
4545
type: string
46+
outputs:
47+
artifact:
48+
description: "Artifact generated by this job"
49+
value: ${{ jobs.test.outputs.artifact }}
4650

4751
env:
4852
GOTOOLCHAIN: local
@@ -60,6 +64,8 @@ jobs:
6064
defaults:
6165
run:
6266
shell: bash
67+
outputs:
68+
artifact: ${{ steps.artifact-upload.outputs.artifact-url }}
6369

6470
env:
6571
SHOULD_RUN: "yes"
@@ -107,6 +113,7 @@ jobs:
107113
if [ "${{ contains(inputs.binary, 'docker') }}" == true ]; then
108114
echo "::group:: configure cdi for docker"
109115
sudo mkdir -p /etc/docker
116+
sudo touch /etc/docker/daemon.json
110117
sudo jq '.features.cdi = true' /etc/docker/daemon.json | sudo tee /etc/docker/daemon.json.tmp && sudo mv /etc/docker/daemon.json.tmp /etc/docker/daemon.json
111118
echo "::endgroup::"
112119
echo "::group:: downgrade docker to the specific version we want to test (${{ inputs.docker-version }})"
@@ -183,22 +190,30 @@ jobs:
183190
make install-dev-tools
184191
echo "::endgroup::"
185192
193+
194+
- if: ${{ env.SHOULD_RUN == 'yes' }}
195+
name: "Init: prepare artifacts directories"
196+
run: |
197+
mkdir -p ~/report/main
198+
mkdir -p ~/report/ipv6
199+
mkdir -p ~/report/flaky
200+
186201
# ipv6 is tested only on linux
187-
- if: ${{ contains(inputs.runner, 'ubuntu') && env.SHOULD_RUN == 'yes' }}
202+
- if: ${{ env.SHOULD_RUN == 'yes' && contains(inputs.runner, 'ubuntu' )}}
188203
name: "Run (linux): integration tests (IPv6)"
189204
run: |
190205
. ./hack/github/action-helpers.sh
191206
github::md::h2 "ipv6" >> "$GITHUB_STEP_SUMMARY"
192207
193-
./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-ipv6
208+
WAX_REPORT_LOCATION=$HOME/report/ipv6 ./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-ipv6
194209
195210
- if: ${{ env.SHOULD_RUN == 'yes' }}
196211
name: "Run: integration tests"
197212
run: |
198213
. ./hack/github/action-helpers.sh
199214
github::md::h2 "non-flaky" >> "$GITHUB_STEP_SUMMARY"
200215
201-
./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-flaky=false
216+
WAX_REPORT_LOCATION=$HOME/report/main ./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-flaky=false
202217
203218
# FIXME: this must go
204219
- if: ${{ env.SHOULD_RUN == 'yes' }}
@@ -207,4 +222,42 @@ jobs:
207222
. ./hack/github/action-helpers.sh
208223
github::md::h2 "flaky" >> "$GITHUB_STEP_SUMMARY"
209224
210-
./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-flaky=true
225+
WAX_REPORT_LOCATION=$HOME/report/flaky ./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-flaky=true
226+
227+
- name: "Wrap: collector"
228+
if: ${{ env.SHOULD_RUN == 'yes' && (failure() || success()) }}
229+
run: |
230+
# Add metadata info to the runs
231+
. ./mod/wax/scripts/collector.sh
232+
collect::metadata \
233+
"runner=${{ inputs.runner }}" \
234+
"binary=${{ inputs.binary }}" \
235+
"canary=${{ inputs.canary }}" \
236+
"ipv6=false" \
237+
"attempt=$GITHUB_RUN_ATTEMPT" \
238+
"sha=$GITHUB_SHA" \
239+
"id=$GITHUB_RUN_ID" \
240+
"number=$GITHUB_RUN_NUMBER" \
241+
> ~/report/main/metadata.json || true
242+
243+
collect::metadata \
244+
"runner=${{ inputs.runner }}" \
245+
"binary=${{ inputs.binary }}" \
246+
"canary=${{ inputs.canary }}" \
247+
"ipv6=true" \
248+
"attempt=$GITHUB_RUN_ATTEMPT" \
249+
"sha=$GITHUB_SHA" \
250+
"id=$GITHUB_RUN_ID" \
251+
"number=$GITHUB_RUN_NUMBER" \
252+
> ~/report/ipv6/metadata.json || true
253+
254+
cp ~/report/main/metadata.json ~/report/flaky/metadata.json || true
255+
256+
- name: "Wrap: upload"
257+
id: artifact-upload
258+
if: ${{ env.SHOULD_RUN == 'yes' && (failure() || success()) }}
259+
uses: actions/upload-artifact@v4
260+
with:
261+
path: ~/report/*
262+
retention-days: 1
263+
name: wax-${{ inputs.runner }}-${{ inputs.binary }}-${{ inputs.canary }}

‎.github/workflows/workflow-lint.yml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,3 +78,13 @@ jobs:
7878
go-version: ${{ matrix.go-version }}
7979
runner: ubuntu-24.04
8080
canary: ${{ matrix.canary && true || false }}
81+
82+
83+
reporter:
84+
if: ${{ failure() || success() }}
85+
name: "DEBUG"
86+
runs-on: ubuntu-24.04
87+
steps:
88+
- name: Process
89+
run: |
90+
echo "::error title=TESTTHIS,file=.github/workflows/job-test-in-container.yml,line=1::ONE<br><br>TWO%0A%0ATHREE"

‎.github/workflows/workflow-test.yml

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,3 +147,33 @@ jobs:
147147
containerd-service-sha: 1941362cbaa89dd591b99c32b050d82c583d3cd2e5fa63085d7017457ec5fca8
148148
linux-cni-version: v1.7.1
149149
linux-cni-sha: 1a28a0506bfe5bcdc981caf1a49eeab7e72da8321f1119b7be85f22621013098
150+
151+
reporter:
152+
if: ${{ failure() || success() }}
153+
name: "reporter${{ inputs.hack }}"
154+
needs:
155+
- test-integration-host
156+
- test-integration-container
157+
runs-on: ubuntu-24.04
158+
steps:
159+
- name: Fetch Repository
160+
uses: actions/checkout@v4
161+
with:
162+
fetch-depth: 2
163+
- name: "Init: install go"
164+
uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
165+
with:
166+
go-version: 1.24
167+
check-latest: true
168+
- name: Download all workflow run artifacts
169+
uses: actions/download-artifact@v4
170+
with:
171+
path: ~/report
172+
- name: Process
173+
env:
174+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
175+
run: |
176+
cd ./mod/wax
177+
go build -o ../../wax ./cmd/wax
178+
cd -
179+
./wax ~/report

‎cmd/nerdctl/main_test.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,11 @@ func TestMain(m *testing.M) {
3333
testutil.M(m)
3434
}
3535

36+
func TestWax(t *testing.T) {
37+
t.Log("This test is voluntarily failing")
38+
t.FailNow()
39+
}
40+
3641
// TestUnknownCommand tests https://github.com/containerd/nerdctl/issues/487
3742
func TestUnknownCommand(t *testing.T) {
3843
testCase := nerdtest.Setup()

‎hack/github/action-helpers.sh

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,3 +117,8 @@ github::timer::format() {
117117
[[ "$m" == 0 ]] || printf "%d minutes " "$m"
118118
printf '%d seconds' "$s"
119119
}
120+
121+
# echo "::error title=ErrorReport::MEH .github/workflows/job-test-in-host.yml${{steps.artifact-upload.outputs.artifact-url}}"
122+
# echo "::notice title=NoticeReport::SHEESH ${{steps.artifact-upload.outputs.artifact-url}}"
123+
# echo "::error file=cmd/nerdctl/main_test_test.go,line=1,endLine=10,title=AgainErrorReport::FOO ${{steps.artifact-upload.outputs.artifact-url}}"
124+
# echo "::error file=cmd/nerdctl/main_test.go,line=38,endLine=41,title=AgainErrorReport::BLA ${{steps.artifact-upload.outputs.artifact-url}}"

‎hack/github/gotestsum-reporter.sh

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,11 @@
1414
# See the License for the specific language governing permissions and
1515
# limitations under the License.
1616

17+
# DEPRECATED: favor post-processing inside integration.sh rather than using a post-run hook.
18+
# Reasons are:
19+
# - post-run hook do not work on windows
20+
# - post-run hook is limited to a single run of gotestsum, while it is desirable to process multiple runs together
21+
1722
# shellcheck disable=SC2034,SC2015
1823
set -o errexit -o errtrace -o functrace -o nounset -o pipefail
1924
root="$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)"

‎hack/test-integration.sh

Lines changed: 63 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -19,38 +19,79 @@ set -o errexit -o errtrace -o functrace -o nounset -o pipefail
1919
root="$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)"
2020
readonly root
2121

22-
if [[ "$(id -u)" = "0" ]]; then
23-
# Ensure securityfs is mounted for apparmor to work
24-
if ! mountpoint -q /sys/kernel/security; then
25-
mount -tsecurityfs securityfs /sys/kernel/security
26-
fi
22+
# If no argument is provided, run both flaky and not-flaky test suites.
23+
if [ "$#" == 0 ]; then
24+
"$root"/integration.sh -test.only-flaky=false
25+
"$root"/integration.sh -test.only-flaky=true
26+
exit
2727
fi
2828

29+
##### Import helper libraries
30+
# shellcheck source=/dev/null
31+
. "$root"/../mod/wax/scripts/collector.sh
32+
33+
##### Configuration
34+
# Where to store report files
35+
readonly report_location="${WAX_REPORT_LOCATION:-$HOME/nerdctl-test-report}"
36+
# Where to store gotestsum log file
37+
readonly gotestsum_log_main="$report_location"/test-integration.log
38+
readonly gotestsum_log_flaky="$report_location"/test-integration-flaky.log
39+
# Total run timeout
2940
readonly timeout="60m"
41+
# Number of retries for flaky tests
3042
readonly retries="2"
31-
readonly needsudo="${WITH_SUDO:-}"
32-
33-
# See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
34-
args=(--format=testname --jsonfile /tmp/test-integration.log --packages="$root"/../cmd/nerdctl/...)
35-
# FIXME: not working on windows. Need to change approach: move away from --post-run-command and
36-
# just process the log file. This might also allow multi-steps/multi-target results aggregation.
37-
[ "$(uname -s)" != "Linux" ] || args+=(--post-run-command "$root"/github/gotestsum-reporter.sh)
38-
39-
if [ "$#" == 0 ]; then
40-
"$root"/test-integration.sh -test.only-flaky=false
41-
"$root"/test-integration.sh -test.only-flaky=true
42-
exit
43-
fi
43+
readonly need_sudo="${WITH_SUDO:-}"
4444

45+
##### Prepare gotestsum arguments
46+
mkdir -p "$report_location"
47+
# Format and packages to test
48+
args=(--format=testname --packages="$root"/../cmd/nerdctl/...)
49+
# Log file
50+
gotestsum_log="$gotestsum_log_main"
4551
for arg in "$@"; do
4652
if [ "$arg" == "-test.only-flaky=true" ] || [ "$arg" == "-test.only-flaky" ]; then
4753
args+=("--rerun-fails=$retries")
54+
gotestsum_log="$gotestsum_log_flaky"
4855
break
4956
fi
5057
done
58+
args+=(--jsonfile "$gotestsum_log" --)
59+
60+
##### Append go test arguments
61+
# Honor sudo
62+
[ "$need_sudo" != true ] && [ "$need_sudo" != yes ] && [ "$need_sudo" != 1 ] || args+=(-exec sudo)
63+
# About `-p 1`, see https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
64+
args+=(-timeout="$timeout" -p 1 -args -test.allow-kill-daemon "$@")
5165

52-
if [ "$needsudo" == "true" ] || [ "$needsudo" == "yes" ] || [ "$needsudo" == "1" ]; then
53-
gotestsum "${args[@]}" -- -timeout="$timeout" -p 1 -exec sudo -args -test.allow-kill-daemon "$@"
54-
else
55-
gotestsum "${args[@]}" -- -timeout="$timeout" -p 1 -args -test.allow-kill-daemon "$@"
66+
# FIXME: this should not be the responsibility of the test script
67+
# Instead, it should be in the Dockerfile (or other stack provisioning script) - eg: /etc/systemd/system/securityfs.service
68+
# [Unit]
69+
# Description=Kernel Security File System
70+
# DefaultDependencies=no
71+
# Before=sysinit.target
72+
# Before=apparmor.service
73+
# ConditionSecurity=apparmor
74+
# ConditionPathIsMountPoint=!/sys/kernel/security
75+
#
76+
# [Service]
77+
# Type=oneshot
78+
# ExecStart=/bin/mount -t securityfs -o nosuid,nodev,noexec securityfs /sys/kernel/security
79+
#
80+
# [Install]
81+
# WantedBy=sysinit.target
82+
if [[ "$(id -u)" = "0" ]]; then
83+
# Ensure securityfs is mounted for apparmor to work
84+
if ! mountpoint -q /sys/kernel/security; then
85+
mount -tsecurityfs securityfs /sys/kernel/security
86+
fi
5687
fi
88+
89+
##### Run it
90+
ex=0
91+
gotestsum "${args[@]}" || ex=$?
92+
93+
##### Post: collect logs into the report location
94+
collect::logs "$report_location"
95+
96+
# Honor gotestsum exit code
97+
exit "$ex"

0 commit comments

Comments
 (0)