Skip to content

Commit e782c93

Browse files
mikehallermikehaller
authored
Merge pull request #35 from SoftwareDefinedVehicle/main
Populate downloads mirror
2 parents 6b275b0 + 4b586e7 commit e782c93

File tree

10 files changed

+372
-21
lines changed

10 files changed

+372
-21
lines changed

‎.github/workflows/build-selfhosted.yml‎

Lines changed: 243 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,243 @@
1+
# /********************************************************************************
2+
# * Copyright (c) 2023 Contributors to the Eclipse Foundation
3+
# *
4+
# * See the NOTICE file(s) distributed with this work for additional
5+
# * information regarding copyright ownership.
6+
# *
7+
# * This program and the accompanying materials are made available under the
8+
# * terms of the Apache License 2.0 which is available at
9+
# * https://www.apache.org/licenses/LICENSE-2.0
10+
# *
11+
# * SPDX-License-Identifier: Apache-2.0
12+
# ********************************************************************************/
13+
#
14+
# To run on Eclipse managed GitHub Runner, set to run on self-hosted and add a container name to the job:
15+
# runs-on: self-hosted
16+
# container: debian:bullseye
17+
#
18+
name: BitBake Build (Self-Hosted)
19+
on:
20+
workflow_dispatch:
21+
workflow_call:
22+
23+
jobs:
24+
25+
leda-qemux86_64:
26+
name: "qemux86_64"
27+
runs-on: self-hosted
28+
continue-on-error: true
29+
steps:
30+
- name: Checkout
31+
uses: actions/checkout@v3
32+
- name: Install tools
33+
run: |
34+
sudo apt-get update
35+
sudo apt-get install -y --no-install-recommends socat file gawk wget git diffstat unzip texinfo gcc build-essential chrpath socat cpio python3 python3-pip python3-pexpect xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev pylint xterm python3-subunit mesa-common-dev zstd liblz4-tool tmux mc skopeo fdisk ruby-full jq libvirt-clients libvirt-daemon-system qemu-system-x86 qemu-system-arm qemu-kvm squashfs-tools rauc python3-newt ca-certificates curl gnupg lsb-release
36+
- name: Install kas
37+
run: sudo pip3 install kas
38+
- name: Cache SState
39+
id: cache-sstate
40+
uses: pat-s/[email protected]
41+
with:
42+
path: build/sstate-cache
43+
key: bb-sstate-cache-qemux86-64
44+
- name: Build sdv-image-all
45+
env:
46+
KAS_MACHINE: qemux86-64
47+
run: |
48+
(r=3;while ! kas build kas/leda-qemux86-64.yaml:kas/mirrors.yaml ; do ((--r))||exit;sleep 60;done)
49+
- name: Generate CHANGELOG.md
50+
run: |
51+
docker run --rm -v "$(pwd)":/usr/local/src/your-app githubchangeloggenerator/github-changelog-generator github_changelog_generator -u eclipse-leda -p leda-distro --no-author -t ${{ secrets.GITHUB_TOKEN }}
52+
- name: Package all files
53+
run: |
54+
export ARC=eclipse-leda-qemu-x86_64.tar
55+
rm -f ${ARC}.xz
56+
time tar --create --gzip --file 3rdparty-licenses.tar.gz -C ./build/tmp/deploy/licenses/ .
57+
time tar --create --verbose --file ${ARC} -C ./resources/runners/qemux86_64/ \
58+
run-leda.sh \
59+
run-leda.cmd \
60+
readme.txt
61+
time tar --append --verbose --file ${ARC} -C scripts/ \
62+
provision-device.sh
63+
time tar --append --verbose --file ${ARC} -C ./ \
64+
LICENSE \
65+
NOTICE.md \
66+
CHANGELOG.md \
67+
README.md \
68+
3rdparty-licenses.tar.gz \
69+
examples
70+
time tar --append --verbose --dereference --file ${ARC} -C ./build/tmp/deploy/images/qemux86-64/ \
71+
bzImage-qemux86-64.bin \
72+
sdv-image-all-qemux86-64.wic.qcow2 \
73+
sdv-rauc-bundle-qemux86-64.raucb \
74+
ovmf.qcow2
75+
time xz -v -8 -T0 ${ARC}
76+
- name: Upload files
77+
uses: actions/upload-artifact@v3
78+
with:
79+
name: eclipse-leda-qemu-x86_64.tar.xz
80+
if-no-files-found: error
81+
path: eclipse-leda-qemu-x86_64.tar.xz
82+
- name: Package SPDX files
83+
run: |
84+
tar --create --gzip --file eclipse-leda-sbom-qemux86_64.tar.gz -C ./build/tmp/deploy/spdx/qemux86-64/ .
85+
- name: Upload files
86+
uses: actions/upload-artifact@v3
87+
with:
88+
name: eclipse-leda-sbom-qemux86_64.tar.gz
89+
if-no-files-found: error
90+
path: eclipse-leda-sbom-qemux86_64.tar.gz
91+
- uses: bacongobbler/[email protected]
92+
if: github.repository_owner == 'SoftwareDefinedVehicle'
93+
with:
94+
source_dir: 'build/sstate-cache/'
95+
extra_args: '--destination-path sstate-cache/'
96+
container_name: 'yocto-sstate-cache'
97+
overwrite: 'true'
98+
connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }}
99+
100+
leda-qemuarm64:
101+
name: "qemuarm64"
102+
runs-on: self-hosted
103+
continue-on-error: true
104+
steps:
105+
- name: Checkout
106+
uses: actions/checkout@v3
107+
- name: Install tools
108+
run: |
109+
sudo apt-get update
110+
sudo apt-get install -y --no-install-recommends socat file gawk wget git diffstat unzip texinfo gcc build-essential chrpath socat cpio python3 python3-pip python3-pexpect xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev pylint xterm python3-subunit mesa-common-dev zstd liblz4-tool tmux mc skopeo fdisk ruby-full jq libvirt-clients libvirt-daemon-system qemu-system-x86 qemu-system-arm qemu-kvm squashfs-tools rauc python3-newt ca-certificates curl gnupg lsb-release
111+
- name: Install kas
112+
run: sudo pip3 install kas
113+
- name: Cache SState
114+
id: cache-sstate
115+
uses: pat-s/[email protected]
116+
with:
117+
path: build/sstate-cache
118+
key: bb-sstate-cache-qemuarm64
119+
- name: Build sdv-image-all
120+
env:
121+
KAS_MACHINE: qemuarm64
122+
run: |
123+
(r=3;while ! kas build kas/leda-qemuarm64.yaml:kas/mirrors.yaml ; do ((--r))||exit;sleep 60;done)
124+
- name: Generate CHANGELOG.md
125+
run: |
126+
docker run --rm -v "$(pwd)":/usr/local/src/your-app githubchangeloggenerator/github-changelog-generator github_changelog_generator -u eclipse-leda -p leda-distro --no-author -t ${{ secrets.GITHUB_TOKEN }}
127+
- name: Package all files
128+
run: |
129+
export ARC=eclipse-leda-qemu-arm64.tar
130+
rm -f ${ARC}.xz
131+
time tar --create --gzip --file 3rdparty-licenses.tar.gz -C ./build/tmp/deploy/licenses/ .
132+
time tar --create --verbose --file ${ARC} -C ./resources/runners/qemuarm64/ \
133+
run-leda.sh \
134+
run-leda.cmd \
135+
readme.txt
136+
time tar --append --verbose --file ${ARC} -C scripts/ \
137+
provision-device.sh
138+
time tar --append --verbose --file ${ARC} -C ./ \
139+
LICENSE \
140+
NOTICE.md \
141+
CHANGELOG.md \
142+
README.md \
143+
3rdparty-licenses.tar.gz \
144+
examples
145+
time tar --append --verbose --dereference --file ${ARC} -C ./build/tmp/deploy/images/qemuarm64/ \
146+
Image-qemuarm64.bin \
147+
sdv-image-all-qemuarm64.wic.qcow2 \
148+
sdv-rauc-bundle-qemuarm64.raucb \
149+
u-boot.bin
150+
time xz -v -8 -T0 ${ARC}
151+
- name: Upload files
152+
uses: actions/upload-artifact@v3
153+
with:
154+
name: eclipse-leda-qemu-arm64.tar.xz
155+
if-no-files-found: error
156+
path: eclipse-leda-qemu-arm64.tar.xz
157+
- name: Package SPDX files
158+
run: |
159+
tar --create --gzip --file eclipse-leda-sbom-qemuarm64.tar.gz -C ./build/tmp/deploy/spdx/qemuarm64/ .
160+
- name: Upload files
161+
uses: actions/upload-artifact@v3
162+
with:
163+
name: eclipse-leda-sbom-qemuarm64.tar.gz
164+
if-no-files-found: error
165+
path: eclipse-leda-sbom-qemuarm64.tar.gz
166+
- uses: bacongobbler/[email protected]
167+
if: github.repository_owner == 'SoftwareDefinedVehicle'
168+
with:
169+
source_dir: 'build/sstate-cache/'
170+
extra_args: '--destination-path sstate-cache/'
171+
container_name: 'yocto-sstate-cache'
172+
overwrite: 'true'
173+
connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }}
174+
175+
leda-raspberrypi4-64:
176+
name: "raspberrypi4-64"
177+
runs-on: self-hosted
178+
needs: leda-qemuarm64
179+
continue-on-error: true
180+
steps:
181+
- name: Checkout
182+
uses: actions/checkout@v3
183+
- name: Install tools
184+
run: sudo apt-get install -y --no-install-recommends socat file gawk wget git diffstat unzip texinfo gcc build-essential chrpath socat cpio python3 python3-pip python3-pexpect xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev pylint xterm python3-subunit mesa-common-dev zstd liblz4-tool tmux mc skopeo fdisk ruby-full jq libvirt-clients libvirt-daemon-system qemu-system-x86 qemu-system-arm qemu-kvm squashfs-tools rauc python3-newt ca-certificates curl gnupg lsb-release
185+
- name: Install kas
186+
run: sudo pip3 install kas
187+
- name: Cache SState
188+
id: cache-sstate
189+
uses: pat-s/[email protected]
190+
with:
191+
path: build/sstate-cache
192+
key: bb-sstate-cache-raspberrypi4-64
193+
- name: Build sdv-image-all
194+
env:
195+
KAS_MACHINE: raspberrypi4-64
196+
run: |
197+
(r=3;while ! kas build kas/leda-raspberrypi4-64.yaml:kas/mirrors.yaml ; do ((--r))||exit;sleep 60;done)
198+
- name: Generate CHANGELOG.md
199+
run: |
200+
docker run --rm -v "$(pwd)":/usr/local/src/your-app githubchangeloggenerator/github-changelog-generator github_changelog_generator -u eclipse-leda -p leda-distro --no-author -t ${{ secrets.GITHUB_TOKEN }}
201+
- name: Package all files
202+
run: |
203+
export ARC=eclipse-leda-raspberrypi.tar
204+
rm -f ${ARC}.xz
205+
time tar --create --verbose --gzip --file 3rdparty-licenses.tar.gz -C ./build/tmp/deploy/licenses/ .
206+
time tar --create --verbose --file ${ARC} -C resources/runners/raspberrypi4/ \
207+
readme.txt
208+
time tar --append --verbose --file ${ARC} -C ./ \
209+
LICENSE \
210+
NOTICE.md \
211+
CHANGELOG.md \
212+
README.md \
213+
3rdparty-licenses.tar.gz \
214+
examples
215+
time tar --append --verbose --dereference --file ${ARC} -C ./build/tmp/deploy/images/raspberrypi4-64/ \
216+
Image-raspberrypi4-64.bin \
217+
sdv-image-all-raspberrypi4-64.wic.bz2 \
218+
sdv-image-all-raspberrypi4-64.wic.bmap \
219+
sdv-rauc-bundle-raspberrypi4-64.raucb
220+
time xz -v -1 -T0 ${ARC}
221+
- name: Upload files
222+
uses: actions/upload-artifact@v3
223+
with:
224+
name: eclipse-leda-raspberrypi.tar.xz
225+
if-no-files-found: error
226+
path: eclipse-leda-raspberrypi.tar.xz
227+
- name: Package SPDX files
228+
run: |
229+
tar --create --gzip --file eclipse-leda-sbom-raspberrypi4-64.tar.gz -C ./build/tmp/deploy/spdx/raspberrypi4-64/ .
230+
- name: Upload files
231+
uses: actions/upload-artifact@v3
232+
with:
233+
name: eclipse-leda-sbom-raspberrypi4-64.tar.gz
234+
if-no-files-found: error
235+
path: eclipse-leda-sbom-raspberrypi4-64.tar.gz
236+
- uses: bacongobbler/[email protected]
237+
if: github.repository_owner == 'SoftwareDefinedVehicle'
238+
with:
239+
source_dir: 'build/sstate-cache/'
240+
extra_args: '--destination-path sstate-cache/'
241+
container_name: 'yocto-sstate-cache'
242+
overwrite: 'true'
243+
connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }}

‎.github/workflows/build.yml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -47,10 +47,10 @@ jobs:
4747
env:
4848
KAS_MACHINE: qemux86-64
4949
run: |
50-
(r=3;while ! kas build kas/leda-qemux86-64 ; do ((--r))||exit;sleep 60;done)
50+
(r=3;while ! kas build kas/leda-qemux86-64.yaml:kas/mirrors.yaml ; do ((--r))||exit;sleep 60;done)
5151
- name: Generate CHANGELOG.md
5252
run: |
53-
docker run --rm -v "$(pwd)":/usr/local/src/your-app githubchangeloggenerator/github-changelog-generator github_changelog_generator -u eclipse-leda -p leda-distro --no-author
53+
docker run --rm -v "$(pwd)":/usr/local/src/your-app githubchangeloggenerator/github-changelog-generator github_changelog_generator -u eclipse-leda -p leda-distro --no-author -t ${{ secrets.GITHUB_TOKEN }}
5454
- name: Package all files
5555
run: |
5656
export ARC=eclipse-leda-qemu-x86_64.tar
@@ -125,10 +125,10 @@ jobs:
125125
env:
126126
KAS_MACHINE: qemuarm64
127127
run: |
128-
(r=3;while ! kas build kas/leda-qemuarm64.yaml ; do ((--r))||exit;sleep 60;done)
128+
(r=3;while ! kas build kas/leda-qemuarm64.yaml:kas/mirrors.yaml ; do ((--r))||exit;sleep 60;done)
129129
- name: Generate CHANGELOG.md
130130
run: |
131-
docker run --rm -v "$(pwd)":/usr/local/src/your-app githubchangeloggenerator/github-changelog-generator github_changelog_generator -u eclipse-leda -p leda-distro --no-author
131+
docker run --rm -v "$(pwd)":/usr/local/src/your-app githubchangeloggenerator/github-changelog-generator github_changelog_generator -u eclipse-leda -p leda-distro --no-author -t ${{ secrets.GITHUB_TOKEN }}
132132
- name: Package all files
133133
run: |
134134
export ARC=eclipse-leda-qemu-arm64.tar
@@ -201,10 +201,10 @@ jobs:
201201
env:
202202
KAS_MACHINE: raspberrypi4-64
203203
run: |
204-
(r=3;while ! kas build kas/leda-raspberrypi4-64.yaml ; do ((--r))||exit;sleep 60;done)
204+
(r=3;while ! kas build kas/leda-raspberrypi4-64.yaml:kas/mirrors.yaml ; do ((--r))||exit;sleep 60;done)
205205
- name: Generate CHANGELOG.md
206206
run: |
207-
docker run --rm -v "$(pwd)":/usr/local/src/your-app githubchangeloggenerator/github-changelog-generator github_changelog_generator -u eclipse-leda -p leda-distro --no-author
207+
docker run --rm -v "$(pwd)":/usr/local/src/your-app githubchangeloggenerator/github-changelog-generator github_changelog_generator -u eclipse-leda -p leda-distro --no-author -t ${{ secrets.GITHUB_TOKEN }}
208208
- name: Package all files
209209
run: |
210210
export ARC=eclipse-leda-raspberrypi.tar

‎.github/workflows/dryrun.yml‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,9 @@ jobs:
3131
- name: Checkout
3232
uses: actions/checkout@v3
3333
- name: Install tools
34-
run: sudo apt-get install -y --no-install-recommends socat file gawk wget git diffstat unzip texinfo gcc build-essential chrpath socat cpio python3 python3-pip python3-pexpect xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev pylint xterm python3-subunit mesa-common-dev zstd liblz4-tool python3-newt ca-certificates curl gnupg lsb-release
34+
run: |
35+
sudo apt-get update
36+
sudo apt-get install -y --no-install-recommends socat file gawk wget git diffstat unzip texinfo gcc build-essential chrpath socat cpio python3 python3-pip python3-pexpect xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev pylint xterm python3-subunit mesa-common-dev zstd liblz4-tool python3-newt ca-certificates curl gnupg lsb-release
3537
- name: Install kas
3638
run: sudo pip3 install kas
3739
- name: DryRun sdv-image-all
@@ -44,7 +46,9 @@ jobs:
4446
- name: Checkout
4547
uses: actions/checkout@v3
4648
- name: Install tools
47-
run: sudo apt-get install -y --no-install-recommends socat file gawk wget git diffstat unzip texinfo gcc build-essential chrpath socat cpio python3 python3-pip python3-pexpect xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev pylint xterm python3-subunit mesa-common-dev zstd liblz4-tool python3-newt ca-certificates curl gnupg lsb-release
49+
run: |
50+
sudo apt-get update
51+
sudo apt-get install -y --no-install-recommends socat file gawk wget git diffstat unzip texinfo gcc build-essential chrpath socat cpio python3 python3-pip python3-pexpect xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev pylint xterm python3-subunit mesa-common-dev zstd liblz4-tool python3-newt ca-certificates curl gnupg lsb-release
4852
- name: Install kas
4953
run: sudo pip3 install kas
5054
- name: DryRun sdv-image-all
@@ -57,7 +61,9 @@ jobs:
5761
- name: Checkout
5862
uses: actions/checkout@v3
5963
- name: Install tools
60-
run: sudo apt-get install -y --no-install-recommends socat file gawk wget git diffstat unzip texinfo gcc build-essential chrpath socat cpio python3 python3-pip python3-pexpect xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev pylint xterm python3-subunit mesa-common-dev zstd liblz4-tool python3-newt ca-certificates curl gnupg lsb-release
64+
run: |
65+
sudo apt-get update
66+
sudo apt-get install -y --no-install-recommends socat file gawk wget git diffstat unzip texinfo gcc build-essential chrpath socat cpio python3 python3-pip python3-pexpect xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev pylint xterm python3-subunit mesa-common-dev zstd liblz4-tool python3-newt ca-certificates curl gnupg lsb-release
6167
- name: Install kas
6268
run: sudo pip3 install kas
6369
- name: DryRun sdv-image-all

‎.github/workflows/pre-download-source.yml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# /********************************************************************************
2+
# * Copyright (c) 2022 Contributors to the Eclipse Foundation
3+
# *
4+
# * See the NOTICE file(s) distributed with this work for additional
5+
# * information regarding copyright ownership.
6+
# *
7+
# * This program and the accompanying materials are made available under the
8+
# * terms of the Apache License 2.0 which is available at
9+
# * https://www.apache.org/licenses/LICENSE-2.0
10+
# *
11+
# * SPDX-License-Identifier: Apache-2.0
12+
# ********************************************************************************/
13+
#
14+
# Some sources are hosted in slow repositories (ex. yocto linux kernel) which leads to increased build times.
15+
# This workflow pre-downloads and caches the sources required for all current Leda targets.
16+
17+
name: Pre-download sources
18+
on:
19+
workflow_dispatch:
20+
workflow_call:
21+
22+
jobs:
23+
pre-download-all:
24+
name: "Pre-download sources for all targets"
25+
runs-on: ubuntu-22.04
26+
continue-on-error: true
27+
steps:
28+
- name: Checkout
29+
uses: actions/checkout@v3
30+
- name: Install tools
31+
run: |
32+
sudo apt-get update
33+
sudo apt-get install -y --no-install-recommends socat file gawk wget git diffstat unzip texinfo gcc build-essential chrpath socat cpio python3 python3-pip python3-pexpect xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev pylint xterm python3-subunit mesa-common-dev zstd liblz4-tool tmux mc skopeo fdisk ruby-full jq libvirt-clients libvirt-daemon-system qemu-system-x86 qemu-system-arm qemu-kvm squashfs-tools rauc python3-newt ca-certificates curl gnupg lsb-release
34+
- name: Install kas
35+
run: sudo pip3 install kas
36+
- name: Pre-downloads sources
37+
if: always()
38+
continue-on-error: true
39+
run: |
40+
kas shell kas/leda-qemux86-64.yaml:kas/generate-mirror.yaml -c 'bitbake sdv-image-all --runall=fetch'
41+
kas shell kas/leda-qemuarm64.yaml:kas/generate-mirror.yaml -c 'bitbake sdv-image-all --runall=fetch'
42+
kas shell kas/leda-raspberrypi4-64.yaml:kas/generate-mirror.yaml -c 'bitbake sdv-image-all --runall=fetch'
43+
find build/downloads -name "*bad-checksum*" -exec rm -f {} \;
44+
rm -rf build/downloads/git2
45+
rm -rf build/downloads/svn
46+
find build/downloads -name "*.done" -exec rm -f {} \;
47+
- uses: bacongobbler/[email protected]
48+
if: github.repository_owner == 'SoftwareDefinedVehicle'
49+
with:
50+
source_dir: 'build/downloads/'
51+
container_name: 'downloads'
52+
overwrite: 'true'
53+
connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }}

0 commit comments

Comments
 (0)