Skip to content

Commit 92a5d5b

Browse files
nikromennikromen
committed
frontend,backend,rpmbuild,cli,python: uploadrpm - multiple RPMs, optional srpm/logs
Fixes: #4442
1 parent 1c1ca9a commit 92a5d5b

27 files changed

Lines changed: 960 additions & 472 deletions

File tree

‎backend/copr_backend/background_worker_build.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
from copr_common.enums import StatusEnum, StorageEnum
2020
from copr_common.helpers import (
21+
format_evr,
2122
USER_SSH_DEFAULT_EXPIRATION,
2223
USER_SSH_MAX_EXPIRATION,
2324
USER_SSH_EXPIRATION_PATH,
@@ -33,7 +34,7 @@
3334
)
3435
from copr_backend.rpmeta import rpmeta_predict_build_time
3536
from copr_backend.helpers import (
36-
run_cmd, register_build_result, format_evr,
37+
run_cmd, register_build_result,
3738
)
3839
from copr_backend.job import BuildJob
3940
from copr_backend.msgbus import MessageSender

‎backend/copr_backend/helpers.py‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -655,17 +655,6 @@ def get_chroot_arch(chroot):
655655
return chroot.rsplit("-", 2)[2]
656656

657657

658-
def format_evr(epoch, version, release):
659-
"""
660-
Return evr in format (epoch:)version-release. The argument 'epoch' should
661-
be integer value or null (but we rather also consider "strings" values).
662-
"""
663-
if epoch is not None:
664-
if isinstance(epoch, int) or epoch.isdigit():
665-
return f"{epoch}:{version}-{release}"
666-
return f"{version}-{release}"
667-
668-
669658
def format_filename(name, version, release, epoch, arch, zero_epoch=False):
670659
if not epoch.isdigit() and zero_epoch:
671660
epoch = "0"

‎beaker-tests/Sanity/copr-cli-basic-operations/runtest-rpm-upload.sh‎

Lines changed: 177 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ source "$HERE/config"
2323
source "$HERE/helpers"
2424

2525
PACKAGE=copr-rpm-upload-sanity-test
26+
PACKAGE_MULTI=copr-rpm-upload-multi-sanity-test
2627

2728
# Build a throwaway binary RPM locally
2829
build_local_rpm()
@@ -49,13 +50,105 @@ EOF
4950
find "$workdir" -name '*.rpm'
5051
}
5152

53+
# Build a throwaway package with a sub-package (-> multiple binary RPMs) plus
54+
# its srpm, to exercise the multi-RPM tarball upload scenario
55+
build_local_rpms_with_subpackage_and_srpm()
56+
{
57+
local workdir
58+
workdir=$(mktemp -d)
59+
cat > "$workdir/$PACKAGE_MULTI.spec" <<EOF
60+
Name: $PACKAGE_MULTI
61+
Version: 1
62+
Release: 1
63+
Summary: Throwaway package for the direct RPM upload sanity test
64+
License: MIT
65+
BuildArch: $(rpm --eval '%_arch')
66+
67+
%description
68+
Throwaway package for the direct RPM upload sanity test.
69+
70+
%package subpkg
71+
Summary: Throwaway sub-package for the direct RPM upload sanity test
72+
%description subpkg
73+
Throwaway sub-package for the direct RPM upload sanity test.
74+
75+
%files
76+
77+
%files subpkg
78+
EOF
79+
rpmbuild -ba "$workdir/$PACKAGE_MULTI.spec" \
80+
--define "_topdir $workdir" \
81+
--define "_rpmdir $workdir" \
82+
--define "_srcrpmdir $workdir" \
83+
--define "_build_id_links none" >&2
84+
find "$workdir" -name '*.rpm'
85+
}
86+
87+
# Package RPMs/logs/etc. into a single upload tarball with one top-level dir.
88+
# Optional extra args are passed to build_upload_tarball_sha256_json().
89+
build_upload_tarball()
90+
{
91+
local workdir payload_dir tarball_path file basename
92+
workdir=$(mktemp -d)
93+
payload_dir="$workdir/upload"
94+
mkdir -p "$payload_dir"
95+
96+
for file in "$@"; do
97+
basename=$(basename "$file")
98+
cp "$file" "$payload_dir/$basename"
99+
done
100+
101+
if declare -F build_upload_tarball_sha256_json >/dev/null; then
102+
build_upload_tarball_sha256_json "$payload_dir"
103+
fi
104+
105+
tarball_path=$(mktemp --suffix=.tar.gz)
106+
tar -C "$workdir" -czf "$tarball_path" upload
107+
rm -rf "$workdir"
108+
echo "$tarball_path"
109+
}
110+
111+
build_upload_tarball_with_bad_sha256()
112+
{
113+
local rpm_path="$1"
114+
local workdir payload_dir tarball_path rpm_name
115+
116+
workdir=$(mktemp -d)
117+
payload_dir="$workdir/upload"
118+
mkdir -p "$payload_dir"
119+
rpm_name=$(basename "$rpm_path")
120+
cp "$rpm_path" "$payload_dir/$rpm_name"
121+
printf '{"%s": "%s"}\n' "$rpm_name" "$(printf '%0*d' 64 0)" \
122+
> "$payload_dir/sha256.json"
123+
124+
tarball_path=$(mktemp --suffix=.tar.gz)
125+
tar -C "$workdir" -czf "$tarball_path" upload
126+
rm -rf "$workdir"
127+
echo "$tarball_path"
128+
}
129+
130+
build_upload_tarball_sha256_json()
131+
{
132+
local payload_dir="$1"
133+
local file basename checksum
134+
local -a entries=()
135+
136+
for file in "$payload_dir"/*; do
137+
basename=$(basename "$file")
138+
checksum=$(sha256sum "$file" | cut -d' ' -f1)
139+
entries+=("\"$basename\": \"$checksum\"")
140+
done
141+
142+
printf '{%s}\n' "$(IFS=,; echo "${entries[*]}")" > "$payload_dir/sha256.json"
143+
}
144+
52145
rlJournalStart
53146
rlPhaseStartSetup
54147
setup_checks
55148
setupProjectName "rpm-upload"
56149
rlPhaseEnd
57150

58-
rlPhaseStartTest
151+
rlPhaseStartTest "basic uploadrpm tarball"
59152
if [[ $FRONTEND_URL == "https://copr.stg.fedoraproject.org" ]]; then
60153
rlLog "Skipping, RPM uploads are not enabled for the Fedora Copr instance"
61154
exit 0
@@ -65,36 +158,107 @@ rlJournalStart
65158

66159
rlRun "RPM_PATH=\$(build_local_rpm)" 0 "Building a local test RPM"
67160
rlAssertExists "$RPM_PATH"
161+
rlRun "TARBALL_PATH=\$(build_upload_tarball \"$RPM_PATH\")" \
162+
0 "Building upload tarball"
68163

69-
# go through the real `copr-cli uploadrpm` command, like a real user
70-
# would -- publishes the RPM directly, skipping the SRPM build and
71-
# dist-git import phases entirely
72-
rlRun -s "copr-cli uploadrpm --nowait --chroot $CHROOT $PROJECT $RPM_PATH"
164+
rlRun -s "copr-cli uploadrpm --nowait --chroot $CHROOT \
165+
--name $PACKAGE --version 1 --release 1 \
166+
$PROJECT $TARBALL_PATH"
73167
rlRun "parse_build_id"
74168
rlRun "copr watch-build $BUILD_ID"
75169

76-
# verify the uploaded RPM is really installable from the project's repo
77170
rlRun "yes | dnf copr enable $DNF_COPR_ID/$PROJECT $CHROOT"
78171
rlRun "dnf install -y --disablerepo='*' \
79172
--enablerepo=\"copr:${FRONTEND_PUBLIC_HOST}:$(repo_owner):${PROJECTNAME}\" \
80173
$PACKAGE"
81174
rlAssertRpm "$PACKAGE"
175+
rlPhaseEnd
176+
177+
rlPhaseStartTest "uploadrpm tarball with logs"
178+
if [[ $FRONTEND_URL == "https://copr.stg.fedoraproject.org" ]]; then
179+
rlLog "Skipping, RPM uploads are not enabled for the Fedora Copr instance"
180+
exit 0
181+
fi
182+
183+
rlRun "RPM_PATH=\$(build_local_rpm)" 0 "Building a local test RPM"
184+
LOG1=$(mktemp --suffix=.log)
185+
echo "fake builder-live log" > "$LOG1"
186+
LOG2=$(mktemp --suffix=.txt)
187+
echo "fake notes" > "$LOG2"
188+
189+
rlRun "TARBALL_PATH=\$(build_upload_tarball \"$RPM_PATH\" \"$LOG1\" \"$LOG2\")" \
190+
0 "Building upload tarball with logs"
82191

83-
# SHA256 checksum verification -- correct checksum should succeed
84-
rlRun "CHECKSUM=\$(sha256sum $RPM_PATH | cut -d' ' -f1)"
85192
rlRun -s "copr-cli uploadrpm --nowait --chroot $CHROOT \
86-
--sha256 $CHECKSUM $PROJECT $RPM_PATH"
193+
--name $PACKAGE --version 1 --release 1 \
194+
$PROJECT $TARBALL_PATH"
87195
rlRun "parse_build_id"
88196
rlRun "copr watch-build $BUILD_ID"
89197

90-
# SHA256 checksum verification -- wrong checksum should be rejected
198+
DOWNLOAD_DEST=$(mktemp -d)
199+
rlRun "copr-cli download-build --dest $DOWNLOAD_DEST --logs $BUILD_ID"
200+
rlRun "find $DOWNLOAD_DEST -name 'uploaded-logs.tar.gz'"
201+
rlPhaseEnd
202+
203+
rlPhaseStartTest "uploadrpm tarball with bad sha256.json"
204+
if [[ $FRONTEND_URL == "https://copr.stg.fedoraproject.org" ]]; then
205+
rlLog "Skipping, RPM uploads are not enabled for the Fedora Copr instance"
206+
exit 0
207+
fi
208+
209+
rlRun "RPM_PATH=\$(build_local_rpm)" 0 "Building a local test RPM"
210+
rlRun "BAD_TARBALL=\$(build_upload_tarball_with_bad_sha256 \"$RPM_PATH\")" \
211+
0 "Building upload tarball with bad sha256.json"
212+
91213
rlRun "copr-cli uploadrpm --chroot $CHROOT \
92-
--sha256 0000000000000000000000000000000000000000000000000000000000000000 \
93-
$PROJECT $RPM_PATH" 1 "Upload with wrong SHA256 should fail"
214+
--name $PACKAGE --version 1 --release 1 \
215+
$PROJECT $BAD_TARBALL" 1 \
216+
"Upload with bad sha256.json should fail on the builder"
217+
rlPhaseEnd
218+
219+
rlPhaseStartTest "multi-RPM uploadrpm tarball with srpm and logs"
220+
if [[ $FRONTEND_URL == "https://copr.stg.fedoraproject.org" ]]; then
221+
rlLog "Skipping, RPM uploads are not enabled for the Fedora Copr instance"
222+
exit 0
223+
fi
224+
225+
rlRun "RPM_PATHS=(\$(build_local_rpms_with_subpackage_and_srpm))" \
226+
0 "Building local test RPMs (main + sub-package + srpm)"
227+
228+
SRPM_PATH=
229+
BINARY_RPMS=()
230+
for _path in "${RPM_PATHS[@]}"; do
231+
case "$_path" in
232+
*.src.rpm) SRPM_PATH=$_path ;;
233+
*) BINARY_RPMS+=("$_path") ;;
234+
esac
235+
done
236+
rlAssertExists "$SRPM_PATH"
237+
rlRun "test ${#BINARY_RPMS[@]} -eq 2" 0 \
238+
"Expecting 2 binary RPMs (main package + sub-package)"
239+
240+
LOG1=$(mktemp --suffix=.log)
241+
echo "fake builder-live log" > "$LOG1"
242+
243+
rlRun "TARBALL_PATH=\$(build_upload_tarball \
244+
\"${BINARY_RPMS[0]}\" \"${BINARY_RPMS[1]}\" \"$SRPM_PATH\" \"$LOG1\")" \
245+
0 "Building multi-RPM upload tarball"
246+
247+
rlRun -s "copr-cli uploadrpm --nowait --chroot $CHROOT \
248+
--name $PACKAGE_MULTI --version 1 --release 1 \
249+
$PROJECT $TARBALL_PATH"
250+
rlRun "parse_build_id"
251+
rlRun "copr watch-build $BUILD_ID"
252+
253+
rlRun "dnf install -y --disablerepo='*' \
254+
--enablerepo=\"copr:${FRONTEND_PUBLIC_HOST}:$(repo_owner):${PROJECTNAME}\" \
255+
$PACKAGE_MULTI $PACKAGE_MULTI-subpkg"
256+
rlAssertRpm "$PACKAGE_MULTI"
257+
rlAssertRpm "$PACKAGE_MULTI-subpkg"
94258
rlPhaseEnd
95259

96260
rlPhaseStartCleanup
97-
rlRun "dnf -y remove $PACKAGE"
261+
rlRun "dnf -y remove $PACKAGE $PACKAGE_MULTI $PACKAGE_MULTI-subpkg"
98262
rlRun "dnf -y copr remove $DNF_COPR_ID/$PROJECT"
99263
cleanProject
100264
rlPhaseEnd

‎cli/copr_cli/main.py‎

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -464,7 +464,7 @@ def action_upload_rpm(self, args):
464464
username, projectname, project_dirname = self.parse_dirname(args.copr_repo)
465465
buildopts = buildopts_from_args(args)
466466

467-
# Before we start uploading potentially large source RPM file, make sure
467+
# Before we start uploading potentially large tarball, make sure
468468
# that the user has valid credentials and can build in the project.
469469
self.client.build_proxy.check_before_build(
470470
ownername=username,
@@ -473,18 +473,21 @@ def action_upload_rpm(self, args):
473473
buildopts=buildopts,
474474
)
475475

476-
if not os.path.exists(args.rpm):
477-
raise CoprException("File {0} not found".format(args.rpm))
476+
tarball_path = args.tarball
477+
if not os.path.exists(tarball_path):
478+
raise CoprException("File {0} not found".format(tarball_path))
478479

479-
progress_callback = get_progress_callback(os.path.getsize(args.rpm))
480+
total_size = os.path.getsize(tarball_path)
481+
progress_callback = get_progress_callback(total_size)
480482
buildopts["progress_callback"] = progress_callback
481-
print('Uploading package {0}'.format(args.rpm))
483+
print('Uploading tarball {0}'.format(tarball_path))
482484
try:
483485
build = self.client.build_proxy.create_from_rpm_upload(
484486
ownername=username, projectname=projectname,
485487
project_dirname=project_dirname, buildopts=buildopts,
486-
path=args.rpm,
487-
sha256=getattr(args, "sha256", None))
488+
tarball_path=tarball_path, name=args.pkgname,
489+
version=args.version, release=args.release,
490+
epoch=args.epoch)
488491
finally:
489492
if progress_callback:
490493
progress_callback.finish()
@@ -893,6 +896,8 @@ def action_download_build(self, args):
893896

894897
if args.logs:
895898
cmd.extend(["-A", "*.log.gz"])
899+
# tarball of client-uploaded logs for "uploadrpm" builds
900+
cmd.extend(["-A", "uploaded-logs.tar.gz"])
896901

897902
if args.review:
898903
cmd.extend([
@@ -1748,13 +1753,24 @@ def setup_parser():
17481753
# create the parser for the "uploadrpm" command
17491754
parser_upload_rpm = subparsers.add_parser(
17501755
"uploadrpm", parents=[parser_build_parent],
1751-
help="Publish an already-built local RPM directly to a specified copr, "
1752-
"skipping the SRPM build phase entirely")
1756+
help="Publish a pre-built RPM upload tarball directly to a "
1757+
"specified copr, skipping the SRPM build phase entirely")
1758+
parser_upload_rpm.add_argument(
1759+
"tarball",
1760+
help="Local path to a .tar.gz with one top-level directory "
1761+
"containing binary RPMs (see man copr-cli uploadrpm)")
1762+
parser_upload_rpm.add_argument(
1763+
"--name", dest="pkgname", required=True,
1764+
help="Package name")
1765+
parser_upload_rpm.add_argument(
1766+
"--version", dest="version", required=True,
1767+
help="Package version")
17531768
parser_upload_rpm.add_argument(
1754-
"rpm", help="Local path to the already-built .rpm file to publish")
1769+
"--release", dest="release", required=True,
1770+
help="Package release")
17551771
parser_upload_rpm.add_argument(
1756-
"--sha256", help="Expected SHA256 hex digest of the uploaded file; "
1757-
"the server rejects the build on mismatch")
1772+
"--epoch", dest="epoch", type=int, required=False,
1773+
help="Optional package epoch")
17581774
parser_upload_rpm.set_defaults(func="action_upload_rpm")
17591775

17601776
# create the parser for the "buildpypi" command

0 commit comments

Comments
 (0)