diff --git a/resources/rebuild.sh b/resources/rebuild.sh index 708ec87176e..1924d32f153 100755 --- a/resources/rebuild.sh +++ b/resources/rebuild.sh @@ -51,8 +51,9 @@ function compile_and_install { # Build a rootfs function build_ci_rootfs { local IMAGE_NAME=$1 + local SETUP_SCRIPT=$2 prepare_docker - build_rootfs "$IMAGE_NAME" "$OUTPUT_DIR" "$PWD/overlay" "chroot.sh" + build_rootfs "$IMAGE_NAME" "$OUTPUT_DIR" "$PWD/rootfs/overlay" "$SETUP_SCRIPT" } @@ -173,7 +174,7 @@ function build_al_kernel { } function prepare_and_build_rootfs { - BIN_DIR=overlay/usr/local/bin + BIN_DIR=rootfs/overlay/usr/local/bin SRCS=(init.c fillmem.c fast_page_fault_helper.c readmem.c go_sdk_cred_provider.go go_sdk_cred_provider_with_custom_endpoint.go) if [ $ARCH == "aarch64" ]; then @@ -184,7 +185,8 @@ function prepare_and_build_rootfs { compile_and_install $BIN_DIR/$SRC done - build_ci_rootfs ubuntu:24.04 + build_ci_rootfs ubuntu:24.04 "rootfs/setup-ubuntu-ci.sh" + build_ci_rootfs amazonlinux:2023 "rootfs/setup-al2023-ci.sh" build_initramfs for SRC in ${SRCS[@]}; do diff --git a/resources/overlay/etc/systemd/system/fcnet.service b/resources/rootfs/overlay/etc/systemd/system/fcnet.service similarity index 100% rename from resources/overlay/etc/systemd/system/fcnet.service rename to resources/rootfs/overlay/etc/systemd/system/fcnet.service diff --git a/resources/overlay/etc/systemd/system/var-lib-systemd.mount b/resources/rootfs/overlay/etc/systemd/system/var-lib-systemd.mount similarity index 100% rename from resources/overlay/etc/systemd/system/var-lib-systemd.mount rename to resources/rootfs/overlay/etc/systemd/system/var-lib-systemd.mount diff --git a/resources/overlay/usr/local/bin/devmemread.c b/resources/rootfs/overlay/usr/local/bin/devmemread.c similarity index 100% rename from resources/overlay/usr/local/bin/devmemread.c rename to resources/rootfs/overlay/usr/local/bin/devmemread.c diff --git a/resources/overlay/usr/local/bin/fast_page_fault_helper.c b/resources/rootfs/overlay/usr/local/bin/fast_page_fault_helper.c similarity index 100% rename from resources/overlay/usr/local/bin/fast_page_fault_helper.c rename to resources/rootfs/overlay/usr/local/bin/fast_page_fault_helper.c diff --git a/resources/overlay/usr/local/bin/fcnet-setup.sh b/resources/rootfs/overlay/usr/local/bin/fcnet-setup.sh similarity index 100% rename from resources/overlay/usr/local/bin/fcnet-setup.sh rename to resources/rootfs/overlay/usr/local/bin/fcnet-setup.sh diff --git a/resources/overlay/usr/local/bin/fillmem.c b/resources/rootfs/overlay/usr/local/bin/fillmem.c similarity index 100% rename from resources/overlay/usr/local/bin/fillmem.c rename to resources/rootfs/overlay/usr/local/bin/fillmem.c diff --git a/resources/overlay/usr/local/bin/go_sdk_cred_provider.go/main.go b/resources/rootfs/overlay/usr/local/bin/go_sdk_cred_provider.go/main.go similarity index 100% rename from resources/overlay/usr/local/bin/go_sdk_cred_provider.go/main.go rename to resources/rootfs/overlay/usr/local/bin/go_sdk_cred_provider.go/main.go diff --git a/resources/overlay/usr/local/bin/go_sdk_cred_provider_with_custom_endpoint.go/main.go b/resources/rootfs/overlay/usr/local/bin/go_sdk_cred_provider_with_custom_endpoint.go/main.go similarity index 100% rename from resources/overlay/usr/local/bin/go_sdk_cred_provider_with_custom_endpoint.go/main.go rename to resources/rootfs/overlay/usr/local/bin/go_sdk_cred_provider_with_custom_endpoint.go/main.go diff --git a/resources/overlay/usr/local/bin/init.c b/resources/rootfs/overlay/usr/local/bin/init.c similarity index 100% rename from resources/overlay/usr/local/bin/init.c rename to resources/rootfs/overlay/usr/local/bin/init.c diff --git a/resources/overlay/usr/local/bin/readmem.c b/resources/rootfs/overlay/usr/local/bin/readmem.c similarity index 100% rename from resources/overlay/usr/local/bin/readmem.c rename to resources/rootfs/overlay/usr/local/bin/readmem.c diff --git a/resources/rootfs/setup-al2023-ci.sh b/resources/rootfs/setup-al2023-ci.sh new file mode 100644 index 00000000000..363fbcbc7a8 --- /dev/null +++ b/resources/rootfs/setup-al2023-ci.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# Copyright 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# fail if we encounter an error, uninitialized variable or a pipe breaks +set -eu -o pipefail + +# be verbose +set -x +PS4='+\t ' + +packages="systemd-udev openssh-server iproute socat iperf3 iputils fio kmod tmux hwloc vim-minimal trace-cmd strace python3-boto3 pciutils tar passwd procps-ng findutils e2fsprogs" + +# certain packages that are required in our CI tests are not available on Amazon Linux. +# Build these from source so tests work properly. +function build_unavailable_packages() { + dnf install -y gcc make git + + # linuxptp + git clone https://git.code.sf.net/p/linuxptp/code /tmp/linuxptp + make -C /tmp/linuxptp + make -C /tmp/linuxptp install prefix=/usr + rm -rf /tmp/linuxptp + + # msr-tools (x86_64 only) + if [ "$(uname -m)" == "x86_64" ]; then + git clone https://github.com/intel/msr-tools.git /tmp/msr-tools + cd /tmp/msr-tools + CFLAGS="-Wall -O2 -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64" + gcc $CFLAGS -o /usr/sbin/rdmsr rdmsr.c + gcc $CFLAGS -o /usr/sbin/wrmsr wrmsr.c + cd / + rm -rf /tmp/msr-tools + fi + + dnf remove -y gcc make git +} + +arch=$(uname -m) +if [ "${arch}" == "x86_64" ]; then + packages="$packages cpuid" +fi + +# Update local package list and install packages +dnf makecache +dnf install --setopt=install_weak_deps=False -y $packages +build_unavailable_packages + +# Set a hostname. +echo "al2023-fc-uvm" >/etc/hostname + +passwd -d root + +# The serial getty service hooks up the login prompt to the kernel console +# at ttyS0 (where Firecracker connects its serial console). We'll set it up +# for autologin to avoid the login prompt. +mkdir "/etc/systemd/system/serial-getty@ttyS0.service.d/" +cat <<'EOF' >"/etc/systemd/system/serial-getty@ttyS0.service.d/override.conf" +[Service] +# systemd requires this empty ExecStart line to override +ExecStart= +ExecStart=-/sbin/agetty --autologin root -o '-p -- \\u' --keep-baud 115200,38400,9600 %I dumb +EOF + +# Generate an SSH key for openssh-server to use. +ssh-keygen -A + +# Setup fcnet service. This is a custom Firecracker setup for assigning IPs +# to the network interfaces in the guests spawned by the CI. +# openssh-server on AL2023 requires an SSH key to be generated (vs. ubuntu which does not need this). +ln -s /etc/systemd/system/fcnet.service /etc/systemd/system/sysinit.target.wants/fcnet.service + +# Disable resolved and ntpd +rm -f /etc/systemd/system/multi-user.target.wants/systemd-resolved.service +rm -f /etc/systemd/system/dbus-org.freedesktop.resolve1.service +rm -f /etc/systemd/system/sysinit.target.wants/systemd-timesyncd.service + +# don't need this +rm -vf /etc/systemd/system/timers.target.wants/* + +systemctl enable var-lib-systemd.mount + +# disable Predictable Network Interface Names to keep ethN names +# even with PCI enabled +ln -s /dev/null /etc/systemd/network/99-default.link + +#### trim image https://wiki.ubuntu.com/ReducingDiskFootprint +# this does not save much, but oh well +rm -rf /usr/share/{doc,man,info,locale} + +cat >>/etc/sysctl.conf </root/manifest + +# Make systemd mountpoint +mkdir -pv $rootfs/var/lib/systemd + +# So rpm works +mkdir -pv $rootfs/var/lib/rpm diff --git a/resources/chroot.sh b/resources/rootfs/setup-ubuntu-ci.sh similarity index 100% rename from resources/chroot.sh rename to resources/rootfs/setup-ubuntu-ci.sh diff --git a/tests/conftest.py b/tests/conftest.py index 6b49e898ddc..7d341b15d47 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -532,25 +532,49 @@ def guest_kernel_fxt(request, record_property): ) +def rootfs_fxt(request): + """Return a read-only rootfs.""" + if request.param is None: + pytest.fail(f"No disk artifacts found in {ARTIFACT_DIR}") + return request.param + + +def rootfs_rw_fxt(request): + """Return a writeable rootfs.""" + if request.param is None: + pytest.fail(f"No disk artifacts found in {ARTIFACT_DIR}") + return request.param + + +def match_rootfs_to_kernel(request): + """For 5.10, use Ubuntu as rootfs, otherwise use AL2023. + Reason: AL2023 does not officially support 5.10""" + for name in request.fixturenames: + if name.startswith("guest_kernel"): + kernel = request.getfixturevalue(name) + if kernel and kernel.stem[2:] == "linux-5.10": + return "ubuntu" + break + return "amazonlinux" + + @pytest.fixture -def rootfs(): - """Return an Ubuntu 24.04 read-only rootfs""" - disk_list = disks("ubuntu-24.04.squashfs") +def rootfs(request): + """Return a read-only rootfs. Ubuntu for 5.10, AL2023 for 6.1.""" + distro = match_rootfs_to_kernel(request) + disk_list = disks(f"{distro}*.squashfs") if not disk_list: - pytest.fail( - f"No disk artifacts found matching 'ubuntu-24.04.squashfs' in {ARTIFACT_DIR}" - ) + pytest.fail(f"No {distro} squashfs found in {ARTIFACT_DIR}") return disk_list[0] @pytest.fixture -def rootfs_rw(): - """Return an Ubuntu 24.04 ext4 rootfs""" - disk_list = disks("ubuntu-24.04.ext4") +def rootfs_rw(request): + """Return a writeable rootfs. Ubuntu for 5.10, AL2023 for 6.1.""" + distro = match_rootfs_to_kernel(request) + disk_list = disks(f"{distro}*.ext4") if not disk_list: - pytest.fail( - f"No disk artifacts found matching 'ubuntu-24.04.ext4' in {ARTIFACT_DIR}" - ) + pytest.fail(f"No {distro} ext4 found in {ARTIFACT_DIR}") return disk_list[0] @@ -598,7 +622,7 @@ def artifact_dir(): def uvm_plain_any(microvm_factory, guest_kernel, rootfs, pci_enabled): """All guest kernels kernel: all - rootfs: Ubuntu 24.04 + rootfs: Ubuntu for 5.10, AL2023 for 6.1 """ return microvm_factory.build(guest_kernel, rootfs, pci=pci_enabled) diff --git a/tests/framework/artifacts.py b/tests/framework/artifacts.py index 7715727ac01..392c30d024c 100644 --- a/tests/framework/artifacts.py +++ b/tests/framework/artifacts.py @@ -50,3 +50,10 @@ def kernel_params(glob="vmlinux-*", select=kernels, artifact_dir=ARTIFACT_DIR) - return [ pytest.param(kernel, id=kernel.name) for kernel in select(glob, artifact_dir) ] or [pytest.param(None, id="no-kernel-found")] + + +def disk_params(glob, select=disks): + """Return disk artifacts as pytest params""" + return [pytest.param(disk, id=disk.stem) for disk in select(glob)] or [ + pytest.param(None, id="no-disk-found") + ] diff --git a/tests/framework/guest.py b/tests/framework/guest.py new file mode 100644 index 00000000000..fd4e8753ee6 --- /dev/null +++ b/tests/framework/guest.py @@ -0,0 +1,25 @@ +# Copyright 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Guest distro detection and distro-specific properties.""" + +from pathlib import Path + + +class GuestDistro: + """Distro-specific guest properties, inferred from rootfs filename.""" + + def __init__(self, rootfs_path: Path): + name = rootfs_path.stem.lower() + if "ubuntu" in name: + self.hostname = "ubuntu-fc-uvm" + self.ssh_service = "ssh.service" + self.os_release_token = "ID=ubuntu" + self.shell_prompt = f"{self.hostname}:~#" + elif "amazon" in name or "al2023" in name: + self.hostname = "al2023-fc-uvm" + self.ssh_service = "sshd.service" + self.os_release_token = 'ID="amzn"' + self.shell_prompt = f"[root@{self.hostname} ~]#" + else: + raise ValueError(f"Unknown guest distro for rootfs: {rootfs_path}") diff --git a/tests/framework/microvm.py b/tests/framework/microvm.py index 2337723fe7a..a2711ff4c65 100644 --- a/tests/framework/microvm.py +++ b/tests/framework/microvm.py @@ -34,6 +34,7 @@ import host_tools.network as net_tools from framework import utils from framework.defs import DEFAULT_BINARY_DIR, MAX_API_CALL_DURATION_MS +from framework.guest import GuestDistro from framework.http_api import Api from framework.jailer import JailerContext from framework.microvm_helpers import MicrovmHelpers @@ -221,6 +222,7 @@ def __init__( self.kernel_file = None self.rootfs_file = None + self.distro = None self.ssh_key = None self.initrd_file = None self.boot_args = None @@ -1062,6 +1064,7 @@ def make_snapshot( snapshot_type=snapshot_type, meta={ "kernel_file": str(self.kernel_file), + "rootfs_file": str(self.rootfs_file) if self.rootfs_file else None, "vcpus_count": self.vcpus_count, }, ) @@ -1125,6 +1128,9 @@ def restore_from_snapshot( setattr(self, key, value) # Adjust things just in case self.kernel_file = Path(self.kernel_file) + if self.rootfs_file: + self.rootfs_file = Path(self.rootfs_file) + self.distro = GuestDistro(self.rootfs_file) iface_overrides = [] if rename_interfaces is not None: @@ -1306,6 +1312,7 @@ def build(self, kernel=None, rootfs=None, **kwargs): rootfs_path = Path(vm.path) / rootfs.name shutil.copyfile(rootfs, rootfs_path) vm.rootfs_file = rootfs_path + vm.distro = GuestDistro(rootfs_path) vm.ssh_key = ssh_key return vm diff --git a/tests/framework/utils_vsock.py b/tests/framework/utils_vsock.py index 9561c1c26f2..eac27599d47 100644 --- a/tests/framework/utils_vsock.py +++ b/tests/framework/utils_vsock.py @@ -163,7 +163,7 @@ def check_guest_connections(vm, server_port_path, blob_path, blob_hash): # Needed to execute the bash script that tests for concurrent # vsock guest initiated connections. vm.ssh.check_output( - "echo 1024 > /sys/fs/cgroup/system.slice/ssh.service/pids.max" + f"echo 1024 > /sys/fs/cgroup/system.slice/{vm.distro.ssh_service}/pids.max" ) # Build the guest worker sub-command. diff --git a/tests/integration_tests/functional/test_kernel_cmdline.py b/tests/integration_tests/functional/test_kernel_cmdline.py index 520fd88822e..ee05161fcd6 100644 --- a/tests/integration_tests/functional/test_kernel_cmdline.py +++ b/tests/integration_tests/functional/test_kernel_cmdline.py @@ -18,14 +18,14 @@ def test_init_params(uvm_plain): vm.memory_monitor = None # We will override the init with /bin/cat so that we try to read the - # Ubuntu version from the /etc/issue file. + # distro info from the /etc/os-release file. vm.basic_config( vcpu_count=1, - boot_args="console=ttyS0 reboot=k panic=1 swiotlb=noforce init=/bin/cat -- /etc/issue", + boot_args="console=ttyS0 reboot=k panic=1 swiotlb=noforce init=/bin/cat -- /etc/os-release", ) vm.start() serial = Serial(vm) serial.open() # If the string does not show up, the test will fail. - serial.rx(token="Ubuntu 24.04") + serial.rx(token=vm.distro.os_release_token) diff --git a/tests/integration_tests/functional/test_serial_io.py b/tests/integration_tests/functional/test_serial_io.py index f11644b6b54..8a0922a401b 100644 --- a/tests/integration_tests/functional/test_serial_io.py +++ b/tests/integration_tests/functional/test_serial_io.py @@ -31,7 +31,7 @@ def test_serial_after_snapshot(uvm_plain, microvm_factory): microvm.start() # looking for the # prompt at the end - serial.rx("ubuntu-fc-uvm:~#") + serial.rx(microvm.distro.shell_prompt) # Create snapshot. snapshot = microvm.snapshot_full() @@ -99,7 +99,7 @@ def test_serial_active_tx_snapshot(uvm_plain, microvm_factory): # Send Ctrl-C to the guest to stop the ongoing transmission and regain the shell serial.tx("\x03", end="") # looking for the # prompt at the end - serial.rx("ubuntu-fc-uvm:~#") + serial.rx(vm.distro.shell_prompt) serial.tx("pwd") res = serial.rx("#") assert "/root" in res @@ -124,7 +124,7 @@ def test_serial_console_login(uvm_plain_any): serial = Serial(microvm) serial.open() - serial.rx("ubuntu-fc-uvm:") + serial.rx(microvm.distro.shell_prompt) serial.tx("id") serial.rx("uid=0(root) gid=0(root) groups=0(root)") diff --git a/tools/test-popular-containers/build_rootfs.sh b/tools/test-popular-containers/build_rootfs.sh index 9e2a89eab1b..3d39e93d920 100755 --- a/tools/test-popular-containers/build_rootfs.sh +++ b/tools/test-popular-containers/build_rootfs.sh @@ -14,7 +14,7 @@ source "$TOPDIR/tools/functions" USER_UID=$(stat -c '%u' "$TOPDIR") USER_GID=$(stat -c '%g' "$TOPDIR") -OVERLAY_DIR="$TOPDIR/resources/overlay" +OVERLAY_DIR="$TOPDIR/resources/rootfs/overlay" SETUP_SCRIPT="setup-minimal.sh" OUTPUT_DIR=$PWD