feat(artifacts): create an AL2023 CI rootfs

resources/rebuild.sh

@@ -51,8 +51,9 @@ function compile_and_install {
 # Build a rootfs
 function build_ci_rootfs {
     local IMAGE_NAME=$1
+    local SETUP_SCRIPT=$2
     prepare_docker
-    build_rootfs "$IMAGE_NAME" "$OUTPUT_DIR" "$PWD/overlay" "chroot.sh"
+    build_rootfs "$IMAGE_NAME" "$OUTPUT_DIR" "$PWD/rootfs/overlay" "$SETUP_SCRIPT"
 }
 
 
@@ -173,7 +174,7 @@ function build_al_kernel {
 }
 
 function prepare_and_build_rootfs {
-    BIN_DIR=overlay/usr/local/bin
+    BIN_DIR=rootfs/overlay/usr/local/bin
 
     SRCS=(init.c fillmem.c fast_page_fault_helper.c readmem.c go_sdk_cred_provider.go go_sdk_cred_provider_with_custom_endpoint.go)
     if [ $ARCH == "aarch64" ]; then
@@ -184,7 +185,8 @@ function prepare_and_build_rootfs {
         compile_and_install $BIN_DIR/$SRC
     done
 
-    build_ci_rootfs ubuntu:24.04
+    build_ci_rootfs ubuntu:24.04 "rootfs/setup-ubuntu-ci.sh"
+    build_ci_rootfs amazonlinux:2023 "rootfs/setup-al2023-ci.sh"
     build_initramfs
 
     for SRC in ${SRCS[@]}; do

resources/rootfs/setup-al2023-ci.sh

@@ -0,0 +1,106 @@
+#!/bin/bash
+# Copyright 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# fail if we encounter an error, uninitialized variable or a pipe breaks
+set -eu -o pipefail
+
+# be verbose
+set -x
+PS4='+\t '
+
+packages="systemd-udev openssh-server iproute socat iperf3 iputils fio kmod tmux hwloc vim-minimal trace-cmd strace python3-boto3 pciutils tar passwd procps-ng findutils e2fsprogs"
+
+# certain packages that are required in our CI tests are not available on Amazon Linux.
+# Build these from source so tests work properly.
+function build_unavailable_packages() {
+  dnf install -y gcc make git
+
+  # linuxptp
+  git clone https://git.code.sf.net/p/linuxptp/code /tmp/linuxptp
+  make -C /tmp/linuxptp
+  make -C /tmp/linuxptp install prefix=/usr
+  rm -rf /tmp/linuxptp
+
+  # msr-tools (x86_64 only)
+  if [ "$(uname -m)" == "x86_64" ]; then
+    git clone https://github.com/intel/msr-tools.git /tmp/msr-tools
+    cd /tmp/msr-tools
+    CFLAGS="-Wall -O2 -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64"
+    gcc $CFLAGS -o /usr/sbin/rdmsr rdmsr.c
+    gcc $CFLAGS -o /usr/sbin/wrmsr wrmsr.c
+    cd /
+    rm -rf /tmp/msr-tools
+  fi
+
+  dnf remove -y gcc make git
+}
+
+arch=$(uname -m)
+if [ "${arch}" == "x86_64" ]; then
+  packages="$packages cpuid"
+fi
+
+# Update local package list and install packages
+dnf makecache
+dnf install --setopt=install_weak_deps=False -y $packages
+build_unavailable_packages
+
+# Set a hostname.
+echo "al2023-fc-uvm" >/etc/hostname
+
+passwd -d root
+
+# The serial getty service hooks up the login prompt to the kernel console
+# at ttyS0 (where Firecracker connects its serial console). We'll set it up
+# for autologin to avoid the login prompt.
+mkdir "/etc/systemd/system/serial-getty@ttyS0.service.d/"
+cat <<'EOF' >"/etc/systemd/system/serial-getty@ttyS0.service.d/override.conf"
+[Service]
+# systemd requires this empty ExecStart line to override
+ExecStart=
+ExecStart=-/sbin/agetty --autologin root -o '-p -- \\u' --keep-baud 115200,38400,9600 %I dumb
+EOF
+
+# Generate an SSH key for openssh-server to use.
+ssh-keygen -A
+
+# Setup fcnet service. This is a custom Firecracker setup for assigning IPs
+# to the network interfaces in the guests spawned by the CI.
+# openssh-server on AL2023 requires an SSH key to be generated (vs. ubuntu which does not need this).
+ln -s /etc/systemd/system/fcnet.service /etc/systemd/system/sysinit.target.wants/fcnet.service
+
+# Disable resolved and ntpd
+rm -f /etc/systemd/system/multi-user.target.wants/systemd-resolved.service
+rm -f /etc/systemd/system/dbus-org.freedesktop.resolve1.service
+rm -f /etc/systemd/system/sysinit.target.wants/systemd-timesyncd.service
+
+# don't need this
+rm -vf /etc/systemd/system/timers.target.wants/*
+
+systemctl enable var-lib-systemd.mount
+
+# disable Predictable Network Interface Names to keep ethN names
+# even with PCI enabled
+ln -s /dev/null /etc/systemd/network/99-default.link
+
+#### trim image https://wiki.ubuntu.com/ReducingDiskFootprint
+# this does not save much, but oh well
+rm -rf /usr/share/{doc,man,info,locale}
+
+cat >>/etc/sysctl.conf <<EOF
+# This avoids a SPECTRE vuln
+kernel.unprivileged_bpf_disabled=1
+EOF
+
+# Drop dnf caches to reduce size
+dnf clean all
+
+# Build a manifest
+rpm -qa --qf '%{NAME}\t%{VERSION}-%{RELEASE}\n' >/root/manifest
+
+# Make systemd mountpoint
+mkdir -pv $rootfs/var/lib/systemd
+
+# So rpm works
+mkdir -pv $rootfs/var/lib/rpm

tests/conftest.py

@@ -532,25 +532,49 @@ def guest_kernel_fxt(request, record_property):
 )
 
 
+def rootfs_fxt(request):
+    """Return a read-only rootfs."""
+    if request.param is None:
+        pytest.fail(f"No disk artifacts found in {ARTIFACT_DIR}")
+    return request.param
+
+
+def rootfs_rw_fxt(request):
+    """Return a writeable rootfs."""
+    if request.param is None:
+        pytest.fail(f"No disk artifacts found in {ARTIFACT_DIR}")
+    return request.param
+
+
+def match_rootfs_to_kernel(request):
+    """For 5.10, use Ubuntu as rootfs, otherwise use AL2023.
+    Reason: AL2023 does not officially support 5.10"""
+    for name in request.fixturenames:
+        if name.startswith("guest_kernel"):
+            kernel = request.getfixturevalue(name)
+            if kernel and kernel.stem[2:] == "linux-5.10":
+                return "ubuntu"
+            break
+    return "amazonlinux"
+
+
 @pytest.fixture
-def rootfs():
-    """Return an Ubuntu 24.04 read-only rootfs"""
-    disk_list = disks("ubuntu-24.04.squashfs")
+def rootfs(request):
+    """Return a read-only rootfs. Ubuntu for 5.10, AL2023 for 6.1."""
+    distro = match_rootfs_to_kernel(request)
+    disk_list = disks(f"{distro}*.squashfs")
     if not disk_list:
-        pytest.fail(
-            f"No disk artifacts found matching 'ubuntu-24.04.squashfs' in {ARTIFACT_DIR}"
-        )
+        pytest.fail(f"No {distro} squashfs found in {ARTIFACT_DIR}")
     return disk_list[0]
 
 
 @pytest.fixture
-def rootfs_rw():
-    """Return an Ubuntu 24.04 ext4 rootfs"""
-    disk_list = disks("ubuntu-24.04.ext4")
+def rootfs_rw(request):
+    """Return a writeable rootfs. Ubuntu for 5.10, AL2023 for 6.1."""
+    distro = match_rootfs_to_kernel(request)
+    disk_list = disks(f"{distro}*.ext4")
     if not disk_list:
-        pytest.fail(
-            f"No disk artifacts found matching 'ubuntu-24.04.ext4' in {ARTIFACT_DIR}"
-        )
+        pytest.fail(f"No {distro} ext4 found in {ARTIFACT_DIR}")
     return disk_list[0]
 
 
@@ -598,7 +622,7 @@ def artifact_dir():
 def uvm_plain_any(microvm_factory, guest_kernel, rootfs, pci_enabled):
     """All guest kernels
     kernel: all
-    rootfs: Ubuntu 24.04
+    rootfs: Ubuntu for 5.10, AL2023 for 6.1
     """
     return microvm_factory.build(guest_kernel, rootfs, pci=pci_enabled)
 

tests/framework/artifacts.py

@@ -50,3 +50,10 @@ def kernel_params(glob="vmlinux-*", select=kernels, artifact_dir=ARTIFACT_DIR) -
     return [
         pytest.param(kernel, id=kernel.name) for kernel in select(glob, artifact_dir)
     ] or [pytest.param(None, id="no-kernel-found")]
+
+
+def disk_params(glob, select=disks):
+    """Return disk artifacts as pytest params"""
+    return [pytest.param(disk, id=disk.stem) for disk in select(glob)] or [
+        pytest.param(None, id="no-disk-found")
+    ]

tests/framework/guest.py

@@ -0,0 +1,25 @@
+# Copyright 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Guest distro detection and distro-specific properties."""
+
+from pathlib import Path
+
+
+class GuestDistro:
+    """Distro-specific guest properties, inferred from rootfs filename."""
+
+    def __init__(self, rootfs_path: Path):
+        name = rootfs_path.stem.lower()
+        if "ubuntu" in name:
+            self.hostname = "ubuntu-fc-uvm"
+            self.ssh_service = "ssh.service"
+            self.os_release_token = "ID=ubuntu"
+            self.shell_prompt = f"{self.hostname}:~#"
+        elif "amazon" in name or "al2023" in name:
+            self.hostname = "al2023-fc-uvm"
+            self.ssh_service = "sshd.service"
+            self.os_release_token = 'ID="amzn"'
+            self.shell_prompt = f"[root@{self.hostname} ~]#"
+        else:
+            raise ValueError(f"Unknown guest distro for rootfs: {rootfs_path}")

tests/framework/microvm.py

@@ -34,6 +34,7 @@
 import host_tools.network as net_tools
 from framework import utils
 from framework.defs import DEFAULT_BINARY_DIR, MAX_API_CALL_DURATION_MS
+from framework.guest import GuestDistro
 from framework.http_api import Api
 from framework.jailer import JailerContext
 from framework.microvm_helpers import MicrovmHelpers
@@ -221,6 +222,7 @@ def __init__(
 
         self.kernel_file = None
         self.rootfs_file = None
+        self.distro = None
         self.ssh_key = None
         self.initrd_file = None
         self.boot_args = None
@@ -1062,6 +1064,7 @@ def make_snapshot(
             snapshot_type=snapshot_type,
             meta={
                 "kernel_file": str(self.kernel_file),
+                "rootfs_file": str(self.rootfs_file) if self.rootfs_file else None,
                 "vcpus_count": self.vcpus_count,
             },
         )
@@ -1125,6 +1128,9 @@ def restore_from_snapshot(
             setattr(self, key, value)
         # Adjust things just in case
         self.kernel_file = Path(self.kernel_file)
+        if self.rootfs_file:
+            self.rootfs_file = Path(self.rootfs_file)
+            self.distro = GuestDistro(self.rootfs_file)
 
         iface_overrides = []
         if rename_interfaces is not None:
@@ -1306,6 +1312,7 @@ def build(self, kernel=None, rootfs=None, **kwargs):
                 rootfs_path = Path(vm.path) / rootfs.name
                 shutil.copyfile(rootfs, rootfs_path)
             vm.rootfs_file = rootfs_path
+            vm.distro = GuestDistro(rootfs_path)
             vm.ssh_key = ssh_key
         return vm
 

tests/framework/utils_vsock.py

@@ -163,7 +163,7 @@ def check_guest_connections(vm, server_port_path, blob_path, blob_hash):
         # Needed to execute the bash script that tests for concurrent
         # vsock guest initiated connections.
         vm.ssh.check_output(
-            "echo 1024 > /sys/fs/cgroup/system.slice/ssh.service/pids.max"
+            f"echo 1024 > /sys/fs/cgroup/system.slice/{vm.distro.ssh_service}/pids.max"
         )
 
         # Build the guest worker sub-command.

tests/integration_tests/functional/test_kernel_cmdline.py

@@ -18,14 +18,14 @@ def test_init_params(uvm_plain):
     vm.memory_monitor = None
 
     # We will override the init with /bin/cat so that we try to read the
-    # Ubuntu version from the /etc/issue file.
+    # distro info from the /etc/os-release file.
     vm.basic_config(
         vcpu_count=1,
-        boot_args="console=ttyS0 reboot=k panic=1 swiotlb=noforce init=/bin/cat -- /etc/issue",
+        boot_args="console=ttyS0 reboot=k panic=1 swiotlb=noforce init=/bin/cat -- /etc/os-release",
     )
 
     vm.start()
     serial = Serial(vm)
     serial.open()
     # If the string does not show up, the test will fail.
-    serial.rx(token="Ubuntu 24.04")
+    serial.rx(token=vm.distro.os_release_token)

tests/integration_tests/functional/test_serial_io.py

@@ -31,7 +31,7 @@ def test_serial_after_snapshot(uvm_plain, microvm_factory):
     microvm.start()
 
     # looking for the # prompt at the end
-    serial.rx("ubuntu-fc-uvm:~#")
+    serial.rx(microvm.distro.shell_prompt)
 
     # Create snapshot.
     snapshot = microvm.snapshot_full()
@@ -99,7 +99,7 @@ def test_serial_active_tx_snapshot(uvm_plain, microvm_factory):
     # Send Ctrl-C to the guest to stop the ongoing transmission and regain the shell
     serial.tx("\x03", end="")
     # looking for the # prompt at the end
-    serial.rx("ubuntu-fc-uvm:~#")
+    serial.rx(vm.distro.shell_prompt)
     serial.tx("pwd")
     res = serial.rx("#")
     assert "/root" in res
@@ -124,7 +124,7 @@ def test_serial_console_login(uvm_plain_any):
 
     serial = Serial(microvm)
     serial.open()
-    serial.rx("ubuntu-fc-uvm:")
+    serial.rx(microvm.distro.shell_prompt)
     serial.tx("id")
     serial.rx("uid=0(root) gid=0(root) groups=0(root)")
 

tools/test-popular-containers/build_rootfs.sh

@@ -14,7 +14,7 @@ source "$TOPDIR/tools/functions"
 USER_UID=$(stat -c '%u' "$TOPDIR")
 USER_GID=$(stat -c '%g' "$TOPDIR")
 
-OVERLAY_DIR="$TOPDIR/resources/overlay"
+OVERLAY_DIR="$TOPDIR/resources/rootfs/overlay"
 SETUP_SCRIPT="setup-minimal.sh"
 OUTPUT_DIR=$PWD