From 228b68ce5b7405b3bda609645a90da309e465798 Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Sun, 25 Feb 2024 19:52:13 -0800 Subject: [PATCH 01/20] Added u-boot submodule --- .gitmodules | 3 +++ u-boot | 1 + 2 files changed, 4 insertions(+) create mode 160000 u-boot diff --git a/.gitmodules b/.gitmodules index 7eab220..7159958 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ path = linux url = https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/ branch = master +[submodule "u-boot"] + path = u-boot + url = https://github.com/u-boot/u-boot diff --git a/u-boot b/u-boot new file mode 160000 index 0000000..1a66a77 --- /dev/null +++ b/u-boot @@ -0,0 +1 @@ +Subproject commit 1a66a7768af7e8106c2cd93a19f4013877fb85ae From e282e27cce03682c7fd22f6128572bee06d7f402 Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Sun, 25 Feb 2024 19:52:46 -0800 Subject: [PATCH 02/20] Add u-boot build targets --- Makefile | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/Makefile b/Makefile index 0e19731..1a16217 100644 --- a/Makefile +++ b/Makefile @@ -341,6 +341,46 @@ rootfs_clean: $(SUDO) rm -f $(CPIO_FILE) $(SUDO) rm -f $(ROOTFS) +## +## U-Boot +## + +# Note: We're reusing the TARGET variable from the Linux build section + +UBOOT_SRC := $(ROOT_DIR)/u-boot +UBOOT_OUT := $(OUT_DIR)/uboot/$(ARCH) +UBOOT_CONFIG := $(UBOOT_OUT)/.config +UBOOT_BIN := $(UBOOT_OUT)/u-boot.bin + +ifeq ($(ARCH),x86_64) + UBOOT_DEFCONFIG=qemu-x86_64_defconfig +else ifeq ($(ARCH),i386) + UBOOT_DEFCONFIG=qemu-x86_defconfig +else ifeq ($(ARCH),arm64) + UBOOT_DEFCONFIG=qemu_arm64_defconfig +endif + +UBOOT_MAKE := \ + PATH=$(CLANG_DIR)/bin:$(PATH) \ + $(MAKE) \ + -C $(UBOOT_SRC) \ + HOSTCC=clang \ + O=$(UBOOT_OUT) \ + -j `nproc` + +.PHONY: uboot_defconfig +uboot_defconfig $(UBOOT_CONFIG): | $(CLANG_DIR) + + $(UBOOT_MAKE) $(UBOOT_DEFCONFIG) + +.PHONY: uboot +uboot $(UBOOT_BIN): $(UBOOT_CONFIG) | $(CLANG_DIR) + + $(UBOOT_MAKE) CROSS_COMPILE=$(TARGET)- CC=clang + cd $(UBOOT_SRC) && ./scripts/gen_compile_commands.py -d $(UBOOT_OUT) + +.PHONY: uboot_clean +uboot_clean: + + $(UBOOT_MAKE) mrproper + ## ## Run QEMU ## From 7bdb8ef1c2c27c866b04f83e81be5f015c1aa423 Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Sun, 25 Feb 2024 19:53:42 -0800 Subject: [PATCH 03/20] Run qemu with u-boot depending on UBOOT flag --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index 1a16217..d62fa03 100644 --- a/Makefile +++ b/Makefile @@ -397,6 +397,7 @@ ECHR ?= 1 ROOT ?= /dev/vda RW ?= rw KASLR ?= 0 +UBOOT ?= 0 QEMU_KERNEL_CMDLINE := selinux=0 @@ -411,6 +412,10 @@ QEMU_ARGS := \ -echr $(ECHR) \ $(QEMU_EXTRA_ARGS) +ifeq ($(UBOOT),1) + QEMU_ARGS += -bios $(UBOOT_BIN) +endif + ifneq ($(INITRD),) ifeq ($(INITRD),1) INITRD := $(CPIO_FILE) @@ -456,6 +461,9 @@ endif QEMU_ARGS += -append "$(QEMU_KERNEL_CMDLINE) $(QEMU_EXTRA_KERNEL_CMDLINE)" RUN_DEPS := $(QEMU_KERNEL_IMAGE) +ifeq ($(UBOOT),1) + RUN_DEPS += $(UBOOT_BIN) +endif .PHONY: run run: $(RUN_DEPS) | $(SHARED_DIR) From 01131da4ce08de65d2e1009c8aa57a412c75c07c Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Sun, 25 Feb 2024 19:55:20 -0800 Subject: [PATCH 04/20] Make some u-boot variables configurable --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index d62fa03..548b37d 100644 --- a/Makefile +++ b/Makefile @@ -347,17 +347,17 @@ rootfs_clean: # Note: We're reusing the TARGET variable from the Linux build section -UBOOT_SRC := $(ROOT_DIR)/u-boot -UBOOT_OUT := $(OUT_DIR)/uboot/$(ARCH) +UBOOT_SRC ?= $(ROOT_DIR)/u-boot +UBOOT_OUT ?= $(OUT_DIR)/uboot/$(ARCH) UBOOT_CONFIG := $(UBOOT_OUT)/.config UBOOT_BIN := $(UBOOT_OUT)/u-boot.bin ifeq ($(ARCH),x86_64) - UBOOT_DEFCONFIG=qemu-x86_64_defconfig + UBOOT_DEFCONFIG ?= qemu-x86_64_defconfig else ifeq ($(ARCH),i386) - UBOOT_DEFCONFIG=qemu-x86_defconfig + UBOOT_DEFCONFIG ?= qemu-x86_defconfig else ifeq ($(ARCH),arm64) - UBOOT_DEFCONFIG=qemu_arm64_defconfig + UBOOT_DEFCONFIG ?= qemu_arm64_defconfig endif UBOOT_MAKE := \ From 057969dbbc5a2b0ddabab96967f4c3a6ca5401f0 Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Sat, 24 Feb 2024 19:18:54 -0800 Subject: [PATCH 05/20] WIP --- .gitignore | 2 ++ Makefile | 86 +++++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 58dcc52..f9235f3 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ /out /.cache compile_commands.json + +/trusty diff --git a/Makefile b/Makefile index 548b37d..48bee24 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,14 @@ else ifeq ($(filter x86_64 arm64 i386,$(ARCH)),) $(error Invalid architecture $(ARCH)) endif +ifneq ($(ARCH),arm64) + # We currently only support Trusty with arm64, as we rely on ARM TF-A + ifneq ($(filter trusty,$(MAKECMDGOALS)),) + $(error Building Trusty is only supported on arm64) + endif +endif + + .PHONY: default default: linux tools-vm @@ -381,6 +389,39 @@ uboot $(UBOOT_BIN): $(UBOOT_CONFIG) | $(CLANG_DIR) uboot_clean: + $(UBOOT_MAKE) mrproper +## +## Trusty +## + +TRUSTY_SRC ?= $(ROOT_DIR)/trusty +TRUSTY_TARGET ?= qemu-generic-arm64-test-debug +TRUSTY_BUILD_ROOT ?= $(OUT_DIR)/trusty +TRUSTY_OUT := $(TRUSTY_BUILD_ROOT)/build-$(TRUSTY_TARGET) + +ATF_DIR := $(TRUSTY_OUT)/atf/qemu/debug +ATF_BL1 := $(ATF_DIR)/bl1.bin +ATF_BL33 := $(ATF_DIR)/bl33.bin + +.PHONY: trusty-init +trusty-init: + mkdir -p $(TRUSTY_SRC) + cd $(TRUSTY_SRC) && repo init -u https://android.googlesource.com/trusty/manifest -b main + cd $(TRUSTY_SRC) && repo sync -j`nproc` -c --no-tags + +.PHONY: trusty +trusty $(ATF_BL1): | $(TRUSTY_SRC) + $(TRUSTY_SRC)/trusty/vendor/google/aosp/scripts/build.py --build-root $(TRUSTY_BUILD_ROOT) --skip-tests $(TRUSTY_TARGET) + $(MAKE) trusty_bl33 + +.PHONY: trusty_bl33 +trusty_bl33 $(ATF_BL33): $(UBOOT_BIN) + rm $(ATF_BL33) + cp $(UBOOT_BIN) $(ATF_BL33) + +.PHONY: trusty_clean +trusty_clean: + rm -rf $(TRUSTY_OUT) + ## ## Run QEMU ## @@ -397,7 +438,9 @@ ECHR ?= 1 ROOT ?= /dev/vda RW ?= rw KASLR ?= 0 + UBOOT ?= 0 +TRUSTY ?= 0 QEMU_KERNEL_CMDLINE := selinux=0 @@ -407,15 +450,22 @@ QEMU_ARGS := \ -nographic \ -no-reboot \ -kernel $(QEMU_KERNEL_IMAGE) \ - -netdev user,id=eth0,hostfwd=tcp::7777-:7777,hostfwd=tcp::2222-:22,hostfwd=tcp::2223-:23 -device virtio-net-pci,netdev=eth0 \ - -virtfs local,security_model=mapped-xattr,path=$(SHARED_DIR),mount_tag=shared \ -echr $(ECHR) \ $(QEMU_EXTRA_ARGS) -ifeq ($(UBOOT),1) +ifeq ($(TRUSTY),1) + QEMU_ARGS += -bios $(ATF_BL1) +else ifeq ($(UBOOT),1) QEMU_ARGS += -bios $(UBOOT_BIN) endif +ifneq ($(TRUSTY),1) + # Trusty currently only works with a very old version of QEMU, these flags + # don't seem to work with it + QEMU_ARGS += -netdev user,id=eth0,hostfwd=tcp::7777-:7777,hostfwd=tcp::2222-:22,hostfwd=tcp::2223-:23 -device virtio-net-pci,netdev=eth0 + QEMU_ARGS += -virtfs local,security_model=mapped-xattr,path=$(SHARED_DIR),mount_tag=shared +endif + ifneq ($(INITRD),) ifeq ($(INITRD),1) INITRD := $(CPIO_FILE) @@ -432,7 +482,7 @@ ifeq ($(GDB),1) endif ifeq ($(ARCH),x86_64) - QEMU_BIN := qemu-system-x86_64 + QEMU_BIN ?= qemu-system-x86_64 QEMU_KERNEL_CMDLINE += console=ttyS0 kpti no5lvl QEMU_ARGS += -cpu kvm64,+smep,+smap @@ -442,14 +492,26 @@ ifeq ($(ARCH),x86_64) QEMU_ARGS += -accel kvm endif else ifeq ($(ARCH),i386) - QEMU_BIN := qemu-system-i386 + QEMU_BIN ?= qemu-system-i386 QEMU_KERNEL_CMDLINE += console=ttyS0 else - QEMU_BIN := qemu-system-aarch64 + ifeq ($(TRUSTY),1) + # Trusty currently only runs with its patched build of QEMU + QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/aarch64-softmmu/qemu-system-aarch64 + else + QEMU_BIN ?= qemu-system-aarch64 + endif + QEMU_KERNEL_CMDLINE += console=ttyAMA0 + ifeq ($(TRUSTY),1) + MACHINE := -machine virt,secure=on,virtualization=on + else + MACHINE := -machine virt,virtualization=on + endif + QEMU_ARGS += \ - -M virt \ + $(MACHINE) \ -cpu cortex-a53 \ -semihosting-config enable=on,target=native endif @@ -461,10 +523,18 @@ endif QEMU_ARGS += -append "$(QEMU_KERNEL_CMDLINE) $(QEMU_EXTRA_KERNEL_CMDLINE)" RUN_DEPS := $(QEMU_KERNEL_IMAGE) +RUN_DIR := $(ROOT_DIR) + ifeq ($(UBOOT),1) RUN_DEPS += $(UBOOT_BIN) endif +ifeq ($(TRUSTY),1) + RUN_DEPS += $(ATF_BL1) + # TODO: Is there a QEMU flag we can use to make this not necessary? + RUN_DIR := $(ATF_DIR) +endif + .PHONY: run run: $(RUN_DEPS) | $(SHARED_DIR) @echo "$(GREEN)Running QEMU, press 'ctrl-a x' to quit $(NC)" @@ -478,7 +548,7 @@ endif endif @echo '' - $(QEMU_BIN) $(QEMU_ARGS) + cd $(RUN_DIR) && $(QEMU_BIN) $(QEMU_ARGS) .PHONY: run-ack run-ack: run From a40e72f5f60e4a3a74cbf0b2fe514cff801f0ff0 Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Mon, 26 Feb 2024 19:36:44 -0800 Subject: [PATCH 06/20] Add Trusty GDB support --- Makefile | 2 +- scripts/gdb.sh | 11 +++++++++-- scripts/gdbinit.gdb | 9 +++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 48bee24..11e0483 100644 --- a/Makefile +++ b/Makefile @@ -539,7 +539,7 @@ endif run: $(RUN_DEPS) | $(SHARED_DIR) @echo "$(GREEN)Running QEMU, press 'ctrl-a x' to quit $(NC)" ifeq ($(GDB),1) - @echo "$(ARCH) $(ACK)" > $(OUT_DIR)/.gdb + @echo "$(ARCH) $(ACK) $(TRUSTY) $(TRUSTY_TARGET)" > $(OUT_DIR)/.gdb @echo "$(GREEN)Waiting for GDB, attach with \`scripts/gdb.sh\` $(NC)" ifdef TERMINAL_CMD diff --git a/scripts/gdb.sh b/scripts/gdb.sh index 3631098..4bd9a48 100755 --- a/scripts/gdb.sh +++ b/scripts/gdb.sh @@ -18,7 +18,7 @@ alternatively run the following command: EOF fi -IFS=" " read -r ARCH ACK < $GDB_FILE +IFS=" " read -r ARCH ACK TRUSTY TRUSTY_TARGET < $GDB_FILE if [[ "$ACK" -eq 1 ]]; then LINUX_OUT=$OUT_DIR/ack/common/$ARCH @@ -26,8 +26,15 @@ else LINUX_OUT=$OUT_DIR/linux/$ARCH fi +TRUSTY_OUT=$OUT_DIR/trusty/build-$TRUSTY_TARGET + OUTPUT=$(mktemp) -sed "s|##LINUX_OUT##|${LINUX_OUT}|g" "$GDBINIT" > "$OUTPUT" +cp "$GDBINIT" "$OUTPUT" +echo "Using gdbscript in $OUTPUT" + +sed -i "s|##LINUX_OUT##|${LINUX_OUT}|g" "$OUTPUT" +sed -i "s|##TRUSTY_OUT##|${TRUSTY_OUT}|g" "$OUTPUT" +sed -i "s|##TRUSTY##|${TRUSTY}|g" "$OUTPUT" if [[ $ARCH == "x86_64" ]]; then GDB=gdb diff --git a/scripts/gdbinit.gdb b/scripts/gdbinit.gdb index e8cfa17..a1cc0ac 100644 --- a/scripts/gdbinit.gdb +++ b/scripts/gdbinit.gdb @@ -1,4 +1,13 @@ file ##LINUX_OUT##/vmlinux source ##LINUX_OUT##/vmlinux-gdb.py + target remote :1234 + # add-symbol-file ##LINUX_OUT##/modules_install/lib/modules/5.10.107/extra/my_module.ko -s .text 0xffffffc0091b0800 + +if ##TRUSTY## == 1 + add-symbol-file ##TRUSTY_OUT##/atf/qemu/debug/bl1/bl1.elf 0x0 + add-symbol-file ##TRUSTY_OUT##/atf/qemu/debug/bl2/bl2.elf 0x0 + add-symbol-file ##TRUSTY_OUT##/atf/qemu/debug/bl31/bl31.elf 0xe0a0000 + add-symbol-file ##TRUSTY_OUT##/lk.elf 0xe200000 +endif From 9ebec4e826b301a7b975ab0173bbf7effe2aa507 Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Mon, 26 Feb 2024 19:50:04 -0800 Subject: [PATCH 07/20] Support building Trusty with bear --- Makefile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 11e0483..7b32f3e 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,8 @@ VERBOSE ?= 0 ARCH ?= x86_64 +BEAR ?= 0 + GREEN := $(shell tput setaf 2) YELLOW := $(shell tput setaf 3) NC := $(shell tput sgr0) @@ -402,6 +404,11 @@ ATF_DIR := $(TRUSTY_OUT)/atf/qemu/debug ATF_BL1 := $(ATF_DIR)/bl1.bin ATF_BL33 := $(ATF_DIR)/bl33.bin +BEAR_CMD := +ifeq ($(BEAR),1) + BEAR_CMD := bear -- +endif + .PHONY: trusty-init trusty-init: mkdir -p $(TRUSTY_SRC) @@ -410,7 +417,9 @@ trusty-init: .PHONY: trusty trusty $(ATF_BL1): | $(TRUSTY_SRC) - $(TRUSTY_SRC)/trusty/vendor/google/aosp/scripts/build.py --build-root $(TRUSTY_BUILD_ROOT) --skip-tests $(TRUSTY_TARGET) + $(BEAR_CMD) $(TRUSTY_SRC)/trusty/vendor/google/aosp/scripts/build.py \ + --build-root $(TRUSTY_BUILD_ROOT) \ + --skip-tests $(TRUSTY_TARGET) $(MAKE) trusty_bl33 .PHONY: trusty_bl33 From 2634c415947a15d2d590978edf4fc250e52dfe1d Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Tue, 27 Feb 2024 22:56:16 -0800 Subject: [PATCH 08/20] Use custom QEMU build file --- Makefile | 40 ++++++++++----- config/trusty/qemu-qemu-inc.mk | 89 ++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 12 deletions(-) create mode 100644 config/trusty/qemu-qemu-inc.mk diff --git a/Makefile b/Makefile index 7b32f3e..528e3bc 100644 --- a/Makefile +++ b/Makefile @@ -406,15 +406,36 @@ ATF_BL33 := $(ATF_DIR)/bl33.bin BEAR_CMD := ifeq ($(BEAR),1) - BEAR_CMD := bear -- + BEAR_CMD := bear --output $(TRUSTY_SRC) -- endif -.PHONY: trusty-init -trusty-init: +# Trusty is not a submodule, we need to sync it manually +$(TRUSTY_SRC): mkdir -p $(TRUSTY_SRC) + +.PHONY: trusty-init +trusty-init: | $(TRUSTY_SRC) cd $(TRUSTY_SRC) && repo init -u https://android.googlesource.com/trusty/manifest -b main cd $(TRUSTY_SRC) && repo sync -j`nproc` -c --no-tags + $(MAKE) trusty-qemu-init + +# QEMU 3.0 is used by trusty by default. Let's use something newer, we just +# need to apply some Android-specific patches +.PHONY: trusty-qemu-init +trusty-qemu-init: + # TODO: Make this a loop, put commit message of each in a comment + # TODO: Hardcode a stable version commit instead + # TODO: Can we skip the third one + cd $(TRUSTY_SRC)/external/qemu \ + && git checkout aosp/upstream-master \ + && git cherry-pick a4d024b2fdcc478402d00890965eeacb5542c12e \ + && git cherry-pick f060068503259b661be8bd8c803291ff6412d2d6 \ + && git cherry-pick 8a933fbb9c6fb8add1c74f5b523ecb44da7372fa \ + && git cherry-pick 0bfea6599b8a3ebd2c3f98bf6e0d2705e5cb609c + + sed -i 's|include project/qemu-qemu-inc.mk|include $(CONFIG_DIR)/trusty/qemu-qemu-inc.mk|g' $(TRUSTY_SRC)/trusty/device/arm/generic-arm64/project/qemu-inc.mk + .PHONY: trusty trusty $(ATF_BL1): | $(TRUSTY_SRC) $(BEAR_CMD) $(TRUSTY_SRC)/trusty/vendor/google/aosp/scripts/build.py \ @@ -460,6 +481,8 @@ QEMU_ARGS := \ -no-reboot \ -kernel $(QEMU_KERNEL_IMAGE) \ -echr $(ECHR) \ + -netdev user,id=eth0,hostfwd=tcp::7777-:7777,hostfwd=tcp::2222-:22,hostfwd=tcp::2223-:23 -device virtio-net-pci,netdev=eth0 \ + -virtfs local,security_model=mapped-xattr,path=$(SHARED_DIR),mount_tag=shared \ $(QEMU_EXTRA_ARGS) ifeq ($(TRUSTY),1) @@ -468,13 +491,6 @@ else ifeq ($(UBOOT),1) QEMU_ARGS += -bios $(UBOOT_BIN) endif -ifneq ($(TRUSTY),1) - # Trusty currently only works with a very old version of QEMU, these flags - # don't seem to work with it - QEMU_ARGS += -netdev user,id=eth0,hostfwd=tcp::7777-:7777,hostfwd=tcp::2222-:22,hostfwd=tcp::2223-:23 -device virtio-net-pci,netdev=eth0 - QEMU_ARGS += -virtfs local,security_model=mapped-xattr,path=$(SHARED_DIR),mount_tag=shared -endif - ifneq ($(INITRD),) ifeq ($(INITRD),1) INITRD := $(CPIO_FILE) @@ -505,8 +521,8 @@ else ifeq ($(ARCH),i386) QEMU_KERNEL_CMDLINE += console=ttyS0 else ifeq ($(TRUSTY),1) - # Trusty currently only runs with its patched build of QEMU - QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/aarch64-softmmu/qemu-system-aarch64 + # Trusty needs to use its own build of QEMU which has some custom patches + QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/qemu-system-aarch64 else QEMU_BIN ?= qemu-system-aarch64 endif diff --git a/config/trusty/qemu-qemu-inc.mk b/config/trusty/qemu-qemu-inc.mk new file mode 100644 index 0000000..0cee562 --- /dev/null +++ b/config/trusty/qemu-qemu-inc.mk @@ -0,0 +1,89 @@ +# +# Copyright (c) 2018, Google, Inc. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# This makefile contains rules for building QEMU for running Trusty. +# It is expected that it will be included by the project that uses QEMU +# and the caller will configure the following variables: +# +# QEMU_ROOT - Root of qemu project +# QEMU_BUILD_BASE - location that will be used to store temp files and +# build results. +# QEMU_ARCH - qemu arch to build +# QEMU_TARGET - targets to build, use comma to separate targets +# if multiple targets are specified. +# +# The following variable is returned to the caller: +# QEMU_BIN - resulting qemu image +# QEMU_BUILD_BASE - location that will be used to store temp files and +# build results. +# +# + +QEMU_BIN:=$(QEMU_BUILD_BASE)/$(QEMU_ARCH)-softmmu/qemu-system-$(QEMU_ARCH) +QEMU_MAKEFILE:=$(QEMU_BUILD_BASE)/Makefile + +# Set of features disabled by the AOSP emulator. We don't need these features +# either, and we want minimal dependencies. +QEMU_AOSP_DISABLES := \ + --disable-curses \ + --disable-docs \ + --disable-glusterfs \ + --disable-gtk \ + --disable-spice \ + --disable-opengl \ + +# Warnings which pollute the build output and which can make us miss +# warnings in non-external code that we should be paying attention to. +QEMU_EXTRA_CFLAGS := \ + -Wno-address-of-packed-member \ + -Wno-format-truncation \ + -Wno-stringop-truncation \ + -Wno-array-bounds \ + +# Newer capstone releases have the headers under include/capstone +QEMU_EXTRA_CFLAGS += -I$(TRUSTY_TOP)/$(QEMU_ROOT)/capstone/include/capstone + +$(QEMU_MAKEFILE): QEMU_ROOT:=$(QEMU_ROOT) +$(QEMU_MAKEFILE): QEMU_BUILD_BASE:=$(QEMU_BUILD_BASE) +$(QEMU_MAKEFILE): QEMU_TARGET:=$(QEMU_TARGET) +$(QEMU_MAKEFILE): QEMU_AOSP_DISABLES:=$(QEMU_AOSP_DISABLES) +$(QEMU_MAKEFILE): QEMU_EXTRA_CFLAGS:=$(QEMU_EXTRA_CFLAGS) +$(QEMU_MAKEFILE): + mkdir -p $(QEMU_BUILD_BASE) + #--with-git=true sets the "git" program to /bin/true - it essentially disables git + #--disable-git-update may look like what we want, but it requests manual intervention, not disables git + # TODO(b/148904400): Our prebuilt Clang can't build QEMU yet, and there is no + # prebuilts GCC, i.e. currently we can only build QEMU with host toolchain. On + # some hosts compiler will complain about stringop truncation. + cd $(QEMU_BUILD_BASE) && $(abspath $(QEMU_ROOT)/configure) \ + --target-list=$(QEMU_TARGET) --disable-werror \ + --extra-cflags="$(QEMU_EXTRA_CFLAGS)" \ + --disable-gcrypt $(QEMU_AOSP_DISABLES) \ + --enable-slirp + +$(QEMU_BIN): QEMU_BUILD_BASE:=$(QEMU_BUILD_BASE) +$(QEMU_BIN): $(QEMU_MAKEFILE) .PHONY + $(MAKE) -C $(QEMU_BUILD_BASE) + +# Add QEMU_BIN to the list of project dependencies +EXTRA_BUILDDEPS += $(QEMU_BIN) + +QEMU_ARCH:= +QEMU_ROOT:= +QEMU_TARGET:= +QEMU_AOSP_DISABLES:= +QEMU_EXTRA_CFLAGS:= From 646a5138fadd2d277968efaa45a623d508ddd52b Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Wed, 28 Feb 2024 12:30:20 -0800 Subject: [PATCH 09/20] Add QEMU upstream remote and checkout stable branch --- Makefile | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 528e3bc..0caff5d 100644 --- a/Makefile +++ b/Makefile @@ -400,6 +400,8 @@ TRUSTY_TARGET ?= qemu-generic-arm64-test-debug TRUSTY_BUILD_ROOT ?= $(OUT_DIR)/trusty TRUSTY_OUT := $(TRUSTY_BUILD_ROOT)/build-$(TRUSTY_TARGET) +QEMU_BRANCH := stable-7.2 + ATF_DIR := $(TRUSTY_OUT)/atf/qemu/debug ATF_BL1 := $(ATF_DIR)/bl1.bin ATF_BL33 := $(ATF_DIR)/bl33.bin @@ -421,14 +423,19 @@ trusty-init: | $(TRUSTY_SRC) $(MAKE) trusty-qemu-init # QEMU 3.0 is used by trusty by default. Let's use something newer, we just -# need to apply some Android-specific patches +# need to apply some Android-specific patches: +# a4d024b2 arm_gic: Implement GICC_AIAR, GICC_AEOIR and GICC_AHPPIR +# f0600685 Fix GIC model for aliased interrupts +# 8a933fbb hw/virt/arm: double the amount of secure memory +# 0bfea659 hw/arm/virt: Commandeer most of PCIE_MMIO region for secure memory .PHONY: trusty-qemu-init trusty-qemu-init: - # TODO: Make this a loop, put commit message of each in a comment - # TODO: Hardcode a stable version commit instead - # TODO: Can we skip the third one + - cd $(TRUSTY_SRC)/external/qemu \ + && git remote add upstream https://gitlab.com/qemu-project/qemu.git + cd $(TRUSTY_SRC)/external/qemu \ - && git checkout aosp/upstream-master \ + && git fetch upstream $(QEMU_BRANCH) \ + && git checkout upstream/$(QEMU_BRANCH) \ && git cherry-pick a4d024b2fdcc478402d00890965eeacb5542c12e \ && git cherry-pick f060068503259b661be8bd8c803291ff6412d2d6 \ && git cherry-pick 8a933fbb9c6fb8add1c74f5b523ecb44da7372fa \ From 4fbbf58be0eeb1b98e308762509e688c6fbbd16e Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Thu, 29 Feb 2024 12:49:40 -0800 Subject: [PATCH 10/20] WIP: android userspace --- Makefile | 80 ++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 69 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 0caff5d..a80c265 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ SCRIPT_DIR := $(ROOT_DIR)/scripts CONFIG_DIR := $(ROOT_DIR)/config CLANG_DIR ?= $(ROOT_DIR)/toolchain/clang -CLANG_URL := https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.6/clang+llvm-15.0.6-x86_64-linux-gnu-ubuntu-18.04.tar.xz +CLANG_URL := https://github.com/llvm/llvm-project/releases/download/llvmorg-17.0.6/clang+llvm-17.0.6-x86_64-linux-gnu-ubuntu-22.04.tar.xz SHARED_DIR := $(ROOT_DIR)/shared @@ -41,7 +41,7 @@ endif .PHONY: default -default: linux tools-vm +default: linux linux_modules tools-vm .PHONY: clean clean: linux_clean tools-vm_clean @@ -151,6 +151,8 @@ else LINUX_DEFCONFIG ?= defconfig endif +TRUSTY_LINUX_CONFIG_FRAGMENT := $(ROOT_DIR)/trusty/external/linux/arch/arm64/configs/trusty_qemu_defconfig.fragment + LINUX_CONFIG_FRAGMENT ?= $(CONFIG_DIR)/config.fragment LINUX_OUT_MODULES_DEP := $(LINUX_OUT)/modules_install.stamp LINUX_MODULES_INSTALL_PATH := $(LINUX_OUT)/modules_install @@ -448,10 +450,10 @@ trusty $(ATF_BL1): | $(TRUSTY_SRC) $(BEAR_CMD) $(TRUSTY_SRC)/trusty/vendor/google/aosp/scripts/build.py \ --build-root $(TRUSTY_BUILD_ROOT) \ --skip-tests $(TRUSTY_TARGET) - $(MAKE) trusty_bl33 + $(MAKE) trusty-bl33 -.PHONY: trusty_bl33 -trusty_bl33 $(ATF_BL33): $(UBOOT_BIN) +.PHONY: trusty-bl33 +trusty-bl33 $(ATF_BL33): $(UBOOT_BIN) rm $(ATF_BL33) cp $(UBOOT_BIN) $(ATF_BL33) @@ -459,6 +461,32 @@ trusty_bl33 $(ATF_BL33): $(UBOOT_BIN) trusty_clean: rm -rf $(TRUSTY_OUT) +## +## Android +## + +# TODO: We shouldn't depend on Trusty for this +TRUSTY_PREBUILT_IMAGE_DIR := $(TRUSTY_SRC)/trusty/prebuilts/aosp/android/out/target/product/trusty/ +SYSTEM_IMG := $(TRUSTY_PREBUILT_IMAGE_DIR)/system.img +VENDOR_IMG := $(TRUSTY_PREBUILT_IMAGE_DIR)/vendor.img +USERDATA_IMG := $(TRUSTY_PREBUILT_IMAGE_DIR)/userdata.img + +ANDROID_USERSPACE ?= 0 + +DTC := dtc + +ANDROID_DTS := $(OUT_DIR)/android.dts +ANDROID_DTB := $(OUT_DIR)/android.dtb +QEMU_DTB := $(OUT_DIR)/qemu.dtb +QEMU_DTS := $(OUT_DIR)/qemu.dts + +.PHONY: android_dtb +android-dtb $(ANDROID_DTB): + QEMU_EXTRA_ARGS="-M dumpdtb=$(QEMU_DTB)" $(MAKE) run + $(DTC) -I dtb -O dts $(QEMU_DTB) > $(QEMU_DTS) + cat $(QEMU_DTS) $(ATF_DIR)/firmware.android.dts > $(ANDROID_DTS) + $(DTC) -I dts -O dtb $(ANDROID_DTS) > $(ANDROID_DTB) + ## ## Run QEMU ## @@ -479,7 +507,13 @@ KASLR ?= 0 UBOOT ?= 0 TRUSTY ?= 0 -QEMU_KERNEL_CMDLINE := selinux=0 +ifeq ($(ANDROID_USERSPACE),1) + SELINUX ?= 1 +else + SELINUX ?= 0 +endif + +QEMU_KERNEL_CMDLINE := selinux=$(SELINUX) QEMU_ARGS := \ -m $(MEM) \ @@ -488,17 +522,30 @@ QEMU_ARGS := \ -no-reboot \ -kernel $(QEMU_KERNEL_IMAGE) \ -echr $(ECHR) \ - -netdev user,id=eth0,hostfwd=tcp::7777-:7777,hostfwd=tcp::2222-:22,hostfwd=tcp::2223-:23 -device virtio-net-pci,netdev=eth0 \ - -virtfs local,security_model=mapped-xattr,path=$(SHARED_DIR),mount_tag=shared \ $(QEMU_EXTRA_ARGS) +ifneq ($(TRUSTY),1) + QEMU_ARGS += -netdev user,id=eth0,hostfwd=tcp::7777-:7777,hostfwd=tcp::2222-:22,hostfwd=tcp::2223-:23 -device virtio-net-pci,netdev=eth0 + QEMU_ARGS += -virtfs local,security_model=mapped-xattr,path=$(SHARED_DIR),mount_tag=shared + #QEMU_ARGS += -virtfs local,security_model=mapped-xattr,path=$(LINUX_MODULES_INSTALL_PATH)/lib/modules,mount_tag=modules +endif + ifeq ($(TRUSTY),1) QEMU_ARGS += -bios $(ATF_BL1) else ifeq ($(UBOOT),1) QEMU_ARGS += -bios $(UBOOT_BIN) endif -ifneq ($(INITRD),) +# TODO: check that ROOTFS and ANDROID_USERSPACE aren't both passed at the same time? +ifeq ($(ANDROID_USERSPACE),1) + QEMU_ARGS += \ + -device virtio-blk,drive=vda -drive file=$(SYSTEM_IMG),index=0,if=none,id=vda,format=raw \ + -device virtio-blk,drive=vdb -drive file=$(VENDOR_IMG),index=1,if=none,id=vdb,format=raw \ + -device virtio-blk,drive=vdc -drive file=$(USERDATA_IMG),index=2,if=none,id=vdc,format=raw \ + -device virtio-net,netdev=adbnet0 -netdev user,id=adbnet0,hostfwd=tcp::5554-:5554,hostfwd=tcp::5555-:5555 + QEMU_KERNEL_CMDLINE += root=$(ROOT) $(RW) kvm-arm.mode=protected earlyprintk androidboot.hardware=qemu_trusty trusty-log.log_ratelimit_interval=0 trusty-log.log_to_dmesg=always + QEMU_ARGS += -dtb $(ANDROID_DTB) +else ifneq ($(INITRD),) ifeq ($(INITRD),1) INITRD := $(CPIO_FILE) endif @@ -529,9 +576,10 @@ else ifeq ($(ARCH),i386) else ifeq ($(TRUSTY),1) # Trusty needs to use its own build of QEMU which has some custom patches - QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/qemu-system-aarch64 + # QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/qemu-system-aarch64 + QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/aarch64-softmmu/qemu-system-aarch64 else - QEMU_BIN ?= qemu-system-aarch64 + QEMU_BIN ?= qemu-system-aarch64 endif QEMU_KERNEL_CMDLINE += console=ttyAMA0 @@ -554,6 +602,11 @@ endif QEMU_ARGS += -append "$(QEMU_KERNEL_CMDLINE) $(QEMU_EXTRA_KERNEL_CMDLINE)" +# Force using the Trusty kernel image for now, there are boot errors with other images +ifeq ($(TRUSTY),1) + QEMU_KERNEL_IMAGE := $(TRUSTY_OUT)/linux-build/arch/arm64/boot/Image +endif + RUN_DEPS := $(QEMU_KERNEL_IMAGE) RUN_DIR := $(ROOT_DIR) @@ -567,6 +620,11 @@ ifeq ($(TRUSTY),1) RUN_DIR := $(ATF_DIR) endif +ifeq ($(ANDROID_USERSPACE),1) + # We need this device tree blob to mount /vendor + RUN_DEPS += $(ANDROID_DTB) +endif + .PHONY: run run: $(RUN_DEPS) | $(SHARED_DIR) @echo "$(GREEN)Running QEMU, press 'ctrl-a x' to quit $(NC)" From fb7ded956664dceb0b30b14abaa914a6643e5d45 Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Thu, 7 Mar 2024 14:11:52 -0800 Subject: [PATCH 11/20] Add note about libslirp-dev --- config/trusty/qemu-qemu-inc.mk | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/config/trusty/qemu-qemu-inc.mk b/config/trusty/qemu-qemu-inc.mk index 0cee562..f5f55ad 100644 --- a/config/trusty/qemu-qemu-inc.mk +++ b/config/trusty/qemu-qemu-inc.mk @@ -64,11 +64,7 @@ $(QEMU_MAKEFILE): QEMU_AOSP_DISABLES:=$(QEMU_AOSP_DISABLES) $(QEMU_MAKEFILE): QEMU_EXTRA_CFLAGS:=$(QEMU_EXTRA_CFLAGS) $(QEMU_MAKEFILE): mkdir -p $(QEMU_BUILD_BASE) - #--with-git=true sets the "git" program to /bin/true - it essentially disables git - #--disable-git-update may look like what we want, but it requests manual intervention, not disables git - # TODO(b/148904400): Our prebuilt Clang can't build QEMU yet, and there is no - # prebuilts GCC, i.e. currently we can only build QEMU with host toolchain. On - # some hosts compiler will complain about stringop truncation. + # Note: `libslirp-dev` must be installed before running this command cd $(QEMU_BUILD_BASE) && $(abspath $(QEMU_ROOT)/configure) \ --target-list=$(QEMU_TARGET) --disable-werror \ --extra-cflags="$(QEMU_EXTRA_CFLAGS)" \ From e72be95453ed443d8b89caa60dc6da08b0300e4d Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Thu, 7 Mar 2024 14:29:33 -0800 Subject: [PATCH 12/20] Use correct kernel image with Trusty --- Makefile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index a80c265..163024d 100644 --- a/Makefile +++ b/Makefile @@ -401,6 +401,7 @@ TRUSTY_SRC ?= $(ROOT_DIR)/trusty TRUSTY_TARGET ?= qemu-generic-arm64-test-debug TRUSTY_BUILD_ROOT ?= $(OUT_DIR)/trusty TRUSTY_OUT := $(TRUSTY_BUILD_ROOT)/build-$(TRUSTY_TARGET) +TRUSTY_KERNEL_IMAGE := $(TRUSTY_OUT)/linux-build/arch/arm64/boot/Image QEMU_BRANCH := stable-7.2 @@ -446,7 +447,7 @@ trusty-qemu-init: sed -i 's|include project/qemu-qemu-inc.mk|include $(CONFIG_DIR)/trusty/qemu-qemu-inc.mk|g' $(TRUSTY_SRC)/trusty/device/arm/generic-arm64/project/qemu-inc.mk .PHONY: trusty -trusty $(ATF_BL1): | $(TRUSTY_SRC) +trusty $(ATF_BL1) $(TRUSTY_KERNEL_IMAGE): | $(TRUSTY_SRC) $(BEAR_CMD) $(TRUSTY_SRC)/trusty/vendor/google/aosp/scripts/build.py \ --build-root $(TRUSTY_BUILD_ROOT) \ --skip-tests $(TRUSTY_TARGET) @@ -461,6 +462,11 @@ trusty-bl33 $(ATF_BL33): $(UBOOT_BIN) trusty_clean: rm -rf $(TRUSTY_OUT) +# When Trusty is enabled, use the kernel image built by Trusty +ifeq ($(TRUSTY),1) + KERNEL_IMAGE := $(TRUSTY_KERNEL_IMAGE) +endif + ## ## Android ## @@ -602,11 +608,6 @@ endif QEMU_ARGS += -append "$(QEMU_KERNEL_CMDLINE) $(QEMU_EXTRA_KERNEL_CMDLINE)" -# Force using the Trusty kernel image for now, there are boot errors with other images -ifeq ($(TRUSTY),1) - QEMU_KERNEL_IMAGE := $(TRUSTY_OUT)/linux-build/arch/arm64/boot/Image -endif - RUN_DEPS := $(QEMU_KERNEL_IMAGE) RUN_DIR := $(ROOT_DIR) From 883278b859f1f66e84c6a865da9685f077c658c1 Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Thu, 7 Mar 2024 14:37:49 -0800 Subject: [PATCH 13/20] Warn if attempting to use ACK with TRUSTY --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 163024d..cd541fd 100644 --- a/Makefile +++ b/Makefile @@ -464,6 +464,12 @@ trusty_clean: # When Trusty is enabled, use the kernel image built by Trusty ifeq ($(TRUSTY),1) + ifeq ($(ACK),1) + ifndef QEMU_KERNEL_IMAGE + $(warning $(YELLOW)ACK was enabled, but Trusty runs with its own kernel. To force using a specific kernel image, add `QEMU_KERNEL_IMAGE=$(KERNEL_IMAGE)` to the command line $(NC)) + endif + endif + KERNEL_IMAGE := $(TRUSTY_KERNEL_IMAGE) endif From ce9679c2559b91f79a126749a3493e9ffabcbd2c Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Fri, 8 Mar 2024 15:54:56 -0800 Subject: [PATCH 14/20] Only run android-dtb for arm64 --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cd541fd..466d262 100644 --- a/Makefile +++ b/Makefile @@ -492,8 +492,11 @@ ANDROID_DTB := $(OUT_DIR)/android.dtb QEMU_DTB := $(OUT_DIR)/qemu.dtb QEMU_DTS := $(OUT_DIR)/qemu.dts -.PHONY: android_dtb +.PHONY: android-dtb android-dtb $(ANDROID_DTB): +ifneq ($(ARCH),arm64) + $(error android-dtb is only supported from arm64) +endif QEMU_EXTRA_ARGS="-M dumpdtb=$(QEMU_DTB)" $(MAKE) run $(DTC) -I dtb -O dts $(QEMU_DTB) > $(QEMU_DTS) cat $(QEMU_DTS) $(ATF_DIR)/firmware.android.dts > $(ANDROID_DTS) From edf020b4650a67c9dab211328646766bb10bcb82 Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Fri, 8 Mar 2024 15:55:45 -0800 Subject: [PATCH 15/20] Symlink to old QEMU binary --- Makefile | 4 ++-- config/trusty/qemu-qemu-inc.mk | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 466d262..e06d775 100644 --- a/Makefile +++ b/Makefile @@ -495,8 +495,9 @@ QEMU_DTS := $(OUT_DIR)/qemu.dts .PHONY: android-dtb android-dtb $(ANDROID_DTB): ifneq ($(ARCH),arm64) - $(error android-dtb is only supported from arm64) + $(error android-dtb is only supported from arm64) endif + QEMU_EXTRA_ARGS="-M dumpdtb=$(QEMU_DTB)" $(MAKE) run $(DTC) -I dtb -O dts $(QEMU_DTB) > $(QEMU_DTS) cat $(QEMU_DTS) $(ATF_DIR)/firmware.android.dts > $(ANDROID_DTS) @@ -591,7 +592,6 @@ else ifeq ($(ARCH),i386) else ifeq ($(TRUSTY),1) # Trusty needs to use its own build of QEMU which has some custom patches - # QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/qemu-system-aarch64 QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/aarch64-softmmu/qemu-system-aarch64 else QEMU_BIN ?= qemu-system-aarch64 diff --git a/config/trusty/qemu-qemu-inc.mk b/config/trusty/qemu-qemu-inc.mk index f5f55ad..88f0384 100644 --- a/config/trusty/qemu-qemu-inc.mk +++ b/config/trusty/qemu-qemu-inc.mk @@ -57,6 +57,7 @@ QEMU_EXTRA_CFLAGS := \ # Newer capstone releases have the headers under include/capstone QEMU_EXTRA_CFLAGS += -I$(TRUSTY_TOP)/$(QEMU_ROOT)/capstone/include/capstone +$(QEMU_MAKEFILE): QEMU_ARCH:=$(QEMU_ARCH) $(QEMU_MAKEFILE): QEMU_ROOT:=$(QEMU_ROOT) $(QEMU_MAKEFILE): QEMU_BUILD_BASE:=$(QEMU_BUILD_BASE) $(QEMU_MAKEFILE): QEMU_TARGET:=$(QEMU_TARGET) @@ -71,6 +72,9 @@ $(QEMU_MAKEFILE): --disable-gcrypt $(QEMU_AOSP_DISABLES) \ --enable-slirp + # Symlink to the old QEMU path so we can always find it at the same place + ln -s $(QEMU_BUILD_BASE)/qemu-system-$(QEMU_ARCH) $(QEMU_BIN) + $(QEMU_BIN): QEMU_BUILD_BASE:=$(QEMU_BUILD_BASE) $(QEMU_BIN): $(QEMU_MAKEFILE) .PHONY $(MAKE) -C $(QEMU_BUILD_BASE) From 7cdf01e4f12c5e7fa91c2bbc00b688c1f825c121 Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Fri, 8 Mar 2024 17:37:51 -0800 Subject: [PATCH 16/20] Regenerate DTB if QEMU_BIN changes --- Makefile | 50 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index e06d775..30a3deb 100644 --- a/Makefile +++ b/Makefile @@ -405,6 +405,11 @@ TRUSTY_KERNEL_IMAGE := $(TRUSTY_OUT)/linux-build/arch/arm64/boot/Image QEMU_BRANCH := stable-7.2 +ifeq ($(TRUSTY),1) + # Trusty needs to use its own build of QEMU which has some custom patches + QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/aarch64-softmmu/qemu-system-aarch64 +endif + ATF_DIR := $(TRUSTY_OUT)/atf/qemu/debug ATF_BL1 := $(ATF_DIR)/bl1.bin ATF_BL33 := $(ATF_DIR)/bl33.bin @@ -492,13 +497,21 @@ ANDROID_DTB := $(OUT_DIR)/android.dtb QEMU_DTB := $(OUT_DIR)/qemu.dtb QEMU_DTS := $(OUT_DIR)/qemu.dts +# We use DUMPING_DTB to avoid infinite recursion +DUMPING_DTB := 0 + +# We depend on QEMU_BIN because it's best if we generate this every time (as it +# the DTB can change if run with a different version QEMU and we want to make +# sure it exactly matches the DTB for the current QEMU binary being used) .PHONY: android-dtb android-dtb $(ANDROID_DTB): ifneq ($(ARCH),arm64) $(error android-dtb is only supported from arm64) endif - QEMU_EXTRA_ARGS="-M dumpdtb=$(QEMU_DTB)" $(MAKE) run +# DUMPING_DTB=1 prevents infinite recursion. It must be set as a `make` argument, +# not an environment variable + QEMU_EXTRA_ARGS="-M dumpdtb=$(QEMU_DTB)" $(MAKE) run DUMPING_DTB=1 $(DTC) -I dtb -O dts $(QEMU_DTB) > $(QEMU_DTS) cat $(QEMU_DTS) $(ATF_DIR)/firmware.android.dts > $(ANDROID_DTS) $(DTC) -I dts -O dtb $(ANDROID_DTS) > $(ANDROID_DTB) @@ -560,7 +573,12 @@ ifeq ($(ANDROID_USERSPACE),1) -device virtio-blk,drive=vdc -drive file=$(USERDATA_IMG),index=2,if=none,id=vdc,format=raw \ -device virtio-net,netdev=adbnet0 -netdev user,id=adbnet0,hostfwd=tcp::5554-:5554,hostfwd=tcp::5555-:5555 QEMU_KERNEL_CMDLINE += root=$(ROOT) $(RW) kvm-arm.mode=protected earlyprintk androidboot.hardware=qemu_trusty trusty-log.log_ratelimit_interval=0 trusty-log.log_to_dmesg=always + +# Don't add the DTB as an argument if we're in the process of dumping it +ifneq ($(DUMPING_DTB),1) QEMU_ARGS += -dtb $(ANDROID_DTB) +endif + else ifneq ($(INITRD),) ifeq ($(INITRD),1) INITRD := $(CPIO_FILE) @@ -590,13 +608,7 @@ else ifeq ($(ARCH),i386) QEMU_BIN ?= qemu-system-i386 QEMU_KERNEL_CMDLINE += console=ttyS0 else - ifeq ($(TRUSTY),1) - # Trusty needs to use its own build of QEMU which has some custom patches - QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/aarch64-softmmu/qemu-system-aarch64 - else - QEMU_BIN ?= qemu-system-aarch64 - endif - + QEMU_BIN ?= qemu-system-aarch64 QEMU_KERNEL_CMDLINE += console=ttyAMA0 ifeq ($(TRUSTY),1) @@ -631,14 +643,28 @@ ifeq ($(TRUSTY),1) endif ifeq ($(ANDROID_USERSPACE),1) - # We need this device tree blob to mount /vendor - RUN_DEPS += $(ANDROID_DTB) + # We need a device tree blob to mount /vendor. If DUMPING_DTB is set, that + # means we're already in the process of dumping the DTB and shouldn't add it + # as a run dependency, otherwise we'll run into infinite recursion + ifneq ($(DUMPING_DTB),1) + # Add `android-dtb` instead of ANDROID_DTB so that this target is forced to + # be run every time (to prevent accidentally using a DTB generated from a + # different QEMU binary) + RUN_DEPS += android-dtb + endif endif .PHONY: run run: $(RUN_DEPS) | $(SHARED_DIR) + + echo $(RUN_DEPS) +ifneq ($(DUMPING_DTB), 1) @echo "$(GREEN)Running QEMU, press 'ctrl-a x' to quit $(NC)" +endif + ifeq ($(GDB),1) + +ifneq ($(DUMPING_DTB),1) @echo "$(ARCH) $(ACK) $(TRUSTY) $(TRUSTY_TARGET)" > $(OUT_DIR)/.gdb @echo "$(GREEN)Waiting for GDB, attach with \`scripts/gdb.sh\` $(NC)" @@ -646,7 +672,9 @@ ifdef TERMINAL_CMD $(TERMINAL_CMD) $(SCRIPT_DIR)/gdb.sh endif -endif +endif # DUMPING_DTB +endif # GDB + @echo '' cd $(RUN_DIR) && $(QEMU_BIN) $(QEMU_ARGS) From b9ba7700ff209dfafeca2824fba45a4fc87f425f Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Sun, 10 Mar 2024 18:47:26 -0700 Subject: [PATCH 17/20] Configure with Trusty config fragment if Trusty is enabled --- Makefile | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 30a3deb..397b70f 100644 --- a/Makefile +++ b/Makefile @@ -158,6 +158,11 @@ LINUX_OUT_MODULES_DEP := $(LINUX_OUT)/modules_install.stamp LINUX_MODULES_INSTALL_PATH := $(LINUX_OUT)/modules_install LINUX_CONFIG := $(LINUX_OUT)/.config +LINUX_CONFIG_FRAGMENTS := $(LINUX_CONFIG_FRAGMENT) +ifeq ($(TRUSTY),1) + LINUX_CONFIG_FRAGMENTS += $(TRUSTY_LINUX_CONFIG_FRAGMENT) +endif + ifeq ($(ARCH),x86_64) TARGET := x86_64-pc-linux-gnu KERNEL_IMAGE := $(LINUX_OUT)/arch/$(ARCH)/boot/bzImage @@ -193,9 +198,14 @@ linux_defconfig $(LINUX_CONFIG): $(LINUX_CONFIG_FRAGMENT) | $(CLANG_DIR) $(LINUX_SRC)/scripts/kconfig/merge_config.sh \ -m \ $(LINUX_CONFIG) \ - $(LINUX_CONFIG_FRAGMENT) - + $(LINUX_MAKE) olddefconfig - $(SCRIPT_DIR)/check_merged_config.sh $(LINUX_CONFIG) $(LINUX_CONFIG_FRAGMENT) + $(LINUX_CONFIG_FRAGMENTS) + + + $(LINUX_MAKE) olddefconfig + + for fragment in $(LINUX_CONFIG_FRAGMENTS); do \ + $(SCRIPT_DIR)/check_merged_config.sh $(LINUX_CONFIG) $$fragment ; \ + done + .PHONY: linux_menuconfig linux_menuconfig: From 589ef2c129a176ea187ed2d2bf8f2c873b2da704 Mon Sep 17 00:00:00 2001 From: Zi Fan Tan Date: Fri, 5 Apr 2024 00:36:49 +0000 Subject: [PATCH 18/20] Fix indentation in Makefile Running `make help` requires you to specify the ARCH=arm64 architecture due to an indendation error in the Makefile. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 397b70f..efa3e7a 100644 --- a/Makefile +++ b/Makefile @@ -516,7 +516,7 @@ DUMPING_DTB := 0 .PHONY: android-dtb android-dtb $(ANDROID_DTB): ifneq ($(ARCH),arm64) - $(error android-dtb is only supported from arm64) + $(error android-dtb is only supported from arm64) endif # DUMPING_DTB=1 prevents infinite recursion. It must be set as a `make` argument, From 60752ba5dc84d0ba55c134053d33b99ea6bddebf Mon Sep 17 00:00:00 2001 From: Zi Fan Tan Date: Fri, 5 Apr 2024 00:37:46 +0000 Subject: [PATCH 19/20] Update required dependencies in README.md The `device-tree-compiler` is required when building the QEMU program in Trusty repo. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 84ce5b1..4b4af67 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ The only supported host OS is Ubuntu 22.04, things may or may not work on any ot First install the dependencies and clone the project: ```bash sudo apt update -sudo apt install -y bc bison build-essential flex git libelf-dev libssl-dev ncurses-dev gdb gdb-multiarch qemu qemu-system-x86 qemu-system-arm qemu-user-static binfmt-support llvm clang clang-tools lld lz4 binutils-aarch64-linux-gnu gcc-aarch64-linux-gnu pahole dwarves +sudo apt install -y bc bison build-essential flex git libelf-dev libssl-dev ncurses-dev gdb gdb-multiarch qemu qemu-system-x86 qemu-system-arm qemu-user-static binfmt-support llvm clang clang-tools lld lz4 binutils-aarch64-linux-gnu gcc-aarch64-linux-gnu pahole dwarves device-tree-compiler git clone --recursive https://github.com/gsingh93/linux-exploit-dev-env cd linux-exploit-dev-env ``` From 7a3842c313d8dab9d1ff9bae5539c38ddd3e0f1c Mon Sep 17 00:00:00 2001 From: Zi Fan Tan Date: Fri, 5 Apr 2024 00:38:38 +0000 Subject: [PATCH 20/20] Fix gdb scripts for Trusty debugging Update the `vmlinux` load path when debugging Trusty. --- scripts/gdb.sh | 6 ++++-- scripts/gdbinit.gdb | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/gdb.sh b/scripts/gdb.sh index 4bd9a48..3cf3235 100755 --- a/scripts/gdb.sh +++ b/scripts/gdb.sh @@ -20,14 +20,16 @@ fi IFS=" " read -r ARCH ACK TRUSTY TRUSTY_TARGET < $GDB_FILE +TRUSTY_OUT=$OUT_DIR/trusty/build-$TRUSTY_TARGET + if [[ "$ACK" -eq 1 ]]; then LINUX_OUT=$OUT_DIR/ack/common/$ARCH +elif [[ "$TRUSTY" -eq 1 ]] then + LINUX_OUT=$TRUSTY_OUT/linux-build else LINUX_OUT=$OUT_DIR/linux/$ARCH fi -TRUSTY_OUT=$OUT_DIR/trusty/build-$TRUSTY_TARGET - OUTPUT=$(mktemp) cp "$GDBINIT" "$OUTPUT" echo "Using gdbscript in $OUTPUT" diff --git a/scripts/gdbinit.gdb b/scripts/gdbinit.gdb index a1cc0ac..40e12de 100644 --- a/scripts/gdbinit.gdb +++ b/scripts/gdbinit.gdb @@ -10,4 +10,4 @@ if ##TRUSTY## == 1 add-symbol-file ##TRUSTY_OUT##/atf/qemu/debug/bl2/bl2.elf 0x0 add-symbol-file ##TRUSTY_OUT##/atf/qemu/debug/bl31/bl31.elf 0xe0a0000 add-symbol-file ##TRUSTY_OUT##/lk.elf 0xe200000 -endif +end