diff --git a/.gitignore b/.gitignore index 94caeb90f79..fc27e74733f 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,9 @@ build/ install/ +# Python bytecode (e.g. from scripts/Kconfiglib) +__pycache__ + # Development friendly files tags cscope* diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index afae1251d00..a26ffd83466 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -14,6 +14,8 @@ List of OpenSBI Contributors (Alphabetically sorted) * Atish Patra +* Benedikt Freisen + * Bin Meng * Damien Le Moal diff --git a/Makefile b/Makefile index 4654106383a..37793aaf4fd 100644 --- a/Makefile +++ b/Makefile @@ -451,10 +451,28 @@ DTSCPPFLAGS = $(CPPFLAGS) -nostdinc -nostdlib -fno-builtin -D__DTS__ -x assemble ifneq ($(DEBUG),) CFLAGS += -O0 +CPPFLAGS += -DOPENSBI_DEBUG else CFLAGS += -O2 endif +ifeq ($(UBSAN),y) +UBSAN_CC_FLAGS := -fsanitize=undefined +UBSAN_CC_FLAGS += -DUBSAN_ENABLED +UBSAN_CC_FLAGS += -fno-sanitize=vptr +UBSAN_CC_FLAGS += -fno-sanitize=float-cast-overflow +UBSAN_CC_FLAGS += -fno-sanitize=float-divide-by-zero +UBSAN_CC_FLAGS += -fsanitize-recover=undefined +UBSAN_CC_FLAGS += -fsanitize=pointer-overflow +UBSAN_CC_FLAGS += -fsanitize=alignment +UBSAN_CC_FLAGS += -fno-sanitize-recover=alignment +UBSAN_CC_FLAGS += -fno-stack-protector +ifeq ($(LLVM), y) +UBSAN_CC_FLAGS += -fno-sanitize-link-runtime +endif +CFLAGS += $(UBSAN_CC_FLAGS) +endif + ifeq ($(V), 1) ELFFLAGS += -Wl,--print-gc-sections endif diff --git a/README.md b/README.md index fd40be0b498..c19bfdde030 100644 --- a/README.md +++ b/README.md @@ -402,6 +402,5 @@ make I= install_docs [Firmware Documentation]: docs/firmware/fw.md [Domain Support]: docs/domain_support.md [Doxygen manual]: http://www.doxygen.nl/manual/index.html -[Kendryte standalone SDK]: https://github.com/kendryte/kendryte-standalone-sdk [third party notices]: ThirdPartyNotices.md [reproducible builds]: https://reproducible-builds.org diff --git a/docs/domain_support.md b/docs/domain_support.md index 93186c4a97d..e267a9f7905 100644 --- a/docs/domain_support.md +++ b/docs/domain_support.md @@ -159,6 +159,13 @@ The DT properties of a domain instance DT node are as follows: * **possible-harts** (Optional) - The list of CPU DT node phandles for the the domain instance. This list represents the possible HARTs of the domain instance. +* **root-regions-inheritance** (Optional) - A string property controlling + how memory regions are inherited from **the ROOT domain**, which are then + overlaid with regions specified in the **regions** property for additional + restrictions. The allowed values are: + * "all" - inherit all memory regions from **the ROOT domain** + * "m-only" - inherit M-mode only memory regions from **the ROOT domain** + If this DT property is absent, behavior is the same as "m-only". * **regions** (Optional) - The list of domain memory region DT node phandle and access permissions for the domain instance. Each list entry is a pair of DT node phandle and access permissions. The access permissions are @@ -173,9 +180,9 @@ The DT properties of a domain instance DT node are as follows: Any region of a domain defined in DT node cannot have only M-bits set in access permissions i.e. it cannot be an m-mode only accessible region. * **boot-hart** (Optional) - The DT node phandle of the HART booting the - domain instance. If coldboot HART is assigned to the domain instance then - this DT property is ignored and the coldboot HART is assumed to be the - boot HART of the domain instance. + domain instance. If not specified, defaults to the coldboot HART. Note that + if the coldboot HART is assigned to this domain, it will be forced as + the boot HART regardless of this property. * **next-arg1** (Optional) - The 64 bit next booting stage arg1 for the domain instance. If this DT property is not available and coldboot HART is not assigned to the domain instance then **next booting stage arg1 of coldboot HART** diff --git a/docs/platform/generic.md b/docs/platform/generic.md index c48d6a9a12b..0b896ede611 100644 --- a/docs/platform/generic.md +++ b/docs/platform/generic.md @@ -47,6 +47,7 @@ RISC-V Platforms Using Generic Platform * **SiFive HiFive Unleashed** (*[sifive_fu540.md]*) * **Spike** (*[spike.md]*) * **T-HEAD C9xx series Processors** (*[thead-c9xx.md]*) +* **Tenstorrent Atlantis Platform** (*[tt-atlantis.md]*) * **OpenPiton FPGA SoC** (*[fpga-openpiton.md]*) * **Ariane FPGA SoC** (*[fpga-ariane.md]*) @@ -57,5 +58,6 @@ RISC-V Platforms Using Generic Platform [sifive_fu540.md]: sifive_fu540.md [spike.md]: spike.md [thead-c9xx.md]: thead-c9xx.md +[tt-atlantis.md]: tt-atlantis.md [fpga-openpiton.md]: fpga-openpiton.md [fpga-ariane.md]: fpga-ariane.md diff --git a/docs/platform/platform.md b/docs/platform/platform.md index b77112ff5f1..9e78dac251d 100644 --- a/docs/platform/platform.md +++ b/docs/platform/platform.md @@ -18,9 +18,6 @@ OpenSBI currently supports the following virtual and hardware platforms: machine. More details on this platform can be found in the file *[sifive_fu540.md]*. -* **Kendryte K210 SoC**: Platform support for the Kendryte K210 SoC used on - boards such as the Kendryte KD233 or the Sipeed MAIX Dock. - * **Andes AE350 SoC**: Platform support for the Andes's SoC (AE350). More details on this platform can be found in the file *[andes-ae350.md]*. diff --git a/docs/platform/qemu_virt.md b/docs/platform/qemu_virt.md index fd87ee5af96..4513df8b46a 100644 --- a/docs/platform/qemu_virt.md +++ b/docs/platform/qemu_virt.md @@ -158,6 +158,9 @@ qemu-system-riscv32 -M virt -m 256M -nographic \ Debugging with GDB ------------------ +Note: the command line examples here assume that OpenSBI was compiled using +the `DEBUG=1 FW_TEXT_START=0x80000000` configuration. + In a first console start OpenSBI with QEMU: ``` diff --git a/docs/platform/tt-atlantis.md b/docs/platform/tt-atlantis.md new file mode 100644 index 00000000000..b9bdd23868d --- /dev/null +++ b/docs/platform/tt-atlantis.md @@ -0,0 +1,35 @@ +Tenstorrent Atlantis Platform +============================= + +The Tenstorrent Atlantis is an SoC and development board from +Tenstorrent in partnership with CoreLab Technology. It contains 8 RISC-V +RVA23 compliant Tenstorrent Ascalon cores with RISC-V AIA, RISC-V IOMMU, +and a range of devices and IO connectivity. + +To build the platform-specific library and firmware images, provide the +*PLATFORM=generic* parameter to the top level `make` command. + +Platform Options +---------------- + +The *Tenstorrent Atlantis* platform does not have any platform-specific +options. + +Building Tenstorrent Atlantis Platform +-------------------------------------- + +The Atlantis Platform is still under development. This section will be +expanded as firmware and support become available. + +QEMU support is currently being developed and initial support has been +proposed for upstream. To run QEMU that is patched with 'tt-atlantis' +machine support, run: + +``` +qemu-system-riscv64 -M tt-atlantis -nographic \ + -bios build/platform/generic/firmware/fw_payload.bin \ + -kernel /Image +``` + +Recent (6.18) Linux/riscv 64-bit defconfig kernels should run the QEMU +tt-atlantis machine. diff --git a/firmware/fw_base.S b/firmware/fw_base.S index bce9e226842..0c5c65c10e5 100644 --- a/firmware/fw_base.S +++ b/firmware/fw_base.S @@ -93,7 +93,7 @@ _sc_fail: addi t0, t0, (__SIZEOF_LONG__ * 3) blt t0, t1, 2b _relocate_done: - /* At this point we are running from link address */ + /* At this point we are running from load address */ /* Reset all registers except ra, a0, a1, a2, a3 and a4 for boot HART */ li ra, 0 @@ -234,9 +234,10 @@ _scratch_init: /* Store hartid-to-scratch function address in scratch space */ lla a4, _hartid_to_scratch REG_S a4, SBI_SCRATCH_HARTID_TO_SCRATCH_OFFSET(tp) - /* Clear trap_context and tmp0 in scratch space */ + /* Clear trap_context, tmp0 and tmp1 in scratch space */ REG_S zero, SBI_SCRATCH_TRAP_CONTEXT_OFFSET(tp) REG_S zero, SBI_SCRATCH_TMP0_OFFSET(tp) + REG_S zero, SBI_SCRATCH_TMP1_OFFSET(tp) /* Store firmware options in scratch space */ MOV_3R s0, a0, s1, a1, s2, a2 #ifdef FW_OPTIONS @@ -504,6 +505,45 @@ memcmp: csrrw tp, CSR_MSCRATCH, tp .endm +.macro TRAP_SAVE_AND_SETUP_SP_T0_NMI + /* Swap TP and MNSCRATCH (for RNMI) */ + csrrw tp, CSR_MNSCRATCH, tp + + /* Save T0 in scratch space */ + REG_S t0, SBI_SCRATCH_TMP1_OFFSET(tp) + + /* + * Set T0 to appropriate exception stack + * + * Came_From_M_Mode = ((MNSTATUS.MNPP < PRV_M) ? 1 : 0) - 1; + * Exception_Stack = TP ^ (Came_From_M_Mode & (SP ^ TP)) + */ + csrr t0, CSR_MNSTATUS + srl t0, t0, 11 /* MNPP is at bits 11-12 */ + and t0, t0, PRV_M + slti t0, t0, PRV_M + add t0, t0, -1 + xor sp, sp, tp + and t0, t0, sp + xor sp, sp, tp + xor t0, tp, t0 + + /* Save original SP on exception stack */ + REG_S sp, (SBI_TRAP_REGS_OFFSET(sp) - SBI_TRAP_CONTEXT_SIZE)(t0) + + /* Set SP to exception stack and make room for trap context */ + add sp, t0, -(SBI_TRAP_CONTEXT_SIZE) + + /* Restore T0 from scratch space */ + REG_L t0, SBI_SCRATCH_TMP1_OFFSET(tp) + + /* Save T0 on stack */ + REG_S t0, SBI_TRAP_REGS_OFFSET(t0)(sp) + + /* Swap TP and MNSCRATCH */ + csrrw tp, CSR_MNSCRATCH, tp +.endm + .macro TRAP_SAVE_MEPC_MSTATUS have_mstatush /* Save MEPC and MSTATUS CSRs */ csrr t0, CSR_MEPC @@ -518,6 +558,20 @@ memcmp: .endif .endm +.macro TRAP_SAVE_MNEPC_MNSTATUS have_mstatush + /* + * Save MNEPC and MNSTATUS CSRs (for RNMI) + * Note: Trap context structure has generic field names (mepc, mstatus), + * we store MN* CSR values into these same structure fields. + */ + csrr t0, CSR_MNEPC + REG_S t0, SBI_TRAP_REGS_OFFSET(mepc)(sp) + csrr t0, CSR_MNSTATUS + REG_S t0, SBI_TRAP_REGS_OFFSET(mstatus)(sp) + /* MNSTATUSH doesn't exist in SMRNMI spec */ + REG_S zero, SBI_TRAP_REGS_OFFSET(mstatusH)(sp) +.endm + .macro TRAP_SAVE_GENERAL_REGS_EXCEPT_SP_T0 /* Save all general regisers except SP and T0 */ REG_S zero, SBI_TRAP_REGS_OFFSET(zero)(sp) @@ -581,12 +635,36 @@ memcmp: CLEAR_MDT t0 .endm +.macro TRAP_SAVE_NMI_INFO + /* + * Save NMI trap info (MNCAUSE, no MNTVAL in spec) + * Note: Trap info structure has generic field names (cause, tval, etc.), + * we store MN* CSR values into these same structure fields. + */ + csrr t0, CSR_MNCAUSE + REG_S t0, (SBI_TRAP_REGS_SIZE + SBI_TRAP_INFO_OFFSET(cause))(sp) + /* MNTVAL doesn't exist in SMRNMI spec */ + REG_S zero, (SBI_TRAP_REGS_SIZE + SBI_TRAP_INFO_OFFSET(tval))(sp) + REG_S zero, (SBI_TRAP_REGS_SIZE + SBI_TRAP_INFO_OFFSET(tval2))(sp) + REG_S zero, (SBI_TRAP_REGS_SIZE + SBI_TRAP_INFO_OFFSET(tinst))(sp) + REG_S zero, (SBI_TRAP_REGS_SIZE + SBI_TRAP_INFO_OFFSET(gva))(sp) + + /* We are ready to take another trap, clear MDT */ + CLEAR_MDT t0 +.endm + .macro TRAP_CALL_C_ROUTINE /* Call C routine */ add a0, sp, zero call sbi_trap_handler .endm +.macro TRAP_CALL_C_RNMI_ROUTINE + /* Call C routine */ + add a0, sp, zero + call sbi_trap_rnmi_handler +.endm + .macro TRAP_RESTORE_GENERAL_REGS_EXCEPT_A0_T0 /* Restore all general regisers except A0 and T0 */ REG_L ra, SBI_TRAP_REGS_OFFSET(ra)(a0) @@ -635,6 +713,19 @@ memcmp: csrw CSR_MEPC, t0 .endm +.macro TRAP_RESTORE_MNEPC_MNSTATUS + /* + * Restore MNSTATUS and MNEPC CSRs (for RNMI) + * Note: Load from generic structure fields (mstatus, mepc) and + * restore to NMI-specific CSRs (MNSTATUS, MNEPC). + * No MNSTATUSH in SMRNMI spec. + */ + REG_L t0, SBI_TRAP_REGS_OFFSET(mstatus)(a0) + csrw CSR_MNSTATUS, t0 + REG_L t0, SBI_TRAP_REGS_OFFSET(mepc)(a0) + csrw CSR_MNEPC, t0 +.endm + .macro TRAP_RESTORE_A0_T0 /* Restore T0 */ REG_L t0, SBI_TRAP_REGS_OFFSET(t0)(a0) @@ -699,6 +790,37 @@ _trap_handler_hyp: mret + .section .entry, "ax", %progbits + .align 3 + .globl _trap_rnmi_handler +_trap_rnmi_handler: + /* + * NMI interrupt handler using MN* CSRs + * + * Context detection via MNPP (previous privilege mode): + * - If MNPP < M-mode: use exception stack (TP) + * - If MNPP == M-mode: use current stack (SP) + * This handles nested interrupt cases. + */ + TRAP_SAVE_AND_SETUP_SP_T0_NMI + + TRAP_SAVE_MNEPC_MNSTATUS 0 + + TRAP_SAVE_GENERAL_REGS_EXCEPT_SP_T0 + + TRAP_SAVE_NMI_INFO + + TRAP_CALL_C_RNMI_ROUTINE + + TRAP_RESTORE_GENERAL_REGS_EXCEPT_A0_T0 + + TRAP_RESTORE_MNEPC_MNSTATUS + + TRAP_RESTORE_A0_T0 + + /* mnret - return from NMI (SMRNMI extension) */ + .word 0x70200073 + .section .entry, "ax", %progbits .align 3 .globl _reset_regs @@ -749,14 +871,6 @@ __stack_chk_fail: la a0, .Lstack_corrupt_msg call sbi_panic - /* Initial value of the stack guard variable */ - .section .data - .align 3 - .globl __stack_chk_guard - .type __stack_chk_guard, %object -__stack_chk_guard: - RISCV_PTR 0x95B5FF5A - #ifdef FW_FDT_PATH .section .rodata .align 4 diff --git a/firmware/payloads/test_head.S b/firmware/payloads/test_head.S index 9a87e56fcf5..e4fcaaca92c 100644 --- a/firmware/payloads/test_head.S +++ b/firmware/payloads/test_head.S @@ -22,6 +22,7 @@ #define REG_L __REG_SEL(ld, lw) #define REG_S __REG_SEL(sd, sw) +#define REG_ADDW __REG_SEL(addw, add) .section .entry, "ax", %progbits .align 3 @@ -35,7 +36,7 @@ _start: #elif defined(__riscv_zalrsc) _sc_fail: lr.w t0, (a3) - addw t1, t0, a2 + REG_ADDW t1, t0, a2 sc.w t1, t1, (a3) bnez t1, _sc_fail move a3, t0 diff --git a/include/sbi/riscv_asm.h b/include/sbi/riscv_asm.h index 0cf3fc37983..7765edb78cd 100644 --- a/include/sbi/riscv_asm.h +++ b/include/sbi/riscv_asm.h @@ -209,18 +209,6 @@ int misa_xlen(void); /* Get RISC-V ISA string representation */ void misa_string(int xlen, char *out, unsigned int out_sz); -/* Disable pmp entry at a given index */ -int pmp_disable(unsigned int n); - -/* Check if the matching field is set */ -int is_pmp_entry_mapped(unsigned long entry); - -int pmp_set(unsigned int n, unsigned long prot, unsigned long addr, - unsigned long log2len); - -int pmp_get(unsigned int n, unsigned long *prot_out, unsigned long *addr_out, - unsigned long *log2len); - #endif /* !__ASSEMBLER__ */ #endif diff --git a/include/sbi/riscv_encoding.h b/include/sbi/riscv_encoding.h index b5a4ce816be..c4f708088d1 100644 --- a/include/sbi/riscv_encoding.h +++ b/include/sbi/riscv_encoding.h @@ -36,8 +36,10 @@ #define MSTATUS_SDT _UL(0x01000000) #define MSTATUS32_SD _UL(0x80000000) #if __riscv_xlen == 64 -#define MSTATUS_UXL _ULL(0x0000000300000000) -#define MSTATUS_SXL _ULL(0x0000000C00000000) +#define MSTATUS_UXL_SHIFT 32 +#define MSTATUS_UXL (_ULL(3) << MSTATUS_UXL_SHIFT) +#define MSTATUS_SXL_SHIFT 34 +#define MSTATUS_SXL (_ULL(3) << MSTATUS_SXL_SHIFT) #define MSTATUS_SBE _ULL(0x0000001000000000) #define MSTATUS_MBE _ULL(0x0000002000000000) #define MSTATUS_GVA _ULL(0x0000004000000000) @@ -56,6 +58,9 @@ #endif #define MSTATUS32_SD _UL(0x80000000) #define MSTATUS64_SD _ULL(0x8000000000000000) +#define MXL_XLEN_32 1 +#define MXL_XLEN_64 2 +#define MXL_TO_XLEN(x) (1U << (x + 4)) #define SSTATUS_SIE MSTATUS_SIE #define SSTATUS_SPIE_SHIFT MSTATUS_SPIE_SHIFT @@ -215,6 +220,10 @@ #endif +#define MNSTATUS_NMIE (_UL(0x8)) +#define MNSTATUS_MNPV (_UL(0x80)) +#define MNSTATUS_MNPP (_UL(0x1800)) + #define MHPMEVENT_SSCOF_MASK _ULL(0xFF00000000000000) #define ENVCFG_STCE (_ULL(1) << 63) @@ -261,6 +270,16 @@ #define CSR_FRM 0x002 #define CSR_FCSR 0x003 +/* User entropy source */ +#define CSR_SEED 0x015 +#define SEED_OPTS_SHIFT 30 +#define SEED_OPTS_MASK (_UL(3) << SEED_OPTS_SHIFT) +#define SEED_OPTS_BIST (_UL(0) << SEED_OPTS_SHIFT) +#define SEED_OPTS_WAIT (_UL(1) << SEED_OPTS_SHIFT) +#define SEED_OPTS_ES16 (_UL(2) << SEED_OPTS_SHIFT) +#define SEED_OPTS_DEAD (_UL(3) << SEED_OPTS_SHIFT) +#define SEED_ENTROPY_MASK 0xffff + /* User Counters/Timers */ #define CSR_CYCLE 0xc00 #define CSR_TIME 0xc01 @@ -820,6 +839,12 @@ #define CSR_CUSTOM10_M_RO_BASE 0xFC0 #define CSR_CUSTOM10_M_RO_COUNT 0x040 +/* Smrnmi extension registers */ +#define CSR_MNSCRATCH 0x740 +#define CSR_MNEPC 0x741 +#define CSR_MNCAUSE 0x742 +#define CSR_MNSTATUS 0x744 + /* ===== Trap/Exception Causes ===== */ #define CAUSE_MISALIGNED_FETCH 0x0 @@ -892,12 +917,16 @@ #define INSN_MATCH_SD 0x3023 #define INSN_MASK_SD 0x707f +#define INSN_MATCH_FLH 0x1007 +#define INSN_MASK_FLH 0x707f #define INSN_MATCH_FLW 0x2007 #define INSN_MASK_FLW 0x707f #define INSN_MATCH_FLD 0x3007 #define INSN_MASK_FLD 0x707f #define INSN_MATCH_FLQ 0x4007 #define INSN_MASK_FLQ 0x707f +#define INSN_MATCH_FSH 0x1027 +#define INSN_MASK_FSH 0x707f #define INSN_MATCH_FSW 0x2027 #define INSN_MASK_FSW 0x707f #define INSN_MATCH_FSD 0x3027 @@ -939,6 +968,13 @@ #define INSN_MATCH_C_FSWSP 0xe002 #define INSN_MASK_C_FSWSP 0xe003 +#define INSN_MATCH_C_LBU 0x8000 +#define INSN_MASK_C_LBU 0xfc03 +#define INSN_MATCH_C_SB 0x8800 +#define INSN_MASK_C_SB 0xfc03 +#define INSN_MASK_C_GENERIC_RXS_RXS 0xfc63 +#define INSN_MASK_C_GENERIC_RXS 0xfc7f + #define INSN_MATCH_C_LHU 0x8400 #define INSN_MASK_C_LHU 0xfc43 #define INSN_MATCH_C_LH 0x8440 @@ -946,11 +982,158 @@ #define INSN_MATCH_C_SH 0x8c00 #define INSN_MASK_C_SH 0xfc43 +#define INSN_MATCH_C_ZEXT_B 0x9c61 +#define INSN_MATCH_C_SEXT_B 0x9c65 +#define INSN_MATCH_C_ZEXT_H 0x9c69 +#define INSN_MATCH_C_SEXT_H 0x9c6d +#define INSN_MATCH_C_ZEXT_W 0x9c71 +#define INSN_MATCH_C_NOT 0x9c75 + +#define INSN_MATCH_C_MUL 0x9c41 + #define INSN_MASK_WFI 0xffffff00 #define INSN_MATCH_WFI 0x10500000 +#define INSN_MASK_FENCE 0x0000707f +#define INSN_MATCH_FENCE 0x0000000f #define INSN_MASK_FENCE_TSO 0xfff0707f #define INSN_MATCH_FENCE_TSO 0x8330000f +#define INSN_MASK_FENCE_I 0x0000707f +#define INSN_MATCH_FENCE_I 0x0000100f + +#define INSN_MASK_CBO 0xfff07fff +#define INSN_MATCH_CBO_CLEAN 0x0010200f +#define INSN_MATCH_CBO_FLUSH 0x0020200f +#define INSN_MATCH_CBO_INVAL 0x0000200f +#define INSN_MATCH_CBO_ZERO 0x0040200f + +/* Zawrs (no mask) */ +#define INSN_MATCH_WRS_NTO 0x00d00073 +#define INSN_MATCH_WRS_STO 0x01d00073 + +/* generic masks for instruction formats R and I */ +#define INSN_MASK_RTYPE_RD_RS1_RS2 0xfe00707f +#define INSN_MASK_ITYPE_RD_RS 0xfff0707f + +/* Zbs single-bit instructions */ +#define INSN_MATCH_BCLR 0x48001033 +#define INSN_MATCH_BCLRI 0x48001013 +#define INSN_MATCH_BEXT 0x48005033 +#define INSN_MATCH_BEXTI 0x48005013 +#define INSN_MATCH_BINV 0x68001033 +#define INSN_MATCH_BINVI 0x68001013 +#define INSN_MATCH_BSET 0x28001033 +#define INSN_MATCH_BSETI 0x28001013 + +/* Zbb */ +#define INSN_MATCH_ANDN 0x40007033 +#define INSN_MATCH_MAX 0x0a006033 +#define INSN_MATCH_MAXU 0x0a007033 +#define INSN_MATCH_MIN 0x0a004033 +#define INSN_MATCH_MINU 0x0a005033 +#define INSN_MATCH_ORN 0x40006033 +#define INSN_MATCH_ROL 0x60001033 +#define INSN_MATCH_ROR 0x60005033 +#define INSN_MATCH_RORI 0x60005013 +#define INSN_MATCH_XNOR 0x40004033 +#define INSN_MATCH_CLZ 0x60001013 +#define INSN_MATCH_CTZ 0x60101013 +#define INSN_MATCH_CPOP 0x60201013 +#define INSN_MATCH_ORC_B 0x28705013 +#define INSN_MATCH_REV8_RV32 0x69805013 +#define INSN_MATCH_REV8_RV64 0x6b805013 +#define INSN_MATCH_SEXT_B 0x60401013 +#define INSN_MATCH_SEXT_H 0x60501013 + +/* Zba */ +#define INSN_MATCH_SH1ADD 0x20002033 +#define INSN_MATCH_SH2ADD 0x20004033 +#define INSN_MATCH_SH3ADD 0x20006033 + +/* Zbc */ +#define INSN_MATCH_CLMUL 0x0a001033 +#define INSN_MATCH_CLMULH 0x0a003033 +#define INSN_MATCH_CLMULR 0x0a002033 + +/* Zbkb */ +#define INSN_MATCH_PACK 0x08004033 +#define INSN_MATCH_PACKH 0x08007033 + +/* Zba word instructions */ +#define INSN_MASK_SLLI_UW 0xfc00707f + +#define INSN_MATCH_ADD_UW 0x0800003b +#define INSN_MATCH_SH1ADD_UW 0x2000203b +#define INSN_MATCH_SH2ADD_UW 0x2000403b +#define INSN_MATCH_SH3ADD_UW 0x2000603b +#define INSN_MATCH_SLLI_UW 0x0800101b + +/* Zbb word instructions */ +#define INSN_MATCH_ROLW 0x6000103b +#define INSN_MATCH_RORW 0x6000503b + +#define INSN_MATCH_CLZW 0x6000101b +#define INSN_MATCH_CTZW 0x6010101b +#define INSN_MATCH_CPOPW 0x6020101b +#define INSN_MATCH_ZEXT_H_RV32 0x08004033 +#define INSN_MATCH_ZEXT_H_RV64 0x0800403b +#define INSN_MATCH_RORIW 0x6000501b + +/* Zfhmin floating-point FCVT */ +#define INSN_MATCH_FCVT_S_H 0x40200053 +#define INSN_MATCH_FCVT_H_S 0x44000053 +#define INSN_MATCH_FCVT_D_H 0x42200053 +#define INSN_MATCH_FCVT_H_D 0x44100053 +#define INSN_MATCH_FCVT_Q_H 0x46200053 +#define INSN_MATCH_FCVT_H_Q 0x44300053 +/* Zfh floating-point to/from integer FCVT */ +#define INSN_MATCH_FCVT_W_H 0xc4000053 +#define INSN_MATCH_FCVT_WU_H 0xc4100053 +#define INSN_MATCH_FCVT_H_W 0xd4000053 +#define INSN_MATCH_FCVT_H_WU 0xd4100053 +/* Zfhmin FMV */ +#define INSN_MATCH_FMV_X_H 0xe4000053 +#define INSN_MATCH_FMV_H_X 0xf4000053 +/* Zfa */ +#define INSN_MATCH_FLI_S 0xf0100053 +#define INSN_MATCH_FLI_D 0xf2100053 +#define INSN_MATCH_FLI_H 0xf4100053 + +#define INSN_MATCH_FMINM_S 0x28002053 +#define INSN_MATCH_FMAXM_S 0x28003053 +#define INSN_MATCH_FMINM_D 0x2a002053 +#define INSN_MATCH_FMAXM_D 0x2a003053 +#define INSN_MATCH_FMINM_H 0x2c002053 +#define INSN_MATCH_FMAXM_H 0x2c003053 + +#define INSN_MATCH_FROUND_S 0x40400053 +#define INSN_MATCH_FROUNDNX_S 0x40500053 +#define INSN_MATCH_FROUND_D 0x42400053 +#define INSN_MATCH_FROUNDNX_D 0x42500053 +#define INSN_MATCH_FROUND_H 0x44400053 +#define INSN_MATCH_FROUNDNX_H 0x44500053 + +#define INSN_MATCH_FCVTMOD_W_D 0xc2801053 + +#define INSN_MATCH_FLTQ_S 0xa0005053 +#define INSN_MATCH_FLEQ_S 0xa0004053 +#define INSN_MATCH_FLTQ_D 0xa2005053 +#define INSN_MATCH_FLEQ_D 0xa2004053 +#define INSN_MATCH_FLTQ_H 0xa4005053 +#define INSN_MATCH_FLEQ_H 0xa4004053 + +/* Zimop */ +#define INSN_MASK_MOP_R_N 0xb3c0707f +#define INSN_MATCH_MOP_R_N 0x81c04073 +#define INSN_MASK_MOP_RR_N 0xb200707f +#define INSN_MATCH_MOP_RR_N 0x82004073 +/* Zcmop */ +#define INSN_MASK_C_MOP_N 0xf8ff +#define INSN_MATCH_C_MOP_N 0x6081 + +/* Zicond */ +#define INSN_MATCH_CZERO_EQZ 0x0e005033 +#define INSN_MATCH_CZERO_NEZ 0x0e007033 #define INSN_MASK_VECTOR_UNIT_STRIDE 0xfdf0707f #define INSN_MASK_VECTOR_FAULT_ONLY_FIRST 0xfdf0707f @@ -1025,6 +1208,26 @@ #define INSN_MATCH_VS4RV 0x62800027 #define INSN_MATCH_VS8RV 0xe2800027 +/* Zvbb */ +#define INSN_MASK_VXUNARY0 0xfc0ff07f +#define INSN_MASK_VVBINARY0 0xfc00707f +#define INSN_MATCH_VANDNVV 0X04000057 +#define INSN_MATCH_VANDNVX 0x04004057 +#define INSN_MATCH_VBREVV 0x48052057 +#define INSN_MATCH_VBREV8V 0x48042057 +#define INSN_MATCH_VREV8V 0x4804a057 +#define INSN_MATCH_VCLZV 0x48062057 +#define INSN_MATCH_VCTZV 0x4806a057 +#define INSN_MATCH_VCPOPV 0x48072057 +#define INSN_MATCH_VROLVV 0x54000057 +#define INSN_MATCH_VROLVX 0x54004057 +#define INSN_MATCH_VRORVV 0x50000057 +#define INSN_MATCH_VRORVX 0x50004057 +#define INSN_MATCH_VRORVI 0x50003057 +#define INSN_MATCH_VWSLLVV 0xd4000057 +#define INSN_MATCH_VWSLLVX 0xd4004057 +#define INSN_MATCH_VWSLLVI 0xd4003057 + #define INSN_OPCODE_MASK 0x7f #define INSN_OPCODE_VECTOR_LOAD 0x07 #define INSN_OPCODE_VECTOR_STORE 0x27 @@ -1344,6 +1547,7 @@ #define VSEW_MASK 0x3 #define VLMUL_MASK 0x7 #define VD_MASK 0x1f +#define VS1_MASK 0x1f #define VS2_MASK 0x1f #define INSN_16BIT_MASK 0x3 #define INSN_32BIT_MASK 0x1c @@ -1359,6 +1563,7 @@ #define SH_VSEW 3 #define SH_VIEW 12 #define SH_VD 7 +#define SH_VS1 15 #define SH_VS2 20 #define SH_VM 25 #define SH_MEW 28 @@ -1368,6 +1573,9 @@ #define SH_RS2C 2 #define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1)) +#define RVC_LB_IMM(x) ((RV_X(x, 6, 1) << 0) | \ + (RV_X(x, 5, 1) << 1)) +#define RVC_LH_IMM(x) (RV_X(x, 5, 1) << 1) #define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \ (RV_X(x, 10, 3) << 3) | \ (RV_X(x, 5, 1) << 6)) @@ -1379,6 +1587,10 @@ #define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \ (RV_X(x, 12, 1) << 5) | \ (RV_X(x, 2, 3) << 6)) +#define RVC_SB_IMM(x) RVC_LB_IMM(x) +#define RVC_SH_IMM(x) RVC_LH_IMM(x) +#define RVC_SW_IMM(x) RVC_LW_IMM(x) +#define RVC_SD_IMM(x) RVC_LD_IMM(x) #define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \ (RV_X(x, 7, 2) << 6)) #define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \ @@ -1398,6 +1610,7 @@ #define GET_RS2S_NUM(insn) RVC_RS2S(insn) #define GET_RS2C_NUM(insn) RVC_RS2(insn) #define GET_RD_NUM(insn) ((insn & MASK_RD) >> SH_RD) +#define GET_RDS_NUM(insn) RVC_RS2S(insn) #define GET_CSR_NUM(insn) ((insn & MASK_CSR) >> SHIFT_CSR) #define GET_AQRL(insn) ((insn & MASK_AQRL) >> SHIFT_AQRL) @@ -1407,6 +1620,7 @@ #define IS_MASKED(insn) (((insn >> SH_VM) & VM_MASK) == 0) #define GET_VD(insn) ((insn >> SH_VD) & VD_MASK) +#define GET_VS1(insn) ((insn >> SH_VS1) & VS1_MASK) #define GET_VS2(insn) ((insn >> SH_VS2) & VS2_MASK) #define GET_VIEW(insn) (((insn) >> SH_VIEW) & VIEW_MASK) #define GET_MEW(insn) (((insn) >> SH_MEW) & 1) diff --git a/include/sbi/riscv_fp.h b/include/sbi/riscv_fp.h index f523c56ef7b..281fd1d9713 100644 --- a/include/sbi/riscv_fp.h +++ b/include/sbi/riscv_fp.h @@ -76,6 +76,30 @@ : "r"(value), "r"(offset) \ : "t0"); \ }) + +#define GET_F16_REG(insn, pos, regs) ((u16)GET_F32_REG(insn, pos, regs)) + +#define SET_F16_REG(insn, pos, regs, val) \ + (SET_F32_REG(insn, pos, regs, (val) | 0xffff0000)) + +#define GET_F16_REG_OR_NAN(insn, pos, regs) \ + ({ \ + u64 value = GET_F64_REG(insn, pos, regs); \ + if ((value & 0xffffffffffff0000) != 0xffffffffffff0000) \ + value = 0x7c00; \ + (u16) value; \ + }) + +#define GET_F32_REG_OR_NAN(insn, pos, regs) \ + ({ \ + u64 value = GET_F64_REG(insn, pos, regs); \ + if ((value & 0xffffffff00000000) != 0xffffffff00000000) \ + value = 0x7fc00000; \ + (u32) value; \ + }) + +#define GET_F64_REG_OR_NAN(insn, pos, regs) GET_F64_REG(insn, pos, regs) + #define GET_FCSR() csr_read(CSR_FCSR) #define SET_FCSR(value) csr_write(CSR_FCSR, (value)) #define GET_FRM() csr_read(CSR_FRM) @@ -83,7 +107,11 @@ #define GET_FFLAGS() csr_read(CSR_FFLAGS) #define SET_FFLAGS(value) csr_write(CSR_FFLAGS, (value)) -#define SET_FS_DIRTY(regs) (regs->mstatus |= MSTATUS_FS) +#define SET_FS_DIRTY(regs) do { \ + if (sbi_regs_from_virt(regs)) \ + csr_set(CSR_VSSTATUS, MSTATUS_FS); \ + regs->mstatus |= MSTATUS_FS; \ +} while(0) #define GET_F32_RS1(insn, regs) (GET_F32_REG(insn, 15, regs)) #define GET_F32_RS2(insn, regs) (GET_F32_REG(insn, 20, regs)) @@ -91,15 +119,55 @@ #define GET_F64_RS1(insn, regs) (GET_F64_REG(insn, 15, regs)) #define GET_F64_RS2(insn, regs) (GET_F64_REG(insn, 20, regs)) #define GET_F64_RS3(insn, regs) (GET_F64_REG(insn, 27, regs)) -#define SET_F32_RD(insn, regs, val) \ - (SET_F32_REG(insn, 7, regs, val), SET_FS_DIRTY(regs)) -#define SET_F64_RD(insn, regs, val) \ - (SET_F64_REG(insn, 7, regs, val), SET_FS_DIRTY(regs)) +#define SET_F32_RD(insn, regs, val) do { \ + SET_F32_REG(insn, 7, regs, val); \ + SET_FS_DIRTY(regs); \ +} while(0) +#define SET_F64_RD(insn, regs, val) do { \ + SET_F64_REG(insn, 7, regs, val); \ + SET_FS_DIRTY(regs); \ +} while(0) +#define SET_F16_RD(insn, regs, val) do { \ + SET_F16_REG(insn, 7, regs, val); \ + SET_FS_DIRTY(regs); \ +} while (0) +#define GET_F16_RS1(insn, regs) (GET_F16_REG(insn, 15, regs)) +#define GET_F16_RS2(insn, regs) (GET_F16_REG(insn, 20, regs)) +#define GET_F16_RS3(insn, regs) (GET_F16_REG(insn, 27, regs)) + +#define GET_F32_RS1_OR_NAN(insn, regs) (GET_F32_REG_OR_NAN(insn, 15, regs)) +#define GET_F32_RS2_OR_NAN(insn, regs) (GET_F32_REG_OR_NAN(insn, 20, regs)) +#define GET_F32_RS3_OR_NAN(insn, regs) (GET_F32_REG_OR_NAN(insn, 27, regs)) +#define GET_F64_RS1_OR_NAN(insn, regs) (GET_F64_REG_OR_NAN(insn, 15, regs)) +#define GET_F64_RS2_OR_NAN(insn, regs) (GET_F64_REG_OR_NAN(insn, 20, regs)) +#define GET_F64_RS3_OR_NAN(insn, regs) (GET_F64_REG_OR_NAN(insn, 27, regs)) +#define GET_F16_RS1_OR_NAN(insn, regs) (GET_F16_REG_OR_NAN(insn, 15, regs)) +#define GET_F16_RS2_OR_NAN(insn, regs) (GET_F16_REG_OR_NAN(insn, 20, regs)) +#define GET_F16_RS3_OR_NAN(insn, regs) (GET_F16_REG_OR_NAN(insn, 27, regs)) #define GET_F32_RS2C(insn, regs) (GET_F32_REG(insn, 2, regs)) -#define GET_F32_RS2S(insn, regs) (GET_F32_REG(RVC_RS2S(insn), 0, regs)) +#define GET_F32_RS2S(insn, regs) (GET_F32_REG(GET_RS2S_NUM(insn), 0, regs)) #define GET_F64_RS2C(insn, regs) (GET_F64_REG(insn, 2, regs)) -#define GET_F64_RS2S(insn, regs) (GET_F64_REG(RVC_RS2S(insn), 0, regs)) +#define GET_F64_RS2S(insn, regs) (GET_F64_REG(GET_RS2S_NUM(insn), 0, regs)) +#define SET_F32_RDS(insn, regs, val) do { \ + SET_F32_REG(GET_RDS_NUM(insn), 0, regs, val); \ + SET_FS_DIRTY(regs); \ +} while(0) +#define SET_F64_RDS(insn, regs, val) do { \ + SET_F64_REG(GET_RDS_NUM(insn), 0, regs, val); \ + SET_FS_DIRTY(regs); \ +} while(0) +#define SET_F16_RDS(insn, regs, val) do { \ + SET_F16_REG(GET_RDS_NUM(insn), 0, regs, val); \ + SET_FS_DIRTY(regs); \ +} while (0) + +#define GET_F32_RS2C_OR_NAN(insn, regs) (GET_F32_REG_OR_NAN(insn, 2, regs)) +#define GET_F32_RS2S_OR_NAN(insn, regs) \ + (GET_F32_REG_OR_NAN(RVC_RS2S(insn), 0, regs)) +#define GET_F64_RS2C_OR_NAN(insn, regs) (GET_F64_REG_OR_NAN(insn, 2, regs)) +#define GET_F64_RS2S_OR_NAN(insn, regs) \ + (GET_F64_REG_OR_NAN(RVC_RS2S(insn), 0, regs)) #endif diff --git a/include/sbi/sbi_bitmap.h b/include/sbi/sbi_bitmap.h index 596bcc7daa8..80d3fe3b06a 100644 --- a/include/sbi/sbi_bitmap.h +++ b/include/sbi/sbi_bitmap.h @@ -143,4 +143,20 @@ static inline int bitmap_weight(const unsigned long *src, int nbits) return res; } +static inline bool bitmap_empty(const unsigned long *src, int nbits) +{ + if (nbits == 0) + return true; + + if (small_const_nbits(nbits)) + return !(*src & BITMAP_LAST_WORD_MASK(nbits)); + else { + size_t i, len = BITS_TO_LONGS(nbits); + for (i = 0; i < len - 1; i++) + if (src[i]) + return false; + return !(src[len - 1] & BITMAP_LAST_WORD_MASK(nbits)); + } +} + #endif diff --git a/include/sbi/sbi_domain.h b/include/sbi/sbi_domain.h index c8a6da992a5..16edd4ce449 100644 --- a/include/sbi/sbi_domain.h +++ b/include/sbi/sbi_domain.h @@ -170,9 +170,11 @@ static inline bool sbi_domain_memregion_is_subset( const struct sbi_domain_memregion *regB) { ulong regA_start = regA->base; - ulong regA_end = regA->base + (BIT(regA->order) - 1); + ulong regA_mask = (regA->order >= __riscv_xlen) ? ~0UL : (BIT(regA->order) - 1); + ulong regA_end = regA_start + regA_mask; ulong regB_start = regB->base; - ulong regB_end = regB->base + (BIT(regB->order) - 1); + ulong regB_mask = (regB->order >= __riscv_xlen) ? ~0UL : (BIT(regB->order) - 1); + ulong regB_end = regB_start + regB_mask; if ((regB_start <= regA_start) && (regA_start < regB_end) && diff --git a/include/sbi/sbi_ecall_interface.h b/include/sbi/sbi_ecall_interface.h index 76624e3fa9f..acb91a01011 100644 --- a/include/sbi/sbi_ecall_interface.h +++ b/include/sbi/sbi_ecall_interface.h @@ -144,7 +144,7 @@ enum sbi_fwft_feature_t { SBI_FWFT_GLOBAL_PLATFORM_END = 0xffffffff, }; -#define SBI_FWFT_GLOBAL_FEATURE_BIT (1 << 31) +#define SBI_FWFT_GLOBAL_FEATURE_BIT (1UL << 31) #define SBI_FWFT_PLATFORM_FEATURE_BIT (1 << 30) #define SBI_FWFT_SET_FLAG_LOCK (1 << 0) @@ -469,6 +469,15 @@ enum sbi_sse_state { #define SBI_EXT_MPXY_SEND_MSG_WITHOUT_RESP 0x6 #define SBI_EXT_MPXY_GET_NOTIFICATION_EVENTS 0x7 +/** SBI MPXY notification header size */ +#define SBI_MPXY_NOTIF_HDR_SIZE 16 + +/** SBI MPXY notification header offsets */ +#define SBI_MPXY_NOTIF_HDR_REMAINING_OFFSET 0x00 +#define SBI_MPXY_NOTIF_HDR_RETURNED_OFFSET 0x04 +#define SBI_MPXY_NOTIF_HDR_LOST_OFFSET 0x08 +#define SBI_MPXY_NOTIF_HDR_RESERVED_OFFSET 0x0C + /* SBI base specification related macros */ #define SBI_SPEC_VERSION_MAJOR_OFFSET 24 #define SBI_SPEC_VERSION_MAJOR_MASK 0x7f diff --git a/include/sbi/sbi_fp.h b/include/sbi/sbi_fp.h new file mode 100644 index 00000000000..81998215021 --- /dev/null +++ b/include/sbi/sbi_fp.h @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 RISCstar Solutions. + * + * Authors: + * Dave Patel + */ +#ifndef __SBI_FP_H__ +#define __SBI_FP_H__ + +#include + +struct sbi_fp_context { +#if __riscv_d + uint64_t f[32]; +#else + uint32_t f[32]; +#endif + unsigned long fcsr; +}; + +#if defined(__riscv_f) || defined(__riscv_d) +void sbi_fp_save(struct sbi_fp_context *dst); +void sbi_fp_restore(const struct sbi_fp_context *src); +#else +static inline void sbi_fp_save(struct sbi_fp_context *dst) +{ +} +static inline void sbi_fp_restore(const struct sbi_fp_context *src) +{ +} +#endif /* __riscv_f || __riscv_d */ + +#endif /*__SBI_FP_H__ */ diff --git a/include/sbi/sbi_hart.h b/include/sbi/sbi_hart.h index a788b34cd52..ed1362c44fb 100644 --- a/include/sbi/sbi_hart.h +++ b/include/sbi/sbi_hart.h @@ -87,6 +87,14 @@ enum sbi_hart_extensions { SBI_HART_EXT_XSIFIVE_CFLUSH_D_L1, /** Hart has Xsfcease extension */ SBI_HART_EXT_XSIFIVE_CEASE, + /** Hart has Smrnmi extension */ + SBI_HART_EXT_SMRNMI, + /** Hart has V extension */ + SBI_HART_EXT_V, + /** Hart has F extension */ + SBI_HART_EXT_F, + /** Hart has D extension */ + SBI_HART_EXT_D, /** Maximum index of Hart extension */ SBI_HART_EXT_MAX, @@ -104,6 +112,8 @@ enum sbi_hart_csrs { SBI_HART_CSR_CYCLE = 0, SBI_HART_CSR_TIME, SBI_HART_CSR_INSTRET, + SBI_HART_CSR_MENVCFG, + SBI_HART_CSR_SENVCFG, SBI_HART_CSR_MAX, }; diff --git a/include/sbi/sbi_hart_pmp.h b/include/sbi/sbi_hart_pmp.h index a7765d17749..6a8b3e79740 100644 --- a/include/sbi/sbi_hart_pmp.h +++ b/include/sbi/sbi_hart_pmp.h @@ -9,6 +9,18 @@ #include +/* Disable pmp entry at a given index */ +int sbi_hart_pmp_disable(unsigned int n); + +/* Check if the matching field is set */ +bool sbi_hart_is_pmp_enabled(unsigned int n); + +int sbi_hart_pmp_set(unsigned int n, unsigned long prot, + unsigned long addr, unsigned long log2len); + +int sbi_hart_pmp_get(unsigned int n, unsigned long *prot_out, + unsigned long *addr_out, unsigned long *log2len); + struct sbi_scratch; unsigned int sbi_hart_pmp_count(struct sbi_scratch *scratch); diff --git a/include/sbi/sbi_illegal_insn.h b/include/sbi/sbi_illegal_insn.h index 5732e3cd678..a2b5737840a 100644 --- a/include/sbi/sbi_illegal_insn.h +++ b/include/sbi/sbi_illegal_insn.h @@ -13,6 +13,7 @@ #include struct sbi_trap_context; +struct sbi_trap_regs; typedef int (*illegal_insn_func)(ulong insn, struct sbi_trap_regs *regs); diff --git a/include/sbi/sbi_insn_emu.h b/include/sbi/sbi_insn_emu.h new file mode 100644 index 00000000000..3019d6dba10 --- /dev/null +++ b/include/sbi/sbi_insn_emu.h @@ -0,0 +1,65 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 Benedikt Freisen. + * + * Authors: + * Benedikt Freisen + */ + +#ifndef __SBI_INSN_EMU_H__ +#define __SBI_INSN_EMU_H__ + +#include + +#if defined(CONFIG_EMU_ZBB) || defined(CONFIG_EMU_ZBS) +int sbi_insn_emu_op_imm(ulong insn, struct sbi_trap_regs *regs); +#else +#define sbi_insn_emu_op_imm truly_illegal_insn +#endif + +#if defined(CONFIG_EMU_ZBA) || defined(CONFIG_EMU_ZBB) || \ + defined(CONFIG_EMU_ZBC) || defined(CONFIG_EMU_ZBS) || \ + defined(CONFIG_EMU_ZICOND) +int sbi_insn_emu_op(ulong insn, struct sbi_trap_regs *regs); +#else +#define sbi_insn_emu_op truly_illegal_insn +#endif + +#if __riscv_xlen == 64 && (defined(CONFIG_EMU_ZBA) || defined(CONFIG_EMU_ZBB)) +int sbi_insn_emu_op_32(ulong insn, struct sbi_trap_regs *regs); +#else +#define sbi_insn_emu_op_32 truly_illegal_insn +#endif + +#if __riscv_xlen == 64 && (defined(CONFIG_EMU_ZBA) || defined(CONFIG_EMU_ZBB)) +int sbi_insn_emu_op_imm_32(ulong insn, struct sbi_trap_regs *regs); +#else +#define sbi_insn_emu_op_imm_32 truly_illegal_insn +#endif + +#ifdef CONFIG_EMU_ZCB +int sbi_insn_emu_c_reserved(ulong insn, struct sbi_trap_regs *regs); +#else +#define sbi_insn_emu_c_reserved truly_illegal_insn +#endif + +#ifdef CONFIG_EMU_ZCB +int sbi_insn_emu_c_misc_alu(ulong insn, struct sbi_trap_regs *regs); +#else +#define sbi_insn_emu_c_misc_alu truly_illegal_insn +#endif + +#ifdef CONFIG_EMU_ZCMOP +int sbi_insn_emu_c_mop(ulong insn, struct sbi_trap_regs *regs); +#else +#define sbi_insn_emu_c_mop truly_illegal_insn +#endif + +#if defined(CONFIG_EMU_ZICBOM) || defined(CONFIG_EMU_ZICBOZ) +int sbi_insn_emu_zicbom_zicboz(ulong insn, struct sbi_trap_regs *regs); +#else +#define sbi_insn_emu_zicbom_zicboz truly_illegal_insn +#endif + +#endif diff --git a/include/sbi/sbi_insn_emu_fp.h b/include/sbi/sbi_insn_emu_fp.h new file mode 100644 index 00000000000..4f7014b7bc1 --- /dev/null +++ b/include/sbi/sbi_insn_emu_fp.h @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 Benedikt Freisen. + * + * Authors: + * Benedikt Freisen + */ + +#ifndef __SBI_INSN_EMU_FP_H__ +#define __SBI_INSN_EMU_FP_H__ + +#include + +#ifdef CONFIG_EMU_ZFHMIN +int sbi_insn_emu_load_fp(ulong insn, struct sbi_trap_regs *regs); +int sbi_insn_emu_store_fp(ulong insn, struct sbi_trap_regs *regs); +#else +#define sbi_insn_emu_load_fp truly_illegal_insn +#define sbi_insn_emu_store_fp truly_illegal_insn +#endif + +#if defined(CONFIG_EMU_ZFHMIN) || defined(CONFIG_EMU_ZFA) +int sbi_insn_emu_op_fp(ulong insn, struct sbi_trap_regs *regs); +#else +#define sbi_insn_emu_op_fp truly_illegal_insn +#endif + +#endif diff --git a/include/sbi/sbi_insn_emu_v.h b/include/sbi/sbi_insn_emu_v.h new file mode 100644 index 00000000000..0dfc9cce0e6 --- /dev/null +++ b/include/sbi/sbi_insn_emu_v.h @@ -0,0 +1,21 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 Benedikt Freisen. + * + * Authors: + * Benedikt Freisen + */ + +#ifndef __SBI_INSN_EMU_V_H__ +#define __SBI_INSN_EMU_V_H__ + +#include + +#if __riscv_xlen == 64 && CONFIG_EMU_ZVBB && defined(OPENSBI_CC_SUPPORT_VECTOR) +int sbi_insn_emu_op_v(ulong insn, struct sbi_trap_regs *regs); +#else +#define sbi_insn_emu_op_v truly_illegal_insn +#endif + +#endif diff --git a/include/sbi/sbi_irqchip.h b/include/sbi/sbi_irqchip.h index e0ae12f5821..e778d747531 100644 --- a/include/sbi/sbi_irqchip.h +++ b/include/sbi/sbi_irqchip.h @@ -10,21 +10,77 @@ #ifndef __SBI_IRQCHIP_H__ #define __SBI_IRQCHIP_H__ +#include #include #include struct sbi_scratch; +/** irqchip message signalled interrupt (MSI) */ +struct sbi_irqchip_msi_msg { + u32 address_lo; + u32 address_hi; + u32 data; +}; + /** irqchip hardware device */ struct sbi_irqchip_device { - /** Node in the list of irqchip devices */ + /** Node in the list of irqchip devices (private) */ struct sbi_dlist node; + /** Internal data of all hardware interrupts of this irqchip (private) */ + struct sbi_irqchip_hwirq_data *hwirqs; + + /** List of interrupt handlers (private) */ + struct sbi_dlist handler_list; + + /** Unique ID of this irqchip */ + u32 id; + + /** Capabilities of this irqchip */ +#define SBI_IRQCHIP_CAPS_WIRED BIT(0) +#define SBI_IRQCHIP_CAPS_MSI BIT(1) + unsigned long caps; + + /** Number of hardware IRQs of this irqchip */ + u32 num_hwirq; + + /** Set of harts targetted by this irqchip */ + struct sbi_hartmask target_harts; + /** Initialize per-hart state for the current hart */ - int (*warm_init)(struct sbi_irqchip_device *dev); + int (*warm_init)(struct sbi_irqchip_device *chip); - /** Handle an IRQ from this irqchip */ - int (*irq_handle)(void); + /** Process hardware interrupts from this irqchip */ + int (*process_hwirqs)(struct sbi_irqchip_device *chip); + + /** Setup a hardware interrupt of this irqchip */ + int (*hwirq_setup)(struct sbi_irqchip_device *chip, u32 hwirq, + u32 hwirq_flags); +#define SBI_HWIRQ_FLAGS_NONE 0x00000000UL +#define SBI_HWIRQ_FLAGS_EDGE_RISING 0x00000001UL +#define SBI_HWIRQ_FLAGS_EDGE_FALLING 0x00000002UL +#define SBI_HWIRQ_FLAGS_EDGE_BOTH (SBI_HWIRQ_FLAGS_EDGE_RISING | \ + SBI_HWIRQ_FLAGS_EDGE_FALLING) +#define SBI_HWIRQ_FLAGS_LEVEL_HIGH 0x00000004UL +#define SBI_HWIRQ_FLAGS_LEVEL_LOW 0x00000008UL +#define SBI_HWIRQ_FLAGS_LEVEL_SENSE_MASK 0x0000000fUL + + /** Cleanup a hardware interrupt of this irqchip */ + void (*hwirq_cleanup)(struct sbi_irqchip_device *chip, u32 hwirq); + + /** End of hardware interrupt of this irqchip */ + void (*hwirq_eoi)(struct sbi_irqchip_device *chip, u32 hwirq); + + /** Set hardware interrupt affinity */ + int (*hwirq_set_affinity)(struct sbi_irqchip_device *chip, u32 hwirq, + u32 hart_index); + + /** Mask a hardware interrupt of this irqchip */ + void (*hwirq_mask)(struct sbi_irqchip_device *chip, u32 hwirq); + + /** Unmask a hardware interrupt of this irqchip */ + void (*hwirq_unmask)(struct sbi_irqchip_device *chip, u32 hwirq); }; /** @@ -37,8 +93,68 @@ struct sbi_irqchip_device { */ int sbi_irqchip_process(void); +/** + * Process a hwirq of an irqchip device + * + * This function is called by irqchip drivers to handle hardware + * interrupts of the irqchip. + */ +int sbi_irqchip_process_hwirq(struct sbi_irqchip_device *chip, u32 hwirq); + +/** Unmask a hardware interrupt */ +int sbi_irqchip_unmask_hwirq(struct sbi_irqchip_device *chip, u32 hwirq); + +/** Mask a hardware interrupt */ +int sbi_irqchip_mask_hwirq(struct sbi_irqchip_device *chip, u32 hwirq); + +/** Default raw hardware interrupt handler */ +int sbi_irqchip_raw_handler_default(struct sbi_irqchip_device *chip, u32 hwirq); + +/** Set raw hardware interrupt handler */ +int sbi_irqchip_set_raw_handler(struct sbi_irqchip_device *chip, u32 hwirq, + int (*raw_hndl)(struct sbi_irqchip_device *, u32)); + +/** Get hardware interrupt affinity */ +int sbi_irqchip_get_affinity(struct sbi_irqchip_device *chip, u32 hwirq, + u32 *out_hart_index); + +/** Set hardware interrupt affinity */ +int sbi_irqchip_set_affinity(struct sbi_irqchip_device *chip, u32 hwirq, u32 hart_index); + +/** Write MSI message to the hardware interrupt handler */ +int sbi_irqchip_write_msi(struct sbi_irqchip_device *chip, u32 hwirq, + const struct sbi_irqchip_msi_msg *msg); + +/** Register a hardware MSI handler */ +int sbi_irqchip_register_msi(struct sbi_irqchip_device *chip, u32 num_hwirq, + void (*write_msi)(u32 hwirq, + const struct sbi_irqchip_msi_msg *msg, + void *priv), + int (*callback)(u32 hwirq, void *priv), void *priv, + u32 *out_first_hwirq); + +/** Register a hardware interrupt handler */ +int sbi_irqchip_register_handler(struct sbi_irqchip_device *chip, + u32 first_hwirq, u32 num_hwirq, u32 hwirq_flags, + int (*callback)(u32 hwirq, void *priv), void *priv); + +/** Register a hardware interrupts as reserved */ +int sbi_irqchip_register_reserved(struct sbi_irqchip_device *chip, + u32 first_hwirq, u32 num_hwirq); + +/** Unregister a hardware interrupt handler */ +int sbi_irqchip_unregister_handler(struct sbi_irqchip_device *chip, + u32 first_hwirq, u32 num_hwirq); + +/** Find an irqchip device based on matching capabilities */ +struct sbi_irqchip_device *sbi_irqchip_find_device_by_caps(unsigned long caps, + struct sbi_irqchip_device *first); + +/** Find an irqchip device based on unique ID */ +struct sbi_irqchip_device *sbi_irqchip_find_device(u32 id); + /** Register an irqchip device to receive callbacks */ -void sbi_irqchip_add_device(struct sbi_irqchip_device *dev); +int sbi_irqchip_add_device(struct sbi_irqchip_device *chip); /** Initialize interrupt controllers */ int sbi_irqchip_init(struct sbi_scratch *scratch, bool cold_boot); diff --git a/include/sbi/sbi_platform.h b/include/sbi/sbi_platform.h index e65d9877dfc..617a19ef812 100644 --- a/include/sbi/sbi_platform.h +++ b/include/sbi/sbi_platform.h @@ -56,6 +56,7 @@ struct sbi_domain_memregion; struct sbi_ecall_return; struct sbi_trap_regs; struct sbi_hart_features; +struct sbi_tlb_info; union sbi_ldst_data; /** Possible feature flags of a platform */ @@ -105,7 +106,7 @@ struct sbi_platform_operations { int (*misa_get_xlen)(void); /** Initialize (or populate) HART extensions for the platform */ - int (*extensions_init)(struct sbi_hart_features *hfeatures); + int (*extensions_init)(bool cold_boot); /** Initialize (or populate) domains for the platform */ int (*domains_init)(void); @@ -125,6 +126,20 @@ struct sbi_platform_operations { /** Get tlb fifo num entries*/ u32 (*get_tlb_num_entries)(void); + void (*local_fence_i)(struct sbi_tlb_info *tinfo); + + void (*local_sfence_vma)(struct sbi_tlb_info *tinfo); + + void (*local_sfence_vma_asid)(struct sbi_tlb_info *tinfo); + + void (*local_hfence_gvma_vmid)(struct sbi_tlb_info *tinfo); + + void (*local_hfence_gvma)(struct sbi_tlb_info *tinfo); + + void (*local_hfence_vvma_asid)(struct sbi_tlb_info *tinfo); + + void (*local_hfence_vvma)(struct sbi_tlb_info *tinfo); + /** Initialize platform timer during cold boot */ int (*timer_init)(void); @@ -136,12 +151,17 @@ struct sbi_platform_operations { struct sbi_trap_regs *regs, struct sbi_ecall_return *out); - /** platform specific handler to fixup load fault */ - int (*emulate_load)(int rlen, unsigned long addr, - union sbi_ldst_data *out_val); - /** platform specific handler to fixup store fault */ - int (*emulate_store)(int wlen, unsigned long addr, - union sbi_ldst_data in_val); + /** platform specific handler to fixup load fault + * Refer to comments below at sbi_platform_emulate_load */ + int (*emulate_load)(ulong insn, int rlen, ulong addr, + union sbi_ldst_data *out_val, + struct sbi_trap_context *tcntx); + + /** platform specific handler to fixup store fault + * Refer to comments below at sbi_platform_emulate_store */ + int (*emulate_store)(ulong insn, int wlen, ulong addr, + union sbi_ldst_data in_val, + struct sbi_trap_context *tcntx); /** platform specific pmp setup on current HART */ void (*pmp_set)(unsigned int n, unsigned long flags, @@ -149,10 +169,24 @@ struct sbi_platform_operations { unsigned long log2len); /** platform specific pmp disable on current HART */ void (*pmp_disable)(unsigned int n); + + /** platform specific Smrnmi handlers init on current HART */ + void (*smrnmi_handlers_init)(void (*rnmi_handler)(void), + void (*rnme_handler)(void)); + + /** platform specific Smrnmi NMI handler. + * Returns SBI_SUCCESS on success, error code if NMI cannot be handled. */ + int (*rnmi_handler)(struct sbi_trap_context *tcntx); }; +#ifdef CONFIG_EMU_ZVBB +/** Platform default per-HART stack size for exception/interrupt handling, + * tentatively enlarged for buffer arrays used for Zvbb emulation */ +#define SBI_PLATFORM_DEFAULT_HART_STACK_SIZE 16384 +#else /** Platform default per-HART stack size for exception/interrupt handling */ -#define SBI_PLATFORM_DEFAULT_HART_STACK_SIZE 8192 +#define SBI_PLATFORM_DEFAULT_HART_STACK_SIZE CONFIG_DEFAULT_HART_STACK_SIZE +#endif /** Platform default heap size */ #define SBI_PLATFORM_DEFAULT_HEAP_SIZE(__num_hart) \ @@ -305,6 +339,81 @@ static inline u32 sbi_platform_tlb_fifo_num_entries(const struct sbi_platform *p return sbi_hart_count(); } +static inline u32 sbi_platform_local_fence_i( + const struct sbi_platform *plat, + struct sbi_tlb_info *tinfo) +{ + if (plat && sbi_platform_ops(plat)->local_fence_i) { + sbi_platform_ops(plat)->local_fence_i(tinfo); + return 0; + } + return SBI_ENOTSUPP; +} + +static inline u32 sbi_platform_local_sfence_vma( + const struct sbi_platform *plat, + struct sbi_tlb_info *tinfo) +{ + if (plat && sbi_platform_ops(plat)->local_sfence_vma) { + sbi_platform_ops(plat)->local_sfence_vma(tinfo); + return 0; + } + return SBI_ENOTSUPP; +} + +static inline u32 sbi_platform_local_sfence_vma_asid( + const struct sbi_platform *plat, + struct sbi_tlb_info *tinfo) +{ + if (plat && sbi_platform_ops(plat)->local_sfence_vma_asid) { + sbi_platform_ops(plat)->local_sfence_vma_asid(tinfo); + return 0; + } + return SBI_ENOTSUPP; +} + +static inline u32 sbi_platform_local_hfence_gvma_vmid( + const struct sbi_platform *plat, + struct sbi_tlb_info *tinfo) +{ + if (plat && sbi_platform_ops(plat)->local_hfence_gvma_vmid) { + sbi_platform_ops(plat)->local_hfence_gvma_vmid(tinfo); + return 0; + } + return SBI_ENOTSUPP; +} + +static inline u32 sbi_platform_local_hfence_gvma( + const struct sbi_platform *plat, + struct sbi_tlb_info *tinfo) +{ + if (plat && sbi_platform_ops(plat)->local_hfence_gvma) { + sbi_platform_ops(plat)->local_hfence_gvma(tinfo); + return 0; + } + return SBI_ENOTSUPP; +} +static inline u32 sbi_platform_local_hfence_vvma_asid( + const struct sbi_platform *plat, + struct sbi_tlb_info *tinfo) +{ + if (plat && sbi_platform_ops(plat)->local_hfence_vvma_asid) { + sbi_platform_ops(plat)->local_hfence_vvma_asid(tinfo); + return 0; + } + return SBI_ENOTSUPP; +} +static inline u32 sbi_platform_local_hfence_vvma( + const struct sbi_platform *plat, + struct sbi_tlb_info *tinfo) +{ + if (plat && sbi_platform_ops(plat)->local_hfence_vvma) { + sbi_platform_ops(plat)->local_hfence_vvma(tinfo); + return 0; + } + return SBI_ENOTSUPP; +} + /** * Get total number of HARTs supported by the platform * @@ -478,10 +587,10 @@ static inline int sbi_platform_misa_xlen(const struct sbi_platform *plat) */ static inline int sbi_platform_extensions_init( const struct sbi_platform *plat, - struct sbi_hart_features *hfeatures) + bool cold_boot) { if (plat && sbi_platform_ops(plat)->extensions_init) - return sbi_platform_ops(plat)->extensions_init(hfeatures); + return sbi_platform_ops(plat)->extensions_init(cold_boot); return 0; } @@ -612,45 +721,76 @@ static inline int sbi_platform_vendor_ext_provider( } /** - * Ask platform to emulate the trapped load - * - * @param plat pointer to struct sbi_platform - * @param rlen length of the load: 1/2/4/8... - * @param addr virtual address of the load. Platform needs to page-walk and - * find the physical address if necessary - * @param out_val value loaded - * - * @return 0 on success and negative error code on failure + * Ask platform to emulate the trapped load: + * + * @param insn the instruction that caused the load fault. + * It could be a transformed instruction from tinst, thus do + * not rely on the length of insn, and use appropriate return + * code, so the caller can advance mepc properly. + * @param rlen read length in [0, 1, 2, 4, 8]. If 0, it's a special load. + * In that case, it could be a vector load or customized insn, + * which may read/gather a block of memory. The emulator should + * further parse the @insn (fetch if 0), and act accordingly. + * @param raddr read address. If @rlen is not 0, it's the base address of + * the load. It doesn't necessarily match tcntx->trap->tval, + * in case of unaligned load triggering access fault. + * If @rlen is 0, @raddr should be ignored. + * @param out_val the buffer to hold data loaded by the emulator. + * If @rlen == 0, @out_val is ignored by caller. + * @param tcntx trap context saved on load fault entry. + * + * @return >0 success: register will be updated by caller if @rlen != 0, + * and mepc will be advanced by caller. + * 0 success: no register modification; no mepc advancement. + * <0 failure + * + * It's expected that if @rlen != 0, and the emulator returns >0, the + * caller will set the corresponding registers with @out_val to simplify + * things. Otherwise, no register manipulation is done by the caller. */ static inline int sbi_platform_emulate_load(const struct sbi_platform *plat, - int rlen, unsigned long addr, - union sbi_ldst_data *out_val) + ulong insn, int rlen, ulong raddr, + union sbi_ldst_data *out_val, + struct sbi_trap_context *tcntx) { if (plat && sbi_platform_ops(plat)->emulate_load) { - return sbi_platform_ops(plat)->emulate_load(rlen, addr, - out_val); + return sbi_platform_ops(plat)->emulate_load(insn, rlen, raddr, + out_val, tcntx); } return SBI_ENOTSUPP; } /** - * Ask platform to emulate the trapped store - * - * @param plat pointer to struct sbi_platform - * @param wlen length of the store: 1/2/4/8... - * @param addr virtual address of the store. Platform needs to page-walk and - * find the physical address if necessary - * @param in_val value to store - * - * @return 0 on success and negative error code on failure + * Ask platform to emulate the trapped store: + * + * @param insn the instruction that caused the store fault. + * It could be a transformed instruction from tinst, thus do + * not rely on the length of insn, and use appropriate return + * code, so the caller can advance mepc properly. + * @param wlen write length in [0, 1, 2, 4, 8]. If 0, it's a special store. + * In that case, it could be a vector store or customized insn, + * which may write/scatter a block of memory. The emulator should + * further parse the @insn (fetch if 0), and act accordingly. + * @param waddr write address. If @wlen is not 0, it's the base address of + * the store. It doesn't necessarily match tcntx->trap->tval, + * in case of unaligned store triggering access fault. + * If @wlen is 0, @waddr should be ignored. + * @param in_val the buffer to hold data about to be stored by the emulator. + * If @wlen == 0, @in_val should be ignored. + * @param tcntx trap context saved on store fault entry. + * + * @return >0 success: mepc will be advanced by caller. + * 0 success: no mepc advancement. + * <0 failure */ static inline int sbi_platform_emulate_store(const struct sbi_platform *plat, - int wlen, unsigned long addr, - union sbi_ldst_data in_val) + ulong insn, int wlen, ulong waddr, + union sbi_ldst_data in_val, + struct sbi_trap_context *tcntx) { if (plat && sbi_platform_ops(plat)->emulate_store) { - return sbi_platform_ops(plat)->emulate_store(wlen, addr, - in_val); + return sbi_platform_ops(plat)->emulate_store(insn, wlen, waddr, + in_val, tcntx); } return SBI_ENOTSUPP; } diff --git a/include/sbi/sbi_pmp.h b/include/sbi/sbi_pmp.h new file mode 100644 index 00000000000..ce00ec0cbc5 --- /dev/null +++ b/include/sbi/sbi_pmp.h @@ -0,0 +1,23 @@ +/* + * SPDX-FileCopyrightText: (c) 2025-2026 Tenstorrent USA, Inc. + * SPDX-License-Identifier: BSD-2-Clause + */ + +#ifndef __SBI_PMP_H__ +#define __SBI_PMP_H__ + +#include + +struct pmp { + unsigned long addr; + u8 cfg; +}; +typedef struct pmp pmp_t; + +bool sbi_pmp_is_enabled(pmp_t *pmp); +int sbi_pmp_encode(pmp_t *pmp, unsigned long prot, unsigned long addr, + unsigned long log2len); +int sbi_pmp_decode(pmp_t *pmp, unsigned long *prot, unsigned long *addr, + unsigned long *log2len); + +#endif diff --git a/include/sbi/sbi_pmu.h b/include/sbi/sbi_pmu.h index c0e25f5a676..3fff46276c0 100644 --- a/include/sbi/sbi_pmu.h +++ b/include/sbi/sbi_pmu.h @@ -136,7 +136,7 @@ int sbi_pmu_add_hw_event_counter_map(u32 eidx_start, u32 eidx_end, u32 cmap); int sbi_pmu_add_raw_event_counter_map(uint64_t select, uint64_t select_mask, u32 cmap); -int sbi_pmu_ctr_fw_read(uint32_t cidx, uint64_t *cval); +int sbi_pmu_ctr_fw_read(unsigned long cidx, uint64_t *cval, bool high_bits); int sbi_pmu_ctr_stop(unsigned long cidx_base, unsigned long cidx_mask, unsigned long flag); diff --git a/include/sbi/sbi_scratch.h b/include/sbi/sbi_scratch.h index f1b4155d0ac..d0f53eec10b 100644 --- a/include/sbi/sbi_scratch.h +++ b/include/sbi/sbi_scratch.h @@ -40,12 +40,18 @@ #define SBI_SCRATCH_TRAP_CONTEXT_OFFSET (11 * __SIZEOF_POINTER__) /** Offset of tmp0 member in sbi_scratch */ #define SBI_SCRATCH_TMP0_OFFSET (12 * __SIZEOF_POINTER__) +/** Offset of tmp1 member in sbi_scratch */ +#define SBI_SCRATCH_TMP1_OFFSET (13 * __SIZEOF_POINTER__) /** Offset of options member in sbi_scratch */ -#define SBI_SCRATCH_OPTIONS_OFFSET (13 * __SIZEOF_POINTER__) +#define SBI_SCRATCH_OPTIONS_OFFSET (14 * __SIZEOF_POINTER__) /** Offset of hartindex member in sbi_scratch */ -#define SBI_SCRATCH_HARTINDEX_OFFSET (14 * __SIZEOF_POINTER__) +#define SBI_SCRATCH_HARTINDEX_OFFSET (15 * __SIZEOF_POINTER__) +/** Number of masked bits for software-based pointer masking */ +#define SBI_SCRATCH_SW_PM (16 * __SIZEOF_POINTER__) +/** Offset of emulated SENVCFG CSR */ +#define SBI_SCRATCH_SW_SENVCFG (17 * __SIZEOF_POINTER__) /** Offset of extra space in sbi_scratch */ -#define SBI_SCRATCH_EXTRA_SPACE_OFFSET (15 * __SIZEOF_POINTER__) +#define SBI_SCRATCH_EXTRA_SPACE_OFFSET (18 * __SIZEOF_POINTER__) /** Maximum size of sbi_scratch (4KB) */ #define SBI_SCRATCH_SIZE (0x1000) @@ -83,10 +89,16 @@ struct sbi_scratch { unsigned long trap_context; /** Temporary storage */ unsigned long tmp0; + /** Temporary storage */ + unsigned long tmp1; /** Options for OpenSBI library */ unsigned long options; /** Index of the hart */ unsigned long hartindex; + /** Number of masked bits for software-based pointer masking */ + unsigned long sw_pm; + /** Emulated SENVCFG CSR */ + unsigned long sw_senvcfg; }; /** @@ -106,8 +118,11 @@ assert_member_offset(struct sbi_scratch, platform_addr, SBI_SCRATCH_PLATFORM_ADD assert_member_offset(struct sbi_scratch, hartid_to_scratch, SBI_SCRATCH_HARTID_TO_SCRATCH_OFFSET); assert_member_offset(struct sbi_scratch, trap_context, SBI_SCRATCH_TRAP_CONTEXT_OFFSET); assert_member_offset(struct sbi_scratch, tmp0, SBI_SCRATCH_TMP0_OFFSET); +assert_member_offset(struct sbi_scratch, tmp1, SBI_SCRATCH_TMP1_OFFSET); assert_member_offset(struct sbi_scratch, options, SBI_SCRATCH_OPTIONS_OFFSET); assert_member_offset(struct sbi_scratch, hartindex, SBI_SCRATCH_HARTINDEX_OFFSET); +assert_member_offset(struct sbi_scratch, sw_pm, SBI_SCRATCH_SW_PM); +assert_member_offset(struct sbi_scratch, sw_senvcfg, SBI_SCRATCH_SW_SENVCFG); /** Possible options for OpenSBI library */ enum sbi_scratch_options { @@ -166,9 +181,11 @@ do { \ = (__type)(__ptr); \ } while (0) +/** Get the hart index of a particular sbi_scratch */ +#define sbi_scratch_hartindex(__scratch) ((__scratch)->hartindex) + /** Get the hart index of the current hart */ -#define current_hartindex() \ - (sbi_scratch_thishart_ptr()->hartindex) +#define current_hartindex() sbi_scratch_hartindex(sbi_scratch_thishart_ptr()) /** Number of harts managed by this OpenSBI instance */ extern u32 sbi_scratch_hart_count; @@ -186,7 +203,7 @@ extern u32 sbi_scratch_hart_count; /** HART index to HART id table */ extern u32 hartindex_to_hartid_table[]; -/** Get sbi_scratch from HART index */ +/** Get HART id from HART index */ #define sbi_hartindex_to_hartid(__hartindex) \ ({ \ ((__hartindex) < SBI_HARTMASK_MAX_BITS) ? \ diff --git a/include/sbi/sbi_timer.h b/include/sbi/sbi_timer.h index 2e1a78791e3..2a0409274dc 100644 --- a/include/sbi/sbi_timer.h +++ b/include/sbi/sbi_timer.h @@ -10,7 +10,60 @@ #ifndef __SBI_TIMER_H__ #define __SBI_TIMER_H__ -#include +#include + +/** Timer event re-start details */ +struct sbi_timer_event_restart { + /** Flag indicating whether event re-start is required */ + bool required; + /** Next time stamp for event if re-start is required */ + u64 next_event; +}; + +/** Timer event abstraction */ +struct sbi_timer_event { + /** List head for per-HART event list (Internal) */ + struct sbi_dlist head; + + /** Hart on which the event is started / running (Internal) */ + int hart_index; + + /** Time stamp when the event expires (Internal) */ + u64 time_stamp; + + /** + * Event callback to be called upon expiry. + * + * If the callback wants to re-start the event then + * it must update the event re-start details. + * + * NOTE: This will be called with the per-HART timer + * event list lock held. + */ + void (*callback)(struct sbi_timer_event *ev, + struct sbi_timer_event_restart *restart); + + /** + * Event cleanup to be called upon sbi_timer_exit() + * + * NOTE: This will be called with per-HART timer + * event list lock held. + */ + void (*cleanup)(struct sbi_timer_event *ev); + + /** Event specific private data */ + void *priv; +}; + +#define SBI_INIT_TIMER_EVENT(__ptr, __callback, __cleanup, __priv) \ +do { \ + SBI_INIT_LIST_HEAD(&(__ptr)->head); \ + (__ptr)->hart_index = -1; \ + (__ptr)->time_stamp = 0; \ + (__ptr)->callback = (__callback); \ + (__ptr)->cleanup = (__cleanup); \ + (__ptr)->priv = (__priv); \ +} while (0) /** Timer hardware device */ struct sbi_timer_device { @@ -35,6 +88,21 @@ struct sbi_timer_device { struct sbi_scratch; +/** Compute timer value delta based on arbitary units */ +u64 sbi_timer_compute_delta(ulong units, u64 unit_freq); + +/** Compute timer value delta from milliseconds */ +static inline u64 sbi_timer_compute_mdelta(ulong msecs) +{ + return sbi_timer_compute_delta(msecs, 1000); +} + +/** Compute timer value delta from microseconds */ +static inline u64 sbi_timer_compute_udelta(ulong usecs) +{ + return sbi_timer_compute_delta(usecs, 1000000); +} + /** Generic delay loop of desired granularity */ void sbi_timer_delay_loop(ulong units, u64 unit_freq, void (*delay_fn)(void *), void *opaque); @@ -72,6 +140,18 @@ bool sbi_timer_waitms_until(bool (*predicate)(void *), void *arg, /** Get timer value for current HART */ u64 sbi_timer_value(void); +/** Compute timer value after specified milliseconds */ +static inline u64 sbi_timer_value_after_msecs(ulong msecs) +{ + return sbi_timer_value() + sbi_timer_compute_mdelta(msecs); +} + +/** Compute timer value after specified microseconds */ +static inline u64 sbi_timer_value_after_usecs(ulong usecs) +{ + return sbi_timer_value() + sbi_timer_compute_udelta(usecs); +} + /** Get virtualized timer value for current HART */ u64 sbi_timer_virt_value(void); @@ -86,8 +166,14 @@ void sbi_timer_set_delta(ulong delta); void sbi_timer_set_delta_upper(ulong delta_upper); #endif -/** Start timer event for current HART */ -void sbi_timer_event_start(u64 next_event); +/** Start timer event on current HART */ +void sbi_timer_event_start(struct sbi_timer_event *ev, u64 next_event); + +/** Stop timer event on current HART */ +void sbi_timer_event_stop(struct sbi_timer_event *ev); + +/** Start supervisor timer event on current HART */ +void sbi_timer_smode_event_start(u64 next_event); /** Process timer event for current HART */ void sbi_timer_process(void); diff --git a/include/sbi/sbi_trap.h b/include/sbi/sbi_trap.h index 731a0c98b23..f1557ebc40d 100644 --- a/include/sbi/sbi_trap.h +++ b/include/sbi/sbi_trap.h @@ -218,6 +218,9 @@ _Static_assert( #define GET_RS2S(insn, regs) REG_VAL(GET_RS2S_NUM(insn), regs) #define GET_RS2C(insn, regs) REG_VAL(GET_RS2C_NUM(insn), regs) #define SET_RD(insn, regs, val) (REG_VAL(GET_RD_NUM(insn), regs) = (val)) +#define SET_RDS(insn, regs, val) (REG_VAL(GET_RDS_NUM(insn), regs) = (val)) +#define SET_RD1S(insn, regs, val) (REG_VAL(GET_RS1S_NUM(insn), regs) = (val)) +#define SET_RD2S(insn, regs, val) (REG_VAL(GET_RS2S_NUM(insn), regs) = (val)) /** Representation of trap details */ struct sbi_trap_info { @@ -273,6 +276,64 @@ static inline int sbi_mstatus_prev_mode(unsigned long mstatus) return (mstatus & MSTATUS_MPP) >> MSTATUS_MPP_SHIFT; } +#if __riscv_xlen == 32 +static inline int sbi_regs_prev_xlen(const struct sbi_trap_regs *regs) +{ + return __riscv_xlen; +} +#else +static inline int sbi_mstatus_sxl(unsigned long mstatus) +{ + return (mstatus & MSTATUS_SXL) >> MSTATUS_SXL_SHIFT; +} + +static inline int sbi_mstatus_uxl(unsigned long mstatus) +{ + return (mstatus & MSTATUS_UXL) >> MSTATUS_UXL_SHIFT; +} + +static inline int sbi_hstatus_vsxl(unsigned long hstatus) +{ + return (hstatus & HSTATUS_VSXL) >> HSTATUS_VSXL_SHIFT; +} + +static inline int sbi_regs_prev_xlen(const struct sbi_trap_regs *regs) +{ + unsigned long hstatus, vsstatus; + + if (!sbi_regs_from_virt(regs)) { + switch (sbi_mstatus_prev_mode(regs->mstatus)) { + case PRV_M: + return __riscv_xlen; + case PRV_S: + return MXL_TO_XLEN(sbi_mstatus_sxl(regs->mstatus)); + case PRV_U: + return MXL_TO_XLEN(sbi_mstatus_uxl(regs->mstatus)); + default: + __builtin_unreachable(); + } + } + + /* V=1, Check HSXLEN first */ + if (sbi_mstatus_sxl(regs->mstatus) < MXL_XLEN_64) + return 32; + + hstatus = csr_read(CSR_HSTATUS); + /* Check VSXLEN */ + if (sbi_hstatus_vsxl(hstatus) < MXL_XLEN_64) + return 32; + + vsstatus = csr_read(CSR_VSSTATUS); + switch (sbi_mstatus_prev_mode(regs->mstatus)) { + case PRV_S: + return MXL_TO_XLEN(sbi_hstatus_vsxl(hstatus)); + case PRV_U: + return MXL_TO_XLEN(sbi_mstatus_uxl(vsstatus)); + } + __builtin_unreachable(); +} +#endif + int sbi_trap_redirect(struct sbi_trap_regs *regs, const struct sbi_trap_info *trap); @@ -289,6 +350,8 @@ static inline void sbi_trap_set_context(struct sbi_scratch *scratch, struct sbi_trap_context *sbi_trap_handler(struct sbi_trap_context *tcntx); +struct sbi_trap_context *sbi_trap_rnmi_handler(struct sbi_trap_context *tcntx); + #endif #endif diff --git a/include/sbi/sbi_trap_ldst.h b/include/sbi/sbi_trap_ldst.h index a6a6c75bd77..30228d24a8b 100644 --- a/include/sbi/sbi_trap_ldst.h +++ b/include/sbi/sbi_trap_ldst.h @@ -28,13 +28,10 @@ int sbi_load_access_handler(struct sbi_trap_context *tcntx); int sbi_store_access_handler(struct sbi_trap_context *tcntx); -ulong sbi_misaligned_tinst_fixup(ulong orig_tinst, ulong new_tinst, - ulong addr_offset); - -int sbi_misaligned_v_ld_emulator(int rlen, union sbi_ldst_data *out_val, +int sbi_misaligned_v_ld_emulator(ulong insn, struct sbi_trap_context *tcntx); -int sbi_misaligned_v_st_emulator(int wlen, union sbi_ldst_data in_val, +int sbi_misaligned_v_st_emulator(ulong insn, struct sbi_trap_context *tcntx); #endif diff --git a/include/sbi/sbi_ubsan.h b/include/sbi/sbi_ubsan.h new file mode 100644 index 00000000000..f93215c4125 --- /dev/null +++ b/include/sbi/sbi_ubsan.h @@ -0,0 +1,14 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Author: Marcos Oduardo + */ + +#ifndef __SBI_UBSAN_H__ +#define __SBI_UBSAN_H__ + +#include + +extern volatile unsigned long sbi_ubsan_report_count; + +#endif /* __SBI_UBSAN_H__ */ diff --git a/include/sbi/sbi_unpriv.h b/include/sbi/sbi_unpriv.h index 8cbd3de0ccc..be219626741 100644 --- a/include/sbi/sbi_unpriv.h +++ b/include/sbi/sbi_unpriv.h @@ -36,6 +36,12 @@ DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u64) DECLARE_UNPRIVILEGED_STORE_FUNCTION(u64) DECLARE_UNPRIVILEGED_LOAD_FUNCTION(ulong) +void sbi_load_loop(u8 *buffer, ulong addr, ulong len, + struct sbi_trap_info *trap); + +void sbi_store_loop(u8 *buffer, ulong addr, ulong len, + struct sbi_trap_info *trap); + ulong sbi_get_insn(ulong mepc, struct sbi_trap_info *trap); #endif diff --git a/include/sbi/sbi_vector.h b/include/sbi/sbi_vector.h new file mode 100644 index 00000000000..c14f3174f01 --- /dev/null +++ b/include/sbi/sbi_vector.h @@ -0,0 +1,46 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 RISCstar Solutions. + * + * Authors: + * Dave Patel + */ + +#ifndef __SBI_VECTOR_H__ +#define __SBI_VECTOR_H__ + +#include + +struct sbi_vector_context { + unsigned long vcsr; + unsigned long vstart; + + /* size depends on VLEN */ + uint8_t vregs[]; +}; + +#define SET_VS_DIRTY(regs) do { \ + if (sbi_regs_from_virt(regs)) \ + csr_set(CSR_VSSTATUS, MSTATUS_VS); \ + regs->mstatus |= MSTATUS_VS; \ +} while(0) + +#ifdef OPENSBI_CC_SUPPORT_VECTOR +void sbi_vector_save(struct sbi_vector_context *dst); +void sbi_vector_restore(const struct sbi_vector_context *src); +size_t sbi_vector_context_size(void); +#else +static inline void sbi_vector_save(struct sbi_vector_context *dst) +{ +} +static inline void sbi_vector_restore(const struct sbi_vector_context *src) +{ +} +static inline size_t sbi_vector_context_size(void) +{ + return 0; +} +#endif /* OPENSBI_CC_SUPPORT_VECTOR */ + +#endif /* __SBI_VECTOR_H__ */ diff --git a/include/sbi/sbi_version.h b/include/sbi/sbi_version.h index 3b09aa60a67..60bbef8135a 100644 --- a/include/sbi/sbi_version.h +++ b/include/sbi/sbi_version.h @@ -11,7 +11,7 @@ #define __SBI_VERSION_H__ #define OPENSBI_VERSION_MAJOR 1 -#define OPENSBI_VERSION_MINOR 8 +#define OPENSBI_VERSION_MINOR 9 /** * OpenSBI 32-bit version with: diff --git a/include/sbi_utils/cache/cache.h b/include/sbi_utils/cache/cache.h index 70d9286f9a0..de6cf50d1da 100644 --- a/include/sbi_utils/cache/cache.h +++ b/include/sbi_utils/cache/cache.h @@ -19,6 +19,8 @@ struct cache_ops { int (*warm_init)(struct cache_device *dev); /** Flush entire cache **/ int (*cache_flush_all)(struct cache_device *dev); + /** Enable/Disable cache **/ + int (*cache_enable)(struct cache_device *dev, bool enable); }; struct cache_device { @@ -66,4 +68,13 @@ int cache_add(struct cache_device *dev); */ int cache_flush_all(struct cache_device *dev); +/** + * Enable/Disable the cache + * + * @param dev the cache to enable/disable + * + * @return 0 on success, or a negative error code on failure + */ +int cache_enable(struct cache_device *dev, bool enable); + #endif diff --git a/include/sbi_utils/cache/fdt_cmo_helper.h b/include/sbi_utils/cache/fdt_cmo_helper.h index a6a28db9ede..d437caad44a 100644 --- a/include/sbi_utils/cache/fdt_cmo_helper.h +++ b/include/sbi_utils/cache/fdt_cmo_helper.h @@ -22,6 +22,26 @@ int fdt_cmo_private_flc_flush_all(void); */ int fdt_cmo_llc_flush_all(void); +/** + * Enable/Disable the private first level cache of the current hart + * + * @return 0 on success, or a negative error code on failure + */ +int fdt_cmo_private_flc_enable(bool enable); + +/** + * Enable/Disable the last level cache of the current hart + * + * @return 0 on success, or a negative error code on failure + */ +int fdt_cmo_llc_enable(bool enable); +/** + * Flush the cache (all levels) of the current hart + * + * @return 0 on success, or a negative error code on failure + */ +int fdt_cmo_flush_all(void); + /** * Initialize the cache devices for each hart * diff --git a/include/sbi_utils/fdt/fdt_helper.h b/include/sbi_utils/fdt/fdt_helper.h index 04c850ccd63..75a564d1fec 100644 --- a/include/sbi_utils/fdt/fdt_helper.h +++ b/include/sbi_utils/fdt/fdt_helper.h @@ -54,8 +54,7 @@ int fdt_parse_cbom_block_size(const void *fdt, int cpu_offset, unsigned long *c int fdt_parse_timebase_frequency(const void *fdt, unsigned long *freq); -int fdt_parse_isa_extensions(const void *fdt, unsigned int hartid, - unsigned long *extensions); +int fdt_parse_isa_extensions_all_harts(const void *fdt); int fdt_parse_gaisler_uart_node(const void *fdt, int nodeoffset, struct platform_uart_data *uart); diff --git a/include/sbi_utils/hsm/fdt_hsm_andes_atcsmu.h b/include/sbi_utils/hsm/fdt_hsm_andes_atcsmu.h new file mode 100644 index 00000000000..5e71fab7c69 --- /dev/null +++ b/include/sbi_utils/hsm/fdt_hsm_andes_atcsmu.h @@ -0,0 +1,65 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 Andes Technology Corporation + */ + +#ifndef __FDT_HSM_ANDES_ATCSMU_H__ +#define __FDT_HSM_ANDES_ATCSMU_H__ + +#include + +/* clang-format off */ + +#define SCRATCH_PAD_OFFSET 0x40 + +#define RESET_VEC_LO_OFFSET 0x50 +#define RESET_VEC_HI_OFFSET 0x60 +#define RESET_VEC_8CORE_OFFSET 0x1a0 +#define HARTn_RESET_VEC_LO(n) (RESET_VEC_LO_OFFSET + \ + ((n) < 4 ? 0 : RESET_VEC_8CORE_OFFSET) + \ + ((n) * 0x4)) +#define HARTn_RESET_VEC_HI(n) (RESET_VEC_HI_OFFSET + \ + ((n) < 4 ? 0 : RESET_VEC_8CORE_OFFSET) + \ + ((n) * 0x4)) + +#define PCS0_CFG_OFFSET 0x80 +#define PCSm_CFG_OFFSET(i) ((i + 3) * 0x20 + PCS0_CFG_OFFSET) +#define PCS_CFG_LIGHT_SLEEP BIT(2) +#define PCS_CFG_DEEP_SLEEP BIT(3) + +#define PCS0_SCRATCH_OFFSET 0x84 +#define PCSm_SCRATCH_OFFSET(i) ((i + 3) * 0x20 + PCS0_SCRATCH_OFFSET) + +#define PCS0_WE_OFFSET 0x90 +#define PCSm_WE_OFFSET(i) ((i + 3) * 0x20 + PCS0_WE_OFFSET) +#define PCS_WAKEUP_RTC_ALARM_MASK BIT(2) +#define PCS_WAKEUP_UART2_MASK BIT(9) +#define PCS_WAKEUP_MSIP_MASK BIT(29) + +#define PCS0_CTL_OFFSET 0x94 +#define PCSm_CTL_OFFSET(i) ((i + 3) * 0x20 + PCS0_CTL_OFFSET) +#define LIGHT_SLEEP_CMD 0x3 +#define WAKEUP_CMD 0x8 +#define DEEP_SLEEP_CMD 0xb + +#define PCS0_STATUS_OFFSET 0x98 +#define PCSm_STATUS_OFFSET(i) ((i + 3) * 0x20 + PCS0_STATUS_OFFSET) +#define PD_TYPE_MASK GENMASK(2, 0) +#define PD_TYPE_SLEEP 2 +#define PD_STATUS_MASK GENMASK(7, 3) +#define PD_STATUS_LIGHT_SLEEP 0 +#define PD_STATUS_DEEP_SLEEP 0x10 + +/* clang-format on */ + +void atcsmu_set_wakeup_events(u32 events, u32 hartid); +bool atcsmu_support_sleep_mode(u32 sleep_type, u32 hartid); +void atcsmu_set_command(u32 pcs_ctl, u32 hartid); +int atcsmu_set_reset_vector(u64 wakeup_addr, u32 hartid); +u32 atcsmu_get_sleep_type(u32 hartid); +void atcsmu_write_scratch(u32 value); +u32 atcsmu_read_scratch(void); +bool atcsmu_pcs_is_sleep(u32 hartid, bool deep_sleep); + +#endif diff --git a/include/sbi_utils/hsm/fdt_hsm_sifive_inst.h b/include/sbi_utils/hsm/fdt_hsm_sifive_inst.h index 7e9180ea6ba..8de853ed84c 100644 --- a/include/sbi_utils/hsm/fdt_hsm_sifive_inst.h +++ b/include/sbi_utils/hsm/fdt_hsm_sifive_inst.h @@ -7,9 +7,10 @@ #ifndef __FDT_HSM_SIFIVE_INST_H__ #define __FDT_HSM_SIFIVE_INST_H__ -static inline void sifive_cease(void) +static inline void __noreturn sifive_cease(void) { __asm__ __volatile__(".word 0x30500073" ::: "memory"); + __builtin_unreachable(); } static inline void sifive_cflush(void) diff --git a/include/sbi_utils/irqchip/aplic.h b/include/sbi_utils/irqchip/aplic.h index cbfcd3fd17b..3461d1c720b 100644 --- a/include/sbi_utils/irqchip/aplic.h +++ b/include/sbi_utils/irqchip/aplic.h @@ -35,6 +35,7 @@ struct aplic_data { struct sbi_irqchip_device irqchip; struct sbi_dlist node; /* Public members */ + u32 unique_id; unsigned long addr; unsigned long size; unsigned long num_idc; @@ -45,6 +46,7 @@ struct aplic_data { bool has_msicfg_smode; struct aplic_msicfg_data msicfg_smode; struct aplic_delegate_data delegate[APLIC_MAX_DELEGATE]; + u32 *idc_map; }; int aplic_cold_irqchip_init(struct aplic_data *aplic); diff --git a/include/sbi_utils/irqchip/imsic.h b/include/sbi_utils/irqchip/imsic.h index 353cefec663..88dd3cfbe0c 100644 --- a/include/sbi_utils/irqchip/imsic.h +++ b/include/sbi_utils/irqchip/imsic.h @@ -24,6 +24,7 @@ struct imsic_regs { }; struct imsic_data { + u32 unique_id; bool targets_mmode; u32 guest_index_bits; u32 hart_index_bits; diff --git a/include/sbi_utils/irqchip/plic.h b/include/sbi_utils/irqchip/plic.h index 18aa7469d97..0dbf4ed3218 100644 --- a/include/sbi_utils/irqchip/plic.h +++ b/include/sbi_utils/irqchip/plic.h @@ -17,6 +17,7 @@ struct plic_data { /* Private members */ struct sbi_irqchip_device irqchip; /* Public members */ + u32 unique_id; unsigned long addr; unsigned long size; unsigned long num_src; diff --git a/include/sbi_utils/mailbox/rpmi_msgprot.h b/include/sbi_utils/mailbox/rpmi_msgprot.h index f8b16753666..7ae4abe0e2a 100644 --- a/include/sbi_utils/mailbox/rpmi_msgprot.h +++ b/include/sbi_utils/mailbox/rpmi_msgprot.h @@ -87,6 +87,10 @@ #define RPMI_DEF_TX_TIMEOUT 20 #define RPMI_DEF_RX_TIMEOUT 20 +/** RPMI Notification event header constructor */ +#define RPMI_NOTIF_EVENT_HDR(eid, dlen) ((u32)(eid) << 16) | \ + ((u32)(dlen) & 0xFFFF) + /** * Common macro to generate composite version from major * and minor version numbers. @@ -220,6 +224,8 @@ enum rpmi_servicegroup_id { RPMI_SRVGRP_CLOCK = 0x0008, RPMI_SRVGRP_DEVICE_POWER = 0x0009, RPMI_SRVGRP_PERFORMANCE = 0x0000A, + RPMI_SRVGRP_MANAGEMENT_MODE = 0x000B, + RPMI_SRVGRP_LOGGING = 0x000E, RPMI_SRVGRP_ID_MAX_COUNT, /* Reserved range for service groups */ @@ -231,14 +237,24 @@ enum rpmi_servicegroup_id { RPMI_SRVGRP_VENDOR_END = 0xFFFF, }; +/** RPMI event notification state IDs */ +enum rpmi_event_notification_state { + RPMI_EVENT_NOTIF_DISABLE_STATE = 0, + RPMI_EVENT_NOTIF_ENABLE_STATE = 1, + RPMI_EVENT_NOTIF_RET_CURR_STATE = 2, + RPMI_EVENT_NOTIF_MAX_COUNT = 3, +}; + /** RPMI enable notification request */ struct rpmi_enable_notification_req { u32 eventid; + u32 req_state; }; /** RPMI enable notification response */ struct rpmi_enable_notification_resp { s32 status; + u32 current_state; }; /** RPMI Base ServiceGroup Service IDs */ @@ -845,6 +861,33 @@ enum rpmi_performance_service_id { RPMI_PERF_SRV_MAX_COUNT, }; +/** RPMI Performance ServiceGroup Notification Event IDs */ +enum rpmi_performance_event_id { + RPMI_PERF_EVENT_POWER_CHANGE = 0x01, + RPMI_PERF_EVENT_LIMIT_CHANGE = 0x02, + RPMI_PERF_EVENT_LEVEL_CHANGE = 0x03, + RPMI_PERF_EVENT_MAX_COUNT, +}; + +/** RPMI Performance power change notification data */ +struct rpmi_perf_event_power_change { + u32 domain_id; + u32 power_uw; +}; + +/** RPMI Performance limit change notification data */ +struct rpmi_perf_event_limit_change { + u32 domain_id; + u32 max_level; + u32 min_level; +}; + +/** RPMI Performance level change notification data */ +struct rpmi_perf_event_level_change { + u32 domain_id; + u32 level; +}; + struct rpmi_perf_get_num_domain_resp { s32 status; u32 num_domains; @@ -856,6 +899,16 @@ struct rpmi_perf_get_attrs_req { struct rpmi_perf_get_attrs_resp { s32 status; +#define RPMI_PERF_DOMAIN_ATTRS_FLAGS_MASK (7U << 0) +#define RPMI_PERF_DOMAIN_ATTRS_FLAGS_FASTCHANNEL_SUPP_POS 0 +#define RPMI_PERF_DOMAIN_ATTRS_FLAGS_PERF_LVL_CHG_SUPP_POS 1 +#define RPMI_PERF_DOMAIN_ATTRS_FLAGS_PERF_LIMIT_CHG_SUPP_POS 2 +#define RPMI_PERF_DOMAIN_ATTRS_FLAGS_FASTCHANNEL_SUPP \ + (1U << RPMI_PERF_DOMAIN_ATTRS_FLAGS_FASTCHANNEL_SUPP_POS) +#define RPMI_PERF_DOMAIN_ATTRS_FLAGS_PERF_LVL_CHG_SUPP \ + (1U << RPMI_PERF_DOMAIN_ATTRS_FLAGS_PERF_LVL_CHG_SUPP_POS) +#define RPMI_PERF_DOMAIN_ATTRS_FLAGS_PERF_LIMIT_CHG_SUPP \ + (1U << RPMI_PERF_DOMAIN_ATTRS_FLAGS_PERF_LIMIT_CHG_SUPP_POS) u32 flags; u32 num_level; u32 latency; @@ -935,16 +988,80 @@ struct rpmi_perf_get_fast_chn_attr_req { struct rpmi_perf_get_fast_chn_attr_resp { s32 status; +#define RPMI_PERF_FASTCHAN_ATTRS_FLAGS_MASK (7U << 0) +#define RPMI_PERF_FASTCHAN_ATTRS_FLAGS_DB_SUPP_POS 0 +#define RPMI_PERF_FASTCHAN_ATTRS_FLAGS_DB_WIDTH_POS 1 +#define RPMI_PERF_FASTCHAN_ATTRS_FLAGS_DB_SUPP \ + (1U << RPMI_PERF_FASTCHAN_ATTRS_FLAGS_DB_SUPP_POS) +#define RPMI_PERF_FASTCHAN_ATTRS_FLAGS_DB_WIDTH_MASK \ + (3U << RPMI_PERF_FASTCHAN_ATTRS_FLAGS_DB_WIDTH_POS) +#define RPMI_PERF_FASTCHAN_ATTRS_FLAGS_DB_WIDTH_8 \ + (0U << RPMI_PERF_FASTCHAN_ATTRS_FLAGS_DB_WIDTH_POS) +#define RPMI_PERF_FASTCHAN_ATTRS_FLAGS_DB_WIDTH_16 \ + (1U << RPMI_PERF_FASTCHAN_ATTRS_FLAGS_DB_WIDTH_POS) +#define RPMI_PERF_FASTCHAN_ATTRS_FLAGS_DB_WIDTH_32 \ + (2U << RPMI_PERF_FASTCHAN_ATTRS_FLAGS_DB_WIDTH_POS) u32 flags; u32 region_offset_low; u32 region_offset_high; u32 region_size; u32 db_addr_low; u32 db_addr_high; - u32 db_id_low; - u32 db_id_high; - u32 db_perserved_low; - u32 db_perserved_high; + u32 db_write_value; +}; + +/** RPMI MM ServiceGroup Service IDs */ +enum rpmi_mm_service_id { + RPMI_MM_SRV_ENABLE_NOTIFICATION = 0x01, + RPMI_MM_SRV_GET_ATTRIBUTES = 0x02, + RPMI_MM_SRV_COMMUNICATE = 0x03, + RPMI_MM_SRV_MAX_COUNT, +}; + +/** RPMI MM ServiceGroup Get Attributes main struct */ +struct rpmi_mm_attributes { + u32 mm_version; + u32 shmem_addr_lo; + u32 shmem_addr_hi; + u32 shmem_size; +}; + +/** RPMI MM ServiceGroup Get Attributes response struct */ +struct rpmi_mm_get_attributes_rsp { + s32 status; + struct rpmi_mm_attributes mma; +}; + +/** RPMI MM ServiceGroup Communicate request struct */ +struct rpmi_mm_communicate_req { + u32 mm_comm_ipdata_off; + u32 mm_comm_ipdata_size; + u32 mm_comm_opdata_off; + u32 mm_comm_opdata_size; +}; + +/** RPMI MM ServiceGroup Communicate response struct */ +struct rpmi_mm_communicate_rsp { + s32 status; + u32 mm_comm_retdata_size; +}; + +/** RPMI LOGGING ServiceGroup Service IDs */ +enum rpmi_logging_service_id { + RPMI_LOGGING_SRV_ENABLE_NOTIFICATION = 0x01, + RPMI_LOGGING_SRV_LOG_DATA = 0x02, + RPMI_LOGGING_SRV_MAX_COUNT, +}; + +struct rpmi_logging_log_data_req { + u32 type; + u32 num_dwords; +#define MAX_LOGGING_DLEN ((RPMI_MSG_DATA_SIZE(RPMI_SLOT_SIZE_MIN) - (sizeof(u32) * 2)) / sizeof(u32)) + u32 data[MAX_LOGGING_DLEN]; +}; + +struct rpmi_logging_log_data_resp { + s32 status; }; #endif /* !__RPMI_MSGPROT_H__ */ diff --git a/include/sbi_utils/serial/altr-juart.h b/include/sbi_utils/serial/altr-juart.h new file mode 100644 index 00000000000..1304ee0809b --- /dev/null +++ b/include/sbi_utils/serial/altr-juart.h @@ -0,0 +1,17 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 ISCAS + * + * Authors: + * Icenowy Zheng + */ + +#ifndef __SERIAL_ALTR_JUART_H__ +#define __SERIAL_ALTR_JUART_H__ + +#include + +int altr_juart_init(unsigned long base); + +#endif diff --git a/include/sbi_utils/sys/atcsmu.h b/include/sbi_utils/sys/atcsmu.h deleted file mode 100644 index 07be22eef62..00000000000 --- a/include/sbi_utils/sys/atcsmu.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * SPDX-License-Identifier: BSD-3-Clause - * - * Copyright (c) 2023 Andes Technology Corporation - */ - -#ifndef _SYS_ATCSMU_H -#define _SYS_ATCSMU_H - -#include - -/* clang-format off */ - -#define PCS0_WE_OFFSET 0x90 -#define PCSm_WE_OFFSET(i) ((i + 3) * 0x20 + PCS0_WE_OFFSET) - -#define PCS0_CTL_OFFSET 0x94 -#define PCSm_CTL_OFFSET(i) ((i + 3) * 0x20 + PCS0_CTL_OFFSET) -#define PCS_CTL_CMD_SHIFT 0 -#define PCS_CTL_PARAM_SHIFT 3 -#define SLEEP_CMD 0x3 -#define WAKEUP_CMD (0x0 | (1 << PCS_CTL_PARAM_SHIFT)) -#define LIGHTSLEEP_MODE 0 -#define DEEPSLEEP_MODE 1 -#define LIGHT_SLEEP_CMD (SLEEP_CMD | (LIGHTSLEEP_MODE << PCS_CTL_PARAM_SHIFT)) -#define DEEP_SLEEP_CMD (SLEEP_CMD | (DEEPSLEEP_MODE << PCS_CTL_PARAM_SHIFT)) - -#define PCS0_CFG_OFFSET 0x80 -#define PCSm_CFG_OFFSET(i) ((i + 3) * 0x20 + PCS0_CFG_OFFSET) -#define PCS_CFG_LIGHT_SLEEP_SHIFT 2 -#define PCS_CFG_LIGHT_SLEEP (1 << PCS_CFG_LIGHT_SLEEP_SHIFT) -#define PCS_CFG_DEEP_SLEEP_SHIFT 3 -#define PCS_CFG_DEEP_SLEEP (1 << PCS_CFG_DEEP_SLEEP_SHIFT) - -#define RESET_VEC_LO_OFFSET 0x50 -#define RESET_VEC_HI_OFFSET 0x60 -#define RESET_VEC_8CORE_OFFSET 0x1a0 -#define HARTn_RESET_VEC_LO(n) (RESET_VEC_LO_OFFSET + \ - ((n) < 4 ? 0 : RESET_VEC_8CORE_OFFSET) + \ - ((n) * 0x4)) -#define HARTn_RESET_VEC_HI(n) (RESET_VEC_HI_OFFSET + \ - ((n) < 4 ? 0 : RESET_VEC_8CORE_OFFSET) + \ - ((n) * 0x4)) - -#define PCS_MAX_NR 8 -#define FLASH_BASE 0x80000000ULL - -/* clang-format on */ - -struct smu_data { - unsigned long addr; -}; - -int smu_set_wakeup_events(struct smu_data *smu, u32 events, u32 hartid); -bool smu_support_sleep_mode(struct smu_data *smu, u32 sleep_mode, u32 hartid); -int smu_set_command(struct smu_data *smu, u32 pcs_ctl, u32 hartid); -int smu_set_reset_vector(struct smu_data *smu, ulong wakeup_addr, u32 hartid); - -#endif /* _SYS_ATCSMU_H */ diff --git a/lib/sbi/Kconfig b/lib/sbi/Kconfig index c6cc04bc041..6f9450f4d88 100644 --- a/lib/sbi/Kconfig +++ b/lib/sbi/Kconfig @@ -2,10 +2,27 @@ menu "Generic SBI Support" +config DEFAULT_HART_STACK_SIZE + int "Default per-HART stack size (bytes)" + range 8192 1048576 + default 8192 + config CONSOLE_EARLY_BUFFER_SIZE int "Early console buffer size (bytes)" default 256 +config ZKR_POLL_BUDGET + int "Zkr seed polling budget (iterations)" + default 1000 + help + Maximum number of iterations to poll CSR_SEED when initializing + the stack guard variable. The Zkr specification doesn't define + a time limit on transitioning to ES16 between polls, which + makes it impossible to tell whether entropy is being + accumulated slowly or the entropy source is not functioning. + This also limits the wait time on systems with an event-driven + entropy source. A successful read doesn't consume a try. + config SBI_ECALL_TIME bool "Timer extension" default y @@ -69,4 +86,11 @@ config SBI_ECALL_SSE config SBI_ECALL_MPXY bool "MPXY extension" default y + +config SBI_ISA_EXT_EMU + bool "Enable ISA extension emulation" + default n + +source "$(OPENSBI_SRC_DIR)/lib/sbi/sbi_insn_emu.conf" + endmenu diff --git a/lib/sbi/objects.mk b/lib/sbi/objects.mk index 07d1322937b..95e57fd9b08 100644 --- a/lib/sbi/objects.mk +++ b/lib/sbi/objects.mk @@ -83,10 +83,14 @@ libsbi-objs-y += sbi_hfence.o libsbi-objs-y += sbi_hsm.o libsbi-objs-y += sbi_illegal_atomic.o libsbi-objs-y += sbi_illegal_insn.o +libsbi-objs-y += sbi_insn_emu.o +libsbi-objs-y += sbi_insn_emu_fp.o +libsbi-objs-y += sbi_insn_emu_v.o libsbi-objs-y += sbi_init.o libsbi-objs-y += sbi_ipi.o libsbi-objs-y += sbi_irqchip.o libsbi-objs-y += sbi_platform.o +libsbi-objs-y += sbi_pmp.o libsbi-objs-y += sbi_pmu.o libsbi-objs-y += sbi_dbtr.o libsbi-objs-y += sbi_mpxy.o @@ -99,6 +103,11 @@ libsbi-objs-y += sbi_tlb.o libsbi-objs-y += sbi_trap.o libsbi-objs-y += sbi_trap_ldst.o libsbi-objs-y += sbi_trap_v_ldst.o +ifeq ($(UBSAN), y) +libsbi-objs-y += sbi_ubsan.o +endif libsbi-objs-y += sbi_unpriv.o libsbi-objs-y += sbi_expected_trap.o libsbi-objs-y += sbi_cppc.o +libsbi-objs-$(CC_SUPPORT_VECTOR) += sbi_vector.o +libsbi-objs-y += sbi_fp.o diff --git a/lib/sbi/riscv_asm.c b/lib/sbi/riscv_asm.c index 3e44320f376..a1e0f2f8721 100644 --- a/lib/sbi/riscv_asm.c +++ b/lib/sbi/riscv_asm.c @@ -12,6 +12,7 @@ #include #include #include +#include /* determine CPU extension, return non-zero support */ int misa_extension_imp(char ext) @@ -271,164 +272,3 @@ void csr_write_num(int csr_num, unsigned long val) #undef switchcase_csr_write_2 #undef switchcase_csr_write } - -static unsigned long ctz(unsigned long x) -{ - unsigned long ret = 0; - - if (x == 0) - return 8 * sizeof(x); - - while (!(x & 1UL)) { - ret++; - x = x >> 1; - } - - return ret; -} - -int pmp_disable(unsigned int n) -{ - int pmpcfg_csr, pmpcfg_shift; - unsigned long cfgmask, pmpcfg; - - if (n >= PMP_COUNT) - return SBI_EINVAL; - -#if __riscv_xlen == 32 - pmpcfg_csr = CSR_PMPCFG0 + (n >> 2); - pmpcfg_shift = (n & 3) << 3; -#elif __riscv_xlen == 64 - pmpcfg_csr = (CSR_PMPCFG0 + (n >> 2)) & ~1; - pmpcfg_shift = (n & 7) << 3; -#else -# error "Unexpected __riscv_xlen" -#endif - - /* Clear the address matching bits to disable the pmp entry */ - cfgmask = ~(0xffUL << pmpcfg_shift); - pmpcfg = (csr_read_num(pmpcfg_csr) & cfgmask); - - csr_write_num(pmpcfg_csr, pmpcfg); - - return SBI_OK; -} - -int is_pmp_entry_mapped(unsigned long entry) -{ - unsigned long prot; - unsigned long addr; - unsigned long log2len; - - if (pmp_get(entry, &prot, &addr, &log2len) != 0) - return false; - - /* If address matching bits are non-zero, the entry is enable */ - if (prot & PMP_A) - return true; - - return false; -} - -int pmp_set(unsigned int n, unsigned long prot, unsigned long addr, - unsigned long log2len) -{ - int pmpcfg_csr, pmpcfg_shift, pmpaddr_csr; - unsigned long cfgmask, pmpcfg; - unsigned long addrmask, pmpaddr; - - /* check parameters */ - if (n >= PMP_COUNT || log2len > __riscv_xlen || log2len < PMP_SHIFT) - return SBI_EINVAL; - - /* calculate PMP register and offset */ -#if __riscv_xlen == 32 - pmpcfg_csr = CSR_PMPCFG0 + (n >> 2); - pmpcfg_shift = (n & 3) << 3; -#elif __riscv_xlen == 64 - pmpcfg_csr = (CSR_PMPCFG0 + (n >> 2)) & ~1; - pmpcfg_shift = (n & 7) << 3; -#else -# error "Unexpected __riscv_xlen" -#endif - pmpaddr_csr = CSR_PMPADDR0 + n; - - /* encode PMP config */ - prot &= ~PMP_A; - prot |= (log2len == PMP_SHIFT) ? PMP_A_NA4 : PMP_A_NAPOT; - cfgmask = ~(0xffUL << pmpcfg_shift); - pmpcfg = (csr_read_num(pmpcfg_csr) & cfgmask); - pmpcfg |= ((prot << pmpcfg_shift) & ~cfgmask); - - /* encode PMP address */ - if (log2len == PMP_SHIFT) { - pmpaddr = (addr >> PMP_SHIFT); - } else { - if (log2len == __riscv_xlen) { - pmpaddr = -1UL; - } else { - addrmask = (1UL << (log2len - PMP_SHIFT)) - 1; - pmpaddr = ((addr >> PMP_SHIFT) & ~addrmask); - pmpaddr |= (addrmask >> 1); - } - } - - /* write csrs */ - csr_write_num(pmpaddr_csr, pmpaddr); - csr_write_num(pmpcfg_csr, pmpcfg); - - return 0; -} - -int pmp_get(unsigned int n, unsigned long *prot_out, unsigned long *addr_out, - unsigned long *log2len) -{ - int pmpcfg_csr, pmpcfg_shift, pmpaddr_csr; - unsigned long cfgmask, pmpcfg, prot; - unsigned long t1, addr, len; - - /* check parameters */ - if (n >= PMP_COUNT || !prot_out || !addr_out || !log2len) - return SBI_EINVAL; - *prot_out = *addr_out = *log2len = 0; - - /* calculate PMP register and offset */ -#if __riscv_xlen == 32 - pmpcfg_csr = CSR_PMPCFG0 + (n >> 2); - pmpcfg_shift = (n & 3) << 3; -#elif __riscv_xlen == 64 - pmpcfg_csr = (CSR_PMPCFG0 + (n >> 2)) & ~1; - pmpcfg_shift = (n & 7) << 3; -#else -# error "Unexpected __riscv_xlen" -#endif - pmpaddr_csr = CSR_PMPADDR0 + n; - - /* decode PMP config */ - cfgmask = (0xffUL << pmpcfg_shift); - pmpcfg = csr_read_num(pmpcfg_csr) & cfgmask; - prot = pmpcfg >> pmpcfg_shift; - - /* decode PMP address */ - if ((prot & PMP_A) == PMP_A_NAPOT) { - addr = csr_read_num(pmpaddr_csr); - if (addr == -1UL) { - addr = 0; - len = __riscv_xlen; - } else { - t1 = ctz(~addr); - addr = (addr & ~((1UL << t1) - 1)) << PMP_SHIFT; - len = (t1 + PMP_SHIFT + 1); - } - } else { - addr = csr_read_num(pmpaddr_csr) << PMP_SHIFT; - len = PMP_SHIFT; - } - - /* return details */ - *prot_out = prot; - *addr_out = addr; - *log2len = len; - - return 0; -} diff --git a/lib/sbi/riscv_atomic.c b/lib/sbi/riscv_atomic.c index fcf07f27314..d9abe343cb2 100644 --- a/lib/sbi/riscv_atomic.c +++ b/lib/sbi/riscv_atomic.c @@ -48,7 +48,7 @@ long atomic_add_return(atomic_t *atom, long value) long ret, temp; #if __SIZEOF_LONG__ == 4 __asm__ __volatile__("1:lr.w.aqrl %1,%0\n" - " addw %2,%1,%3\n" + " add %2,%1,%3\n" " sc.w.aqrl %2,%2,%0\n" " bnez %2,1b" : "+A"(atom->counter), "=&r"(ret), "=&r"(temp) diff --git a/lib/sbi/riscv_locks.c b/lib/sbi/riscv_locks.c index c29a965907b..f408ef0ce89 100644 --- a/lib/sbi/riscv_locks.c +++ b/lib/sbi/riscv_locks.c @@ -57,12 +57,17 @@ void spin_lock(spinlock_t *lock) " amoadd.w.aqrl %0, %4, %3\n" #elif defined(__riscv_zalrsc) "3: lr.w.aqrl %0, %3\n" +#if __riscv_xlen == 64 " addw %1, %0, %4\n" +#elif __riscv_xlen == 32 + " add %1, %0, %4\n" +#endif " sc.w.aqrl %1, %1, %3\n" " bnez %1, 3b\n" #else #error "need A or Zaamo or Zalrsc" #endif + "fence w, o\n" /* Did we get the lock? */ " srli %1, %0, %6\n" @@ -83,4 +88,5 @@ void spin_lock(spinlock_t *lock) void spin_unlock(spinlock_t *lock) { __smp_store_release(&lock->owner, lock->owner + 1); + RISCV_FENCE(w, o); } diff --git a/lib/sbi/sbi_dbtr.c b/lib/sbi/sbi_dbtr.c index 8bcb4312d61..01047969dbd 100644 --- a/lib/sbi/sbi_dbtr.c +++ b/lib/sbi/sbi_dbtr.c @@ -34,6 +34,25 @@ static unsigned long hart_state_ptr_offset; sbi_scratch_write_type((__scratch), void *, hart_state_ptr_offset, \ (__hart_state)) +#define tdata_read_safe(__csr) \ + ({ \ + struct sbi_trap_info __trap = {0}; \ + csr_read_allowed((__csr), &__trap); \ + }) + +#define tdata_write_safe(__csr, __value) \ + ({ \ + struct sbi_trap_info __trap = {0}; \ + csr_write_allowed((__csr), &__trap, (__value)); \ + }) + +#define tdata_implemented(__csr) \ + ({ \ + struct sbi_trap_info __trap = {0}; \ + csr_read_allowed((__csr), &__trap); \ + !__trap.cause; \ + }) + #define INDEX_TO_TRIGGER(_index) \ ({ \ struct sbi_dbtr_trigger *__trg = NULL; \ @@ -418,7 +437,8 @@ static void dbtr_trigger_enable(struct sbi_dbtr_trigger *trig) */ csr_write(CSR_TSELECT, trig->index); csr_write(CSR_TDATA1, 0x0); - csr_write(CSR_TDATA2, trig->tdata2); + tdata_write_safe(CSR_TDATA2, trig->tdata2); + tdata_write_safe(CSR_TDATA3, trig->tdata3); csr_write(CSR_TDATA1, trig->tdata1); } @@ -463,7 +483,8 @@ static void dbtr_trigger_clear(struct sbi_dbtr_trigger *trig) csr_write(CSR_TSELECT, trig->index); csr_write(CSR_TDATA1, 0x0); - csr_write(CSR_TDATA2, 0x0); + tdata_write_safe(CSR_TDATA2, 0x0); + tdata_write_safe(CSR_TDATA3, 0x0); } static int dbtr_trigger_supported(unsigned long type) @@ -566,8 +587,8 @@ int sbi_dbtr_read_trig(unsigned long smode, trig = INDEX_TO_TRIGGER((_idx + trig_idx_base)); csr_write(CSR_TSELECT, trig->index); trig->tdata1 = csr_read(CSR_TDATA1); - trig->tdata2 = csr_read(CSR_TDATA2); - trig->tdata3 = csr_read(CSR_TDATA3); + trig->tdata2 = tdata_read_safe(CSR_TDATA2); + trig->tdata3 = tdata_read_safe(CSR_TDATA3); xmit->tstate = cpu_to_lle(trig->state); xmit->tdata1 = cpu_to_lle(trig->tdata1); xmit->tdata2 = cpu_to_lle(trig->tdata2); @@ -589,6 +610,7 @@ int sbi_dbtr_install_trig(unsigned long smode, unsigned long ctrl; struct sbi_dbtr_trigger *trig; struct sbi_dbtr_hart_triggers_state *hs = NULL; + bool tdata2_impl, tdata3_impl; hs = dbtr_thishart_state_ptr(); if (!hs) @@ -601,6 +623,15 @@ int sbi_dbtr_install_trig(unsigned long smode, sbi_hart_protection_map_range((unsigned long)shmem_base, trig_count * sizeof(*entry)); + /* + * SBI v3.0 sec 19.4 requires SBI_ERR_NOT_SUPPORTED when a trigger + * programs a non-zero value into an unimplemented optional CSR. Only + * the "whole CSR unimplemented" case is caught; WARL bits tied off + * inside an otherwise-implemented CSR are not. + */ + tdata2_impl = tdata_implemented(CSR_TDATA2); + tdata3_impl = tdata_implemented(CSR_TDATA3); + /* Check requested triggers configuration */ for_each_trig_entry(shmem_base, trig_count, typeof(*entry), entry) { recv = (struct sbi_dbtr_data_msg *)(&entry->data); @@ -619,6 +650,14 @@ int sbi_dbtr_install_trig(unsigned long smode, trig_count * sizeof(*entry)); return SBI_ERR_FAILED; } + + if ((recv->tdata2 && !tdata2_impl) || + (recv->tdata3 && !tdata3_impl)) { + *out = _idx; + sbi_hart_protection_unmap_range((unsigned long)shmem_base, + trig_count * sizeof(*entry)); + return SBI_ERR_NOT_SUPPORTED; + } } if (hs->available_trigs < trig_count) { @@ -705,6 +744,7 @@ int sbi_dbtr_update_trig(unsigned long smode, union sbi_dbtr_shmem_entry *entry; void *shmem_base = NULL; struct sbi_dbtr_hart_triggers_state *hs = NULL; + bool tdata2_impl, tdata3_impl; hs = dbtr_thishart_state_ptr(); if (!hs) @@ -718,6 +758,15 @@ int sbi_dbtr_update_trig(unsigned long smode, if (trig_count >= hs->total_trigs) return SBI_ERR_BAD_RANGE; + /* + * SBI v3.0 sec 19.5 requires SBI_ERR_NOT_SUPPORTED when a trigger + * programs a non-zero value into an unimplemented optional CSR. Only + * the "whole CSR unimplemented" case is caught; WARL bits tied off + * inside an otherwise-implemented CSR are not. + */ + tdata2_impl = tdata_implemented(CSR_TDATA2); + tdata3_impl = tdata_implemented(CSR_TDATA3); + for_each_trig_entry(shmem_base, trig_count, typeof(*entry), entry) { sbi_hart_protection_map_range((unsigned long)entry, sizeof(*entry)); trig_idx = entry->id.idx; @@ -734,6 +783,12 @@ int sbi_dbtr_update_trig(unsigned long smode, return SBI_ERR_FAILED; } + if ((entry->data.tdata2 && !tdata2_impl) || + (entry->data.tdata3 && !tdata3_impl)) { + sbi_hart_protection_unmap_range((unsigned long)entry, sizeof(*entry)); + return SBI_ERR_NOT_SUPPORTED; + } + dbtr_trigger_setup(trig, &entry->data); sbi_hart_protection_unmap_range((unsigned long)entry, sizeof(*entry)); dbtr_trigger_enable(trig); diff --git a/lib/sbi/sbi_domain.c b/lib/sbi/sbi_domain.c index 4f458ac0f56..fa69170bdc9 100644 --- a/lib/sbi/sbi_domain.c +++ b/lib/sbi/sbi_domain.c @@ -505,6 +505,9 @@ bool sbi_domain_check_addr_range(const struct sbi_domain *dom, if (!dom) return false; + if (size && max <= addr) + return false; + while (addr < max) { reg = find_region(dom, addr); if (!reg) @@ -541,7 +544,7 @@ void sbi_domain_dump(const struct sbi_domain *dom, const char *suffix) sbi_printf("Domain%d HARTs %s: ", dom->index, suffix); sbi_hartmask_for_each_hartindex(i, dom->possible_harts) { j = sbi_hartindex_to_hartid(i); - sbi_printf("%s%d%s", (k++) ? "," : "", + sbi_printf("%s0x%x%s", (k++) ? "," : "", j, sbi_domain_is_assigned_hart(dom, i) ? "*" : ""); } sbi_printf("\n"); @@ -871,20 +874,23 @@ int sbi_domain_init(struct sbi_scratch *scratch, u32 cold_hartid) struct sbi_hartmask *root_hmask; struct sbi_domain_memregion *root_memregs; int root_memregs_count = 0; + const struct sbi_platform *plat = sbi_platform_ptr(scratch); + bool fw_single_region = sbi_platform_single_fw_region(plat); SBI_INIT_LIST_HEAD(&domain_list); - if (scratch->fw_rw_offset == 0 || - (scratch->fw_rw_offset & (scratch->fw_rw_offset - 1)) != 0) { - sbi_printf("%s: fw_rw_offset is not a power of 2 (0x%lx)\n", - __func__, scratch->fw_rw_offset); - return SBI_EINVAL; - } - - if ((scratch->fw_start & (scratch->fw_rw_offset - 1)) != 0) { - sbi_printf("%s: fw_start and fw_rw_offset not aligned\n", - __func__); - return SBI_EINVAL; + if (!fw_single_region) { + if (scratch->fw_rw_offset == 0 || + (scratch->fw_rw_offset & (scratch->fw_rw_offset - 1)) != 0) { + sbi_printf("%s: fw_rw_offset is not a power of 2 (0x%lx)\n", + __func__, scratch->fw_rw_offset); + return SBI_EINVAL; + } + if ((scratch->fw_start & (scratch->fw_rw_offset - 1)) != 0) { + sbi_printf("%s: fw_start and fw_rw_offset not aligned\n", + __func__); + return SBI_EINVAL; + } } domain_hart_ptr_offset = sbi_scratch_alloc_type_offset(void *); @@ -913,7 +919,7 @@ int sbi_domain_init(struct sbi_scratch *scratch, u32 cold_hartid) root.possible_harts = root_hmask; /* Root domain firmware memory region */ - if (sbi_platform_single_fw_region(sbi_platform_ptr(scratch))) { + if (fw_single_region) { sbi_domain_memregion_init(scratch->fw_start, scratch->fw_size, (SBI_DOMAIN_MEMREGION_M_READABLE | SBI_DOMAIN_MEMREGION_M_WRITABLE | diff --git a/lib/sbi/sbi_domain_context.c b/lib/sbi/sbi_domain_context.c index 158f4990432..cc4cc04ddb4 100644 --- a/lib/sbi/sbi_domain_context.c +++ b/lib/sbi/sbi_domain_context.c @@ -18,6 +18,8 @@ #include #include #include +#include +#include /** Context representation for a hart within a domain */ struct hart_context { @@ -49,6 +51,11 @@ struct hart_context { /** Supervisor resource management configuration register */ unsigned long srmcfg; + /** Float context state */ + struct sbi_fp_context fp_ctx; + /** Vector context state */ + struct sbi_vector_context *vec_ctx; + /** Reference to the owning domain */ struct sbi_domain *dom; /** Previous context (caller) to jump to during context exits */ @@ -122,10 +129,6 @@ static int switch_to_next_domain_context(struct hart_context *ctx, sbi_hartmask_set_hartindex(hartindex, &target_dom->assigned_harts); spin_unlock(&target_dom->assigned_harts_lock); - /* Reconfigure PMP settings for the new domain */ - sbi_hart_protection_unconfigure(scratch); - sbi_hart_protection_configure(scratch); - /* Save current CSR context and restore target domain's CSR context */ ctx->sstatus = csr_swap(CSR_SSTATUS, dom_ctx->sstatus); ctx->sie = csr_swap(CSR_SIE, dom_ctx->sie); @@ -143,11 +146,34 @@ static int switch_to_next_domain_context(struct hart_context *ctx, if (sbi_hart_has_extension(scratch, SBI_HART_EXT_SSQOSID)) ctx->srmcfg = csr_swap(CSR_SRMCFG, dom_ctx->srmcfg); + /* Eager context switch for float */ + if (sbi_hart_has_extension(scratch, SBI_HART_EXT_F) || + sbi_hart_has_extension(scratch, SBI_HART_EXT_D)) { + sbi_fp_save(&ctx->fp_ctx); + sbi_fp_restore(&dom_ctx->fp_ctx); + } + + /* Eager context switch for vector */ + if (sbi_hart_has_extension(scratch, SBI_HART_EXT_V)) { + sbi_vector_save(ctx->vec_ctx); + sbi_vector_restore(dom_ctx->vec_ctx); + } + /* Save current trap state and restore target domain's trap state */ trap_ctx = sbi_trap_get_context(scratch); sbi_memcpy(&ctx->trap_ctx, trap_ctx, sizeof(*trap_ctx)); sbi_memcpy(trap_ctx, &dom_ctx->trap_ctx, sizeof(*trap_ctx)); + /* + * Re-configure PMP settings for the new domain + * + * This will internally perform full SFENCE / HFENCE which + * is also required for some of the above CSR updates (such + * as satp CSR). + */ + sbi_hart_protection_unconfigure(scratch); + sbi_hart_protection_configure(scratch); + /* Mark current context structure initialized because context saved */ ctx->initialized = true; @@ -168,6 +194,7 @@ static int switch_to_next_domain_context(struct hart_context *ctx, static int hart_context_init(u32 hartindex) { + size_t vec_size; struct hart_context *ctx; struct sbi_domain *dom; @@ -180,6 +207,18 @@ static int hart_context_init(u32 hartindex) if (!ctx) return SBI_ENOMEM; + if (sbi_hart_has_extension(sbi_scratch_thishart_ptr(), + SBI_HART_EXT_V)) { + vec_size = sbi_vector_context_size(); + + /* Allocate the vector context pointer */ + ctx->vec_ctx = sbi_zalloc(vec_size); + if (!ctx->vec_ctx) { + sbi_free(ctx); + return SBI_ENOMEM; + } + } + /* Bind context and domain */ ctx->dom = dom; hart_context_set(dom, hartindex, ctx); diff --git a/lib/sbi/sbi_ecall_legacy.c b/lib/sbi/sbi_ecall_legacy.c index 50a7660df1f..4501c4adb8e 100644 --- a/lib/sbi/sbi_ecall_legacy.c +++ b/lib/sbi/sbi_ecall_legacy.c @@ -54,9 +54,9 @@ static int sbi_ecall_legacy_handler(unsigned long extid, unsigned long funcid, switch (extid) { case SBI_EXT_0_1_SET_TIMER: #if __riscv_xlen == 32 - sbi_timer_event_start((((u64)regs->a1 << 32) | (u64)regs->a0)); + sbi_timer_smode_event_start((((u64)regs->a1 << 32) | (u64)regs->a0)); #else - sbi_timer_event_start((u64)regs->a0); + sbi_timer_smode_event_start((u64)regs->a0); #endif break; case SBI_EXT_0_1_CONSOLE_PUTCHAR: diff --git a/lib/sbi/sbi_ecall_pmu.c b/lib/sbi/sbi_ecall_pmu.c index 868e8665be8..41bb16245d3 100644 --- a/lib/sbi/sbi_ecall_pmu.c +++ b/lib/sbi/sbi_ecall_pmu.c @@ -50,12 +50,12 @@ static int sbi_ecall_pmu_handler(unsigned long extid, unsigned long funcid, break; case SBI_EXT_PMU_COUNTER_FW_READ: - ret = sbi_pmu_ctr_fw_read(regs->a0, &temp); + ret = sbi_pmu_ctr_fw_read(regs->a0, &temp, false); out->value = temp; break; case SBI_EXT_PMU_COUNTER_FW_READ_HI: + ret = sbi_pmu_ctr_fw_read(regs->a0, &temp, true); #if __riscv_xlen == 32 - ret = sbi_pmu_ctr_fw_read(regs->a0, &temp); out->value = temp >> 32; #else out->value = 0; diff --git a/lib/sbi/sbi_ecall_time.c b/lib/sbi/sbi_ecall_time.c index 6ea6f054150..a0aa580d04f 100644 --- a/lib/sbi/sbi_ecall_time.c +++ b/lib/sbi/sbi_ecall_time.c @@ -22,9 +22,9 @@ static int sbi_ecall_time_handler(unsigned long extid, unsigned long funcid, if (funcid == SBI_EXT_TIME_SET_TIMER) { #if __riscv_xlen == 32 - sbi_timer_event_start((((u64)regs->a1 << 32) | (u64)regs->a0)); + sbi_timer_smode_event_start((((u64)regs->a1 << 32) | (u64)regs->a0)); #else - sbi_timer_event_start((u64)regs->a0); + sbi_timer_smode_event_start((u64)regs->a0); #endif } else ret = SBI_ENOTSUPP; diff --git a/lib/sbi/sbi_emulate_csr.c b/lib/sbi/sbi_emulate_csr.c index c2253c82c52..e0e23738741 100644 --- a/lib/sbi/sbi_emulate_csr.c +++ b/lib/sbi/sbi_emulate_csr.c @@ -76,6 +76,12 @@ int sbi_emulate_csr_read(int csr_num, struct sbi_trap_regs *regs, return SBI_ENOTSUPP; *csr_val = csr_read(CSR_MINSTRET); break; + case CSR_SENVCFG: + if (prev_mode == PRV_S && !virt) + *csr_val = scratch->sw_senvcfg; + else + ret = SBI_ENOTSUPP; + break; #if __riscv_xlen == 32 case CSR_HTIMEDELTAH: @@ -162,6 +168,14 @@ int sbi_emulate_csr_write(int csr_num, struct sbi_trap_regs *regs, else ret = SBI_ENOTSUPP; break; + case CSR_SENVCFG: + if (prev_mode == PRV_S && !virt) { + struct sbi_scratch *scratch = sbi_scratch_thishart_ptr(); + scratch->sw_senvcfg = csr_val; + } + else + ret = SBI_ENOTSUPP; + break; #if __riscv_xlen == 32 case CSR_HTIMEDELTAH: if (prev_mode == PRV_S && !virt) diff --git a/lib/sbi/sbi_fp.c b/lib/sbi/sbi_fp.c new file mode 100644 index 00000000000..c8941252a54 --- /dev/null +++ b/lib/sbi/sbi_fp.c @@ -0,0 +1,192 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 RISCstar Solutions. + * + * Authors: + * Dave Patel + */ + +#include +#include +#include + +#if defined(__riscv_f) || defined(__riscv_d) + +void sbi_fp_save(struct sbi_fp_context *dst) +{ + unsigned long mstatus_orig; + + if (!dst) + return; + + mstatus_orig = csr_read_set(CSR_MSTATUS, MSTATUS_VS); + + asm volatile( +#if defined(__riscv_d) + "fsd f0, 0(%0)\n" + "fsd f1, 8(%0)\n" + "fsd f2, 16(%0)\n" + "fsd f3, 24(%0)\n" + "fsd f4, 32(%0)\n" + "fsd f5, 40(%0)\n" + "fsd f6, 48(%0)\n" + "fsd f7, 56(%0)\n" + "fsd f8, 64(%0)\n" + "fsd f9, 72(%0)\n" + "fsd f10, 80(%0)\n" + "fsd f11, 88(%0)\n" + "fsd f12, 96(%0)\n" + "fsd f13, 104(%0)\n" + "fsd f14, 112(%0)\n" + "fsd f15, 120(%0)\n" + "fsd f16, 128(%0)\n" + "fsd f17, 136(%0)\n" + "fsd f18, 144(%0)\n" + "fsd f19, 152(%0)\n" + "fsd f20, 160(%0)\n" + "fsd f21, 168(%0)\n" + "fsd f22, 176(%0)\n" + "fsd f23, 184(%0)\n" + "fsd f24, 192(%0)\n" + "fsd f25, 200(%0)\n" + "fsd f26, 208(%0)\n" + "fsd f27, 216(%0)\n" + "fsd f28, 224(%0)\n" + "fsd f29, 232(%0)\n" + "fsd f30, 240(%0)\n" + "fsd f31, 248(%0)\n" +#else + "fsw f0, 0(%0)\n" + "fsw f1, 4(%0)\n" + "fsw f2, 8(%0)\n" + "fsw f3, 12(%0)\n" + "fsw f4, 16(%0)\n" + "fsw f5, 20(%0)\n" + "fsw f6, 24(%0)\n" + "fsw f7, 28(%0)\n" + "fsw f8, 32(%0)\n" + "fsw f9, 36(%0)\n" + "fsw f10, 40(%0)\n" + "fsw f11, 44(%0)\n" + "fsw f12, 48(%0)\n" + "fsw f13, 52(%0)\n" + "fsw f14, 56(%0)\n" + "fsw f15, 60(%0)\n" + "fsw f16, 64(%0)\n" + "fsw f17, 68(%0)\n" + "fsw f18, 72(%0)\n" + "fsw f19, 76(%0)\n" + "fsw f20, 80(%0)\n" + "fsw f21, 84(%0)\n" + "fsw f22, 88(%0)\n" + "fsw f23, 92(%0)\n" + "fsw f24, 96(%0)\n" + "fsw f25, 100(%0)\n" + "fsw f26, 104(%0)\n" + "fsw f27, 108(%0)\n" + "fsw f28, 112(%0)\n" + "fsw f29, 116(%0)\n" + "fsw f30, 120(%0)\n" + "fsw f31, 124(%0)\n" +#endif /* __riscv_d */ + : + : "r"(dst->f) + : "memory" + ); + + dst->fcsr = csr_read(CSR_FCSR); + + /* Restore original mstatus LAST */ + csr_write(CSR_MSTATUS, mstatus_orig); +} + +void sbi_fp_restore(const struct sbi_fp_context *src) +{ + unsigned long mstatus_orig; + + if (!src) + return; + + /* Save original mstatus */ + mstatus_orig = csr_read_set(CSR_MSTATUS, MSTATUS_FS); + + asm volatile( +#if defined(__riscv_d) + "fld f0, 0(%0)\n" + "fld f1, 8(%0)\n" + "fld f2, 16(%0)\n" + "fld f3, 24(%0)\n" + "fld f4, 32(%0)\n" + "fld f5, 40(%0)\n" + "fld f6, 48(%0)\n" + "fld f7, 56(%0)\n" + "fld f8, 64(%0)\n" + "fld f9, 72(%0)\n" + "fld f10, 80(%0)\n" + "fld f11, 88(%0)\n" + "fld f12, 96(%0)\n" + "fld f13, 104(%0)\n" + "fld f14, 112(%0)\n" + "fld f15, 120(%0)\n" + "fld f16, 128(%0)\n" + "fld f17, 136(%0)\n" + "fld f18, 144(%0)\n" + "fld f19, 152(%0)\n" + "fld f20, 160(%0)\n" + "fld f21, 168(%0)\n" + "fld f22, 176(%0)\n" + "fld f23, 184(%0)\n" + "fld f24, 192(%0)\n" + "fld f25, 200(%0)\n" + "fld f26, 208(%0)\n" + "fld f27, 216(%0)\n" + "fld f28, 224(%0)\n" + "fld f29, 232(%0)\n" + "fld f30, 240(%0)\n" + "fld f31, 248(%0)\n" +#else + "flw f0, 0(%0)\n" + "flw f1, 4(%0)\n" + "flw f2, 8(%0)\n" + "flw f3, 12(%0)\n" + "flw f4, 16(%0)\n" + "flw f5, 20(%0)\n" + "flw f6, 24(%0)\n" + "flw f7, 28(%0)\n" + "flw f8, 32(%0)\n" + "flw f9, 36(%0)\n" + "flw f10, 40(%0)\n" + "flw f11, 44(%0)\n" + "flw f12, 48(%0)\n" + "flw f13, 52(%0)\n" + "flw f14, 56(%0)\n" + "flw f15, 60(%0)\n" + "flw f16, 64(%0)\n" + "flw f17, 68(%0)\n" + "flw f18, 72(%0)\n" + "flw f19, 76(%0)\n" + "flw f20, 80(%0)\n" + "flw f21, 84(%0)\n" + "flw f22, 88(%0)\n" + "flw f23, 92(%0)\n" + "flw f24, 96(%0)\n" + "flw f25, 100(%0)\n" + "flw f26, 104(%0)\n" + "flw f27, 108(%0)\n" + "flw f28, 112(%0)\n" + "flw f29, 116(%0)\n" + "flw f30, 120(%0)\n" + "flw f31, 124(%0)\n" +#endif /* __riscv_d */ + : + : "r"(src->f) + : "memory" + ); + + csr_write(CSR_FCSR, src->fcsr); + + /* Restore original mstatus LAST */ + csr_write(CSR_MSTATUS, mstatus_orig); +} +#endif /* __riscv_f || __riscv_d */ diff --git a/lib/sbi/sbi_fwft.c b/lib/sbi/sbi_fwft.c index 373140b746b..0b97078334e 100644 --- a/lib/sbi/sbi_fwft.c +++ b/lib/sbi/sbi_fwft.c @@ -115,7 +115,7 @@ static int fwft_misaligned_delegation_supported(struct fwft_config *conf) if (!misa_extension('S')) return SBI_ENOTSUPP; - return SBI_OK; + return SBI_ENOTSUPP; } static int fwft_set_misaligned_delegation(struct fwft_config *conf, @@ -160,8 +160,16 @@ static int fwft_get_double_trap(struct fwft_config *conf, unsigned long *value) static int fwft_adue_supported(struct fwft_config *conf) { + /* + * FWFT.PTE_AD_HW_UPDATING is only supported when both Svade and Svadu + * are supported. We need both in order to support toggling and to + * ensure the reset value of zero is valid (it wouldn't be when only + * Svadu is supported). + */ if (!sbi_hart_has_extension(sbi_scratch_thishart_ptr(), - SBI_HART_EXT_SVADU)) + SBI_HART_EXT_SVADU) || + !sbi_hart_has_extension(sbi_scratch_thishart_ptr(), + SBI_HART_EXT_SVADE)) return SBI_ENOTSUPP; return SBI_OK; @@ -227,9 +235,11 @@ static int fwft_get_sstack(struct fwft_config *conf, unsigned long *value) #if __riscv_xlen > 32 static int fwft_pmlen_supported(struct fwft_config *conf) { +#ifndef CONFIG_EMU_SUPM if (!sbi_hart_has_extension(sbi_scratch_thishart_ptr(), SBI_HART_EXT_SMNPM)) return SBI_ENOTSUPP; +#endif return SBI_OK; } @@ -252,11 +262,28 @@ static int fwft_set_pmlen(struct fwft_config *conf, unsigned long value) return SBI_EINVAL; } +#ifdef CONFIG_EMU_SUPM + /* Reset emulated pointer masking */ + struct sbi_scratch *scratch = sbi_scratch_thishart_ptr(); + scratch->sw_pm = 0; + + /* Don't even try to change MENVCFG if absent */ + if (!sbi_hart_has_csr(scratch, SBI_HART_CSR_MENVCFG)) { + scratch->sw_pm = value; + return SBI_OK; + } +#endif + prev = csr_read_clear(CSR_MENVCFG, ENVCFG_PMM); csr_set(CSR_MENVCFG, pmm); if ((csr_read(CSR_MENVCFG) & ENVCFG_PMM) != pmm) { csr_write(CSR_MENVCFG, prev); +#ifdef CONFIG_EMU_SUPM + /* Instead of returning SBI_EINVAL, enable emulation */ + scratch->sw_pm = value; +#else return SBI_EINVAL; +#endif } return SBI_OK; @@ -264,6 +291,21 @@ static int fwft_set_pmlen(struct fwft_config *conf, unsigned long value) static int fwft_get_pmlen(struct fwft_config *conf, unsigned long *value) { +#ifdef CONFIG_EMU_SUPM + /* Check for emulated pointer masking */ + struct sbi_scratch *scratch = sbi_scratch_thishart_ptr(); + if (scratch->sw_pm) { + *value = scratch->sw_pm; + return SBI_OK; + } + + /* Handle disabled pointer masking in the absence of MENVCFG */ + if (!sbi_hart_has_csr(scratch, SBI_HART_CSR_MENVCFG)) { + *value = 0; + return SBI_OK; + } +#endif + switch (csr_read(CSR_MENVCFG) & ENVCFG_PMM) { case ENVCFG_PMM_PMLEN_0: *value = 0; diff --git a/lib/sbi/sbi_hart.c b/lib/sbi/sbi_hart.c index 60e95bca3a8..dd16b199e6f 100644 --- a/lib/sbi/sbi_hart.c +++ b/lib/sbi/sbi_hart.c @@ -70,7 +70,7 @@ static void mstatus_init(struct sbi_scratch *scratch) */ mhpmevent_init_val |= (MHPMEVENT_OF | MHPMEVENT_MINH); for (cidx = 0; cidx <= 28; cidx++) { - if (!(mhpm_mask & 1 << (cidx + 3))) + if (!(mhpm_mask & 1UL << (cidx + 3))) continue; #if __riscv_xlen == 32 csr_write_num(CSR_MHPMEVENT3 + cidx, @@ -137,6 +137,9 @@ static void mstatus_init(struct sbi_scratch *scratch) if (sbi_hart_priv_version(scratch) >= SBI_HART_PRIV_VER_1_12) { menvcfg_val = csr_read64(CSR_MENVCFG); + /* Disable HW A/D updating by default */ + menvcfg_val &= ~ENVCFG_ADUE; + /* Disable double trap by default */ menvcfg_val &= ~ENVCFG_DTE; @@ -158,18 +161,17 @@ static void mstatus_init(struct sbi_scratch *scratch) #endif __set_menvcfg_ext(SBI_HART_EXT_SSTC, ENVCFG_STCE) __set_menvcfg_ext(SBI_HART_EXT_SMCDELEG, ENVCFG_CDE); - __set_menvcfg_ext(SBI_HART_EXT_SVADU, ENVCFG_ADUE); - -#undef __set_menvcfg_ext /* - * When both Svade and Svadu are present in DT, the default scheme for managing - * the PTE A/D bits should use Svade. Check Svadu before Svade extension to ensure - * that the ADUE bit is cleared when the Svade support are specified. + * Assume only Svadu is supported when it is the only extension + * present in the ISA string. Svade is assumed when neither are + * present. When both are present we must default to Svade (see + * the zero reset value of FWFT.PTE_AD_HW_UPDATING). */ + if (!sbi_hart_has_extension(scratch, SBI_HART_EXT_SVADE)) + __set_menvcfg_ext(SBI_HART_EXT_SVADU, ENVCFG_ADUE); - if (sbi_hart_has_extension(scratch, SBI_HART_EXT_SVADE)) - menvcfg_val &= ~ENVCFG_ADUE; +#undef __set_menvcfg_ext csr_write64(CSR_MENVCFG, menvcfg_val); @@ -396,6 +398,10 @@ const struct sbi_hart_ext_data sbi_hart_ext[] = { __SBI_HART_EXT_DATA(ssstateen, SBI_HART_EXT_SSSTATEEN), __SBI_HART_EXT_DATA(xsfcflushdlone, SBI_HART_EXT_XSIFIVE_CFLUSH_D_L1), __SBI_HART_EXT_DATA(xsfcease, SBI_HART_EXT_XSIFIVE_CEASE), + __SBI_HART_EXT_DATA(smrnmi, SBI_HART_EXT_SMRNMI), + __SBI_HART_EXT_DATA(v, SBI_HART_EXT_V), + __SBI_HART_EXT_DATA(f, SBI_HART_EXT_F), + __SBI_HART_EXT_DATA(d, SBI_HART_EXT_D), }; _Static_assert(SBI_HART_EXT_MAX == array_size(sbi_hart_ext), @@ -502,7 +508,7 @@ static int hart_mhpm_get_allowed_bits(void) return num_bits; } -static int hart_detect_features(struct sbi_scratch *scratch) +static int hart_detect_features(struct sbi_scratch *scratch, bool cold_boot) { struct sbi_trap_info trap = {0}; struct sbi_hart_features *hfeatures = @@ -514,12 +520,34 @@ static int hart_detect_features(struct sbi_scratch *scratch) if (hfeatures->detected) return 0; - /* Clear hart features */ - sbi_memset(hfeatures->extensions, 0, sizeof(hfeatures->extensions)); - sbi_memset(hfeatures->csrs, 0, sizeof(hfeatures->csrs)); - hfeatures->pmp_count = 0; - hfeatures->mhpm_mask = 0; - hfeatures->priv_version = SBI_HART_PRIV_VER_UNKNOWN; + /* + * Parse device tree extensions early, before any trap-based checks. + * Needed to detect Smrnmi and install NMI handlers before CSR probes + * that may trigger traps. + */ + rc = sbi_platform_extensions_init(sbi_platform_ptr(scratch), cold_boot); + if (rc) + return rc; + + if (sbi_hart_has_extension(scratch, SBI_HART_EXT_SMRNMI)) { + const struct sbi_platform *plat = sbi_platform_thishart_ptr(); + const struct sbi_platform_operations *ops = sbi_platform_ops(plat); + extern void _trap_rnmi_handler(void); + extern void _trap_handler(void); + + if (!ops || !ops->smrnmi_handlers_init) + sbi_panic("Smrnmi detected, but platform lacks smrnmi_handlers_init callback\n"); + + /* Reuse _trap_handler for the RNME slot since RNME is taken + * as a regular M-mode trap with NMIE=0. */ + ops->smrnmi_handlers_init(_trap_rnmi_handler, _trap_handler); + + /* Initialize MNSCRATCH for the RNMI handler */ + csr_write(CSR_MNSCRATCH, scratch); + + /* Enable NMIs */ + csr_set(CSR_MNSTATUS, MNSTATUS_NMIE); + } #define __check_hpm_csr(__csr, __mask) \ oldval = csr_read_allowed(__csr, &trap); \ @@ -672,15 +700,13 @@ static int hart_detect_features(struct sbi_scratch *scratch) __check_csr_existence(CSR_CYCLE, SBI_HART_CSR_CYCLE); __check_csr_existence(CSR_TIME, SBI_HART_CSR_TIME); __check_csr_existence(CSR_INSTRET, SBI_HART_CSR_INSTRET); + __check_csr_existence(CSR_MENVCFG, SBI_HART_CSR_MENVCFG); + __check_csr_existence(CSR_SENVCFG, SBI_HART_CSR_SENVCFG); + /* Initialize value of emulated SENVCFG CSR */ + scratch->sw_senvcfg = ENVCFG_CBZE | ENVCFG_CBCFE | ENVCFG_CBIE; #undef __check_csr_existence - /* Let platform populate extensions */ - rc = sbi_platform_extensions_init(sbi_platform_thishart_ptr(), - hfeatures); - if (rc) - return rc; - /* Zicntr should only be detected using traps */ __sbi_hart_update_extension(hfeatures, SBI_HART_EXT_ZICNTR, sbi_hart_has_csr(scratch, SBI_HART_CSR_CYCLE) && @@ -741,7 +767,7 @@ int sbi_hart_init(struct sbi_scratch *scratch, bool cold_boot) return SBI_ENOMEM; } - rc = hart_detect_features(scratch); + rc = hart_detect_features(scratch, cold_boot); if (rc) return rc; diff --git a/lib/sbi/sbi_hart_pmp.c b/lib/sbi/sbi_hart_pmp.c index be459129c52..4528258d9a2 100644 --- a/lib/sbi/sbi_hart_pmp.c +++ b/lib/sbi/sbi_hart_pmp.c @@ -12,9 +12,119 @@ #include #include #include +#include #include #include +static int hart_pmp_read(pmp_t *pmp, unsigned int n) +{ + int pmpcfg_csr, pmpcfg_shift, pmpaddr_csr; + unsigned long cfgmask; + + /* check parameters */ + if (n >= PMP_COUNT) + return SBI_EINVAL; + + /* calculate PMP register and offset */ +#if __riscv_xlen == 32 + pmpcfg_csr = CSR_PMPCFG0 + (n >> 2); + pmpcfg_shift = (n & 3) << 3; +#elif __riscv_xlen == 64 + pmpcfg_csr = (CSR_PMPCFG0 + (n >> 2)) & ~1; + pmpcfg_shift = (n & 7) << 3; +#else +# error "Unexpected __riscv_xlen" +#endif + pmpaddr_csr = CSR_PMPADDR0 + n; + + cfgmask = (0xffUL << pmpcfg_shift); + pmp->cfg = (csr_read_num(pmpcfg_csr) & cfgmask) >> pmpcfg_shift; + pmp->addr = csr_read_num(pmpaddr_csr); + + return SBI_OK; +} + +static int hart_pmp_write(pmp_t *pmp, unsigned int n) +{ + int pmpcfg_csr, pmpcfg_shift, pmpaddr_csr; + unsigned long cfgmask, pmpcfg; + + /* check parameters */ + if (n >= PMP_COUNT) + return SBI_EINVAL; + + /* calculate PMP register and offset */ +#if __riscv_xlen == 32 + pmpcfg_csr = CSR_PMPCFG0 + (n >> 2); + pmpcfg_shift = (n & 3) << 3; +#elif __riscv_xlen == 64 + pmpcfg_csr = (CSR_PMPCFG0 + (n >> 2)) & ~1; + pmpcfg_shift = (n & 7) << 3; +#else +# error "Unexpected __riscv_xlen" +#endif + pmpaddr_csr = CSR_PMPADDR0 + n; + + /* write csrs */ + csr_write_num(pmpaddr_csr, pmp->addr); + cfgmask = ~(0xffUL << pmpcfg_shift); + pmpcfg = (csr_read_num(pmpcfg_csr) & cfgmask); + pmpcfg |= (((unsigned long)pmp->cfg << pmpcfg_shift) & ~cfgmask); + csr_write_num(pmpcfg_csr, pmpcfg); + + return SBI_OK; +} + +int sbi_hart_pmp_disable(unsigned int n) +{ + pmp_t pmp; + int rc; + + rc = hart_pmp_read(&pmp, n); + if (rc) + return rc; + + pmp.cfg = 0; + + return hart_pmp_write(&pmp, n); +} + +bool sbi_hart_is_pmp_enabled(unsigned int n) +{ + pmp_t pmp; + + if (hart_pmp_read(&pmp, n) != SBI_OK) + return false; + + return sbi_pmp_is_enabled(&pmp); +} + +int sbi_hart_pmp_set(unsigned int n, unsigned long prot, unsigned long addr, + unsigned long log2len) +{ + pmp_t pmp; + int rc; + + rc = sbi_pmp_encode(&pmp, prot, addr, log2len); + if (rc) + return rc; + + return hart_pmp_write(&pmp, n); +} + +int sbi_hart_pmp_get(unsigned int n, unsigned long *prot_out, unsigned long *addr_out, + unsigned long *log2len) +{ + pmp_t pmp; + int rc; + + rc = hart_pmp_read(&pmp, n); + if (rc) + return rc; + + return sbi_pmp_decode(&pmp, prot_out, addr_out, log2len); +} + /* * Smepmp enforces access boundaries between M-mode and * S/U-mode. When it is enabled, the PMPs are programmed @@ -100,7 +210,7 @@ static void sbi_hart_smepmp_set(struct sbi_scratch *scratch, sbi_platform_pmp_set(sbi_platform_ptr(scratch), pmp_idx, reg->flags, pmp_flags, reg->base, reg->order); - pmp_set(pmp_idx, pmp_flags, reg->base, reg->order); + sbi_hart_pmp_set(pmp_idx, pmp_flags, reg->base, reg->order); } else { sbi_printf("Can not configure pmp for domain %s because" " memory region address 0x%lx or size 0x%lx " @@ -139,7 +249,7 @@ static int sbi_hart_smepmp_configure(struct sbi_scratch *scratch) csr_set(CSR_MSECCFG, MSECCFG_RLB); /* Disable the reserved entry */ - pmp_disable(SBI_SMEPMP_RESV_ENTRY); + sbi_hart_pmp_disable(SBI_SMEPMP_RESV_ENTRY); /* Program M-only regions when MML is not set. */ pmp_idx = 0; @@ -204,6 +314,9 @@ static int sbi_hart_smepmp_configure(struct sbi_scratch *scratch) sbi_hart_smepmp_set(scratch, dom, reg, pmp_idx++, pmp_flags, pmp_log2gran, pmp_addr_max); } + /* Disable remaining PMP entries */ + for(; pmp_idx < pmp_count; pmp_idx++) + sbi_hart_pmp_disable(pmp_idx); /* * All entries are programmed. @@ -221,7 +334,7 @@ static int sbi_hart_smepmp_map_range(struct sbi_scratch *scratch, unsigned int pmp_flags = (PMP_W | PMP_X); unsigned long order, base = 0; - if (is_pmp_entry_mapped(SBI_SMEPMP_RESV_ENTRY)) + if (sbi_hart_is_pmp_enabled(SBI_SMEPMP_RESV_ENTRY)) return SBI_ENOSPC; for (order = MAX(sbi_hart_pmp_log2gran(scratch), log2roundup(size)); @@ -241,7 +354,7 @@ static int sbi_hart_smepmp_map_range(struct sbi_scratch *scratch, sbi_platform_pmp_set(sbi_platform_ptr(scratch), SBI_SMEPMP_RESV_ENTRY, SBI_DOMAIN_MEMREGION_SHARED_SURW_MRW, pmp_flags, base, order); - pmp_set(SBI_SMEPMP_RESV_ENTRY, pmp_flags, base, order); + sbi_hart_pmp_set(SBI_SMEPMP_RESV_ENTRY, pmp_flags, base, order); return SBI_OK; } @@ -250,7 +363,7 @@ static int sbi_hart_smepmp_unmap_range(struct sbi_scratch *scratch, unsigned long addr, unsigned long size) { sbi_platform_pmp_disable(sbi_platform_ptr(scratch), SBI_SMEPMP_RESV_ENTRY); - return pmp_disable(SBI_SMEPMP_RESV_ENTRY); + return sbi_hart_pmp_disable(SBI_SMEPMP_RESV_ENTRY); } static int sbi_hart_oldpmp_configure(struct sbi_scratch *scratch) @@ -278,7 +391,7 @@ static int sbi_hart_oldpmp_configure(struct sbi_scratch *scratch) sbi_platform_pmp_set(sbi_platform_ptr(scratch), pmp_idx, reg->flags, pmp_flags, reg->base, reg->order); - pmp_set(pmp_idx++, pmp_flags, reg->base, reg->order); + sbi_hart_pmp_set(pmp_idx++, pmp_flags, reg->base, reg->order); } else { sbi_printf("Can not configure pmp for domain %s because" " memory region address 0x%lx or size 0x%lx " @@ -286,6 +399,9 @@ static int sbi_hart_oldpmp_configure(struct sbi_scratch *scratch) reg->order); } } + /* Disable remaining PMP entries */ + for(; pmp_idx < pmp_count; pmp_idx++) + sbi_hart_pmp_disable(pmp_idx); sbi_hart_pmp_fence(); return 0; @@ -301,7 +417,7 @@ static void sbi_hart_pmp_unconfigure(struct sbi_scratch *scratch) continue; sbi_platform_pmp_disable(sbi_platform_ptr(scratch), i); - pmp_disable(i); + sbi_hart_pmp_disable(i); } } diff --git a/lib/sbi/sbi_hart_protection.c b/lib/sbi/sbi_hart_protection.c index 306f5afbdae..fbebfd1a041 100644 --- a/lib/sbi/sbi_hart_protection.c +++ b/lib/sbi/sbi_hart_protection.c @@ -54,7 +54,7 @@ int sbi_hart_protection_configure(struct sbi_scratch *scratch) struct sbi_hart_protection *hprot = sbi_hart_protection_best(); if (!hprot) - return SBI_EINVAL; + return 0; if (!hprot->configure) return SBI_ENOSYS; @@ -75,9 +75,7 @@ int sbi_hart_protection_map_range(unsigned long base, unsigned long size) { struct sbi_hart_protection *hprot = sbi_hart_protection_best(); - if (!hprot) - return SBI_EINVAL; - if (!hprot->map_range) + if (!hprot || !hprot->map_range) return 0; return hprot->map_range(sbi_scratch_thishart_ptr(), base, size); @@ -87,9 +85,7 @@ int sbi_hart_protection_unmap_range(unsigned long base, unsigned long size) { struct sbi_hart_protection *hprot = sbi_hart_protection_best(); - if (!hprot) - return SBI_EINVAL; - if (!hprot->unmap_range) + if (!hprot || !hprot->unmap_range) return 0; return hprot->unmap_range(sbi_scratch_thishart_ptr(), base, size); diff --git a/lib/sbi/sbi_illegal_insn.c b/lib/sbi/sbi_illegal_insn.c index fa82264a924..47f050c1d84 100644 --- a/lib/sbi/sbi_illegal_insn.c +++ b/lib/sbi/sbi_illegal_insn.c @@ -2,9 +2,13 @@ * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Western Digital Corporation or its affiliates. + * Copyright (c) 2025 Benedikt Freisen*. * * Authors: * Anup Patel + * Benedikt Freisen + * + * *) ISA extension emulation */ #include @@ -15,6 +19,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -25,7 +32,7 @@ int truly_illegal_insn(ulong insn, struct sbi_trap_regs *regs) struct sbi_trap_info trap; trap.cause = CAUSE_ILLEGAL_INSTRUCTION; - trap.tval = insn; + trap.tval = insn; trap.tval2 = 0; trap.tinst = 0; trap.gva = 0; @@ -42,7 +49,23 @@ static int misc_mem_opcode_insn(ulong insn, struct sbi_trap_regs *regs) return 0; } - return truly_illegal_insn(insn, regs); + /* Errata workaround: C906, C910 fail to ignore reserved fields + * in the `fence` and `fence.i` encodings. [Thomas2024RISCVuzz] */ + if ((insn & INSN_MASK_FENCE) == INSN_MATCH_FENCE) { + /* NOTE: Emulation should ideally preserve the `pred` and + * `succ` fields, but that is not easily possible here. */ + mb(); + regs->mepc += 4; + return 0; + } + if ((insn & INSN_MASK_FENCE_I) == INSN_MATCH_FENCE_I) { + RISCV_FENCE_I; + regs->mepc += 4; + return 0; + } + + /* Delegate Zicbom and Zicboz emulation */ + return sbi_insn_emu_zicbom_zicboz(insn, regs); } static int system_opcode_insn(ulong insn, struct sbi_trap_regs *regs) @@ -55,14 +78,29 @@ static int system_opcode_insn(ulong insn, struct sbi_trap_regs *regs) ulong csr_val, new_csr_val; if (prev_mode == PRV_M) { - sbi_printf("%s: Failed to access CSR %#x from M-mode", - __func__, csr_num); + sbi_printf("%s: Failed to access CSR %#x from M-mode", __func__, + csr_num); return SBI_EFAIL; } /* Ensure that we got CSR read/write instruction */ int funct3 = GET_RM(insn); if (funct3 == 0 || funct3 == 4) { + /* Handle "Zawrs" Wait-on-Reservation-Set */ + if (insn == INSN_MATCH_WRS_NTO || insn == INSN_MATCH_WRS_STO) { + /* do nothing */ + regs->mepc += 4; + return 0; + } + /* Handle "Zimop" May-Be-Operations */ + if ((insn & INSN_MASK_MOP_R_N) == INSN_MATCH_MOP_R_N || + (insn & INSN_MASK_MOP_RR_N) == INSN_MATCH_MOP_RR_N) { + SET_RD(insn, regs, 0); + regs->mepc += 4; + return 0; + } + + /* Otherwise treat this as an error */ sbi_printf("%s: Invalid opcode for CSR read/write instruction", __func__); return truly_illegal_insn(insn, regs); @@ -111,50 +149,77 @@ static int system_opcode_insn(ulong insn, struct sbi_trap_regs *regs) } static const illegal_insn_func illegal_insn_table[32] = { - truly_illegal_insn, /* 0 */ - truly_illegal_insn, /* 1 */ - truly_illegal_insn, /* 2 */ - misc_mem_opcode_insn, /* 3 */ - truly_illegal_insn, /* 4 */ - truly_illegal_insn, /* 5 */ - truly_illegal_insn, /* 6 */ - truly_illegal_insn, /* 7 */ - truly_illegal_insn, /* 8 */ - truly_illegal_insn, /* 9 */ - truly_illegal_insn, /* 10 */ - sbi_illegal_atomic, /* 11 */ - truly_illegal_insn, /* 12 */ - truly_illegal_insn, /* 13 */ - truly_illegal_insn, /* 14 */ - truly_illegal_insn, /* 15 */ - truly_illegal_insn, /* 16 */ - truly_illegal_insn, /* 17 */ - truly_illegal_insn, /* 18 */ - truly_illegal_insn, /* 19 */ - truly_illegal_insn, /* 20 */ - truly_illegal_insn, /* 21 */ - truly_illegal_insn, /* 22 */ - truly_illegal_insn, /* 23 */ - truly_illegal_insn, /* 24 */ - truly_illegal_insn, /* 25 */ - truly_illegal_insn, /* 26 */ - truly_illegal_insn, /* 27 */ - system_opcode_insn, /* 28 */ - truly_illegal_insn, /* 29 */ - truly_illegal_insn, /* 30 */ - truly_illegal_insn /* 31 */ + truly_illegal_insn, /* 0 */ + sbi_insn_emu_load_fp, /* 1 */ + truly_illegal_insn, /* 2 */ + misc_mem_opcode_insn, /* 3 */ + sbi_insn_emu_op_imm, /* 4 */ + truly_illegal_insn, /* 5 */ + sbi_insn_emu_op_imm_32, /* 6 */ + truly_illegal_insn, /* 7 */ + truly_illegal_insn, /* 8 */ + sbi_insn_emu_store_fp, /* 9 */ + truly_illegal_insn, /* 10 */ + sbi_illegal_atomic, /* 11 */ + sbi_insn_emu_op, /* 12 */ + truly_illegal_insn, /* 13 */ + sbi_insn_emu_op_32, /* 14 */ + truly_illegal_insn, /* 15 */ + truly_illegal_insn, /* 16 */ + truly_illegal_insn, /* 17 */ + truly_illegal_insn, /* 18 */ + truly_illegal_insn, /* 19 */ + sbi_insn_emu_op_fp, /* 20 */ + sbi_insn_emu_op_v, /* 21 */ + truly_illegal_insn, /* 22 */ + truly_illegal_insn, /* 23 */ + truly_illegal_insn, /* 24 */ + truly_illegal_insn, /* 25 */ + truly_illegal_insn, /* 26 */ + truly_illegal_insn, /* 27 */ + system_opcode_insn, /* 28 */ + truly_illegal_insn, /* 29 */ + truly_illegal_insn, /* 30 */ + truly_illegal_insn /* 31 */ +}; + +static const illegal_insn_func illegal_insn16_table[24] = { + truly_illegal_insn, /* 0 */ + truly_illegal_insn, /* 1 */ + truly_illegal_insn, /* 2 */ + truly_illegal_insn, /* 3 */ + sbi_insn_emu_c_reserved, /* 4 */ + truly_illegal_insn, /* 5 */ + truly_illegal_insn, /* 6 */ + truly_illegal_insn, /* 7 */ + truly_illegal_insn, /* 8 */ + truly_illegal_insn, /* 9 */ + truly_illegal_insn, /* 10 */ + sbi_insn_emu_c_mop, /* 11 */ + sbi_insn_emu_c_misc_alu, /* 12 */ + truly_illegal_insn, /* 13 */ + truly_illegal_insn, /* 14 */ + truly_illegal_insn, /* 15 */ + truly_illegal_insn, /* 16 */ + truly_illegal_insn, /* 17 */ + truly_illegal_insn, /* 18 */ + truly_illegal_insn, /* 19 */ + truly_illegal_insn, /* 20 */ + truly_illegal_insn, /* 21 */ + truly_illegal_insn, /* 22 */ + truly_illegal_insn /* 23 */ }; int sbi_illegal_insn_handler(struct sbi_trap_context *tcntx) { struct sbi_trap_regs *regs = &tcntx->regs; - ulong insn = tcntx->trap.tval; + ulong insn = tcntx->trap.tval; struct sbi_trap_info uptrap; /* - * We only deal with 32-bit (or longer) illegal instructions. If we - * see instruction is zero OR instruction is 16-bit then we fetch and - * check the instruction encoding using unprivilege access. + * We only deal with 32-bit (or longer) illegal instructions directly. + * If we see instruction is zero OR instruction is 16-bit then we fetch + * and check the instruction encoding using unprivilege access. * * The program counter (PC) in RISC-V world is always 2-byte aligned * so handling only 32-bit (or longer) illegal instructions also help @@ -168,7 +233,8 @@ int sbi_illegal_insn_handler(struct sbi_trap_context *tcntx) if (uptrap.cause) return sbi_trap_redirect(regs, &uptrap); if ((insn & 3) != 3) - return truly_illegal_insn(insn, regs); + return illegal_insn16_table[(insn & 3) << 3 | + insn >> 13](insn, regs); } return illegal_insn_table[(insn & 0x7c) >> 2](insn, regs); diff --git a/lib/sbi/sbi_init.c b/lib/sbi/sbi_init.c index 5259064b191..8cf62fae4c9 100644 --- a/lib/sbi/sbi_init.c +++ b/lib/sbi/sbi_init.c @@ -216,8 +216,11 @@ static void wake_coldboot_harts(struct sbi_scratch *scratch) { /* Mark coldboot done */ __smp_store_release(&coldboot_done, 1); + RISCV_FENCE(w, o); } +unsigned long __attribute__((weak)) __stack_chk_guard = 0x95B5FF5A; + static unsigned long entry_count_offset; static unsigned long init_count_offset; @@ -265,11 +268,53 @@ static void __noreturn init_coldboot(struct sbi_scratch *scratch, u32 hartid) */ wake_coldboot_harts(scratch); - rc = sbi_platform_early_init(plat, true); + rc = sbi_hart_init(scratch, true); if (rc) sbi_hart_hang(); - rc = sbi_hart_init(scratch, true); + /* + * Initialize stack guard via Zkr entropy source if Zkr is + * implemented according to device tree. Writing new seed value + * to __stack_chk_guard is safe here because function doesn't + * return and no check against value on entry will be done. + */ + if (sbi_hart_has_extension(scratch, SBI_HART_EXT_ZKR)) { + unsigned long guard_val = 0; + int chunks = sizeof(unsigned long) / sizeof(uint16_t); +#ifndef CONFIG_ZKR_POLL_BUDGET +#define CONFIG_ZKR_POLL_BUDGET 1000 +#endif + unsigned int tries = CONFIG_ZKR_POLL_BUDGET; + bool res = false; + + while (chunks && tries) { + unsigned long seed = csr_swap(CSR_SEED, 0); + unsigned long opst = seed & SEED_OPTS_MASK; + res = false; + + if (opst == SEED_OPTS_DEAD) { + break; + } + if (opst == SEED_OPTS_ES16) { + guard_val = (guard_val << 16) | (seed & SEED_ENTROPY_MASK); + chunks--; + res = true; + /* Successful read doesn't consume a try */ + tries++; + } + + tries--; + continue; + } + if (res) + __stack_chk_guard = guard_val; + } + + rc = sbi_timer_init(scratch, true); + if (rc) + sbi_hart_hang(); + + rc = sbi_platform_early_init(plat, true); if (rc) sbi_hart_hang(); @@ -307,12 +352,6 @@ static void __noreturn init_coldboot(struct sbi_scratch *scratch, u32 hartid) sbi_hart_hang(); } - rc = sbi_timer_init(scratch, true); - if (rc) { - sbi_printf("%s: timer init failed (error %d)\n", __func__, rc); - sbi_hart_hang(); - } - rc = sbi_fwft_init(scratch, true); if (rc) { sbi_printf("%s: fwft init failed (error %d)\n", __func__, rc); @@ -424,11 +463,15 @@ static void __noreturn init_warm_startup(struct sbi_scratch *scratch, if (rc) sbi_hart_hang(); - rc = sbi_platform_early_init(plat, false); + rc = sbi_hart_init(scratch, false); if (rc) sbi_hart_hang(); - rc = sbi_hart_init(scratch, false); + rc = sbi_timer_init(scratch, false); + if (rc) + sbi_hart_hang(); + + rc = sbi_platform_early_init(plat, false); if (rc) sbi_hart_hang(); @@ -452,10 +495,6 @@ static void __noreturn init_warm_startup(struct sbi_scratch *scratch, if (rc) sbi_hart_hang(); - rc = sbi_timer_init(scratch, false); - if (rc) - sbi_hart_hang(); - rc = sbi_fwft_init(scratch, false); if (rc) sbi_hart_hang(); diff --git a/lib/sbi/sbi_insn_emu.c b/lib/sbi/sbi_insn_emu.c new file mode 100644 index 00000000000..ffc3baded5f --- /dev/null +++ b/lib/sbi/sbi_insn_emu.c @@ -0,0 +1,624 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 Benedikt Freisen. + * + * Authors: + * Benedikt Freisen + */ + +#include +#include +#include +#include +#include +#include +#include + +#define MASK_SHAMT32 0x1f +#define MASK_SHAMT (__riscv_xlen - 1) +#define GET_SHAMT32(insn) ((insn >> 20) & MASK_SHAMT32) +#define GET_SHAMT(insn) ((insn >> 20) & MASK_SHAMT) + +#if defined(CONFIG_EMU_ZBB) || defined(CONFIG_EMU_ZBS) +int sbi_insn_emu_op_imm(ulong insn, struct sbi_trap_regs *regs) +{ + ulong rs1_val = GET_RS1(insn, regs); + ulong rd_val; + + switch (insn & INSN_MASK_RTYPE_RD_RS1_RS2) { +#ifdef CONFIG_EMU_ZBS + /* Emulate Zbs immediate instructions */ + case INSN_MATCH_BCLRI: +#if __riscv_xlen == 64 + case INSN_MATCH_BCLRI | 0x02000000: +#endif + rd_val = rs1_val & ~(1ull << GET_SHAMT(insn)); + break; + case INSN_MATCH_BEXTI: +#if __riscv_xlen == 64 + case INSN_MATCH_BEXTI | 0x02000000: +#endif + rd_val = (rs1_val >> GET_SHAMT(insn)) & 1; + break; + case INSN_MATCH_BINVI: +#if __riscv_xlen == 64 + case INSN_MATCH_BINVI | 0x02000000: +#endif + rd_val = rs1_val ^ (1ull << GET_SHAMT(insn)); + break; + case INSN_MATCH_BSETI: +#if __riscv_xlen == 64 + case INSN_MATCH_BSETI | 0x02000000: +#endif + rd_val = rs1_val | (1ull << GET_SHAMT(insn)); + break; +#endif +#ifdef CONFIG_EMU_ZBB + /* Emulate Zbb immediate instructions */ + case INSN_MATCH_RORI: +#if __riscv_xlen == 64 + case INSN_MATCH_RORI | 0x02000000: +#endif + rd_val = rs1_val >> GET_SHAMT(insn) | + rs1_val << (__riscv_xlen - GET_SHAMT(insn)); + break; +#endif + default: + switch (insn & INSN_MASK_ITYPE_RD_RS) { +#ifdef CONFIG_EMU_ZBB + /* Emulate Zbb immediate instructions */ + case INSN_MATCH_CLZ: + for (rd_val = 0; (long)rs1_val >= 0; rd_val++) { + rs1_val <<= 1; + if (rd_val == __riscv_xlen) + break; + } + break; + case INSN_MATCH_CTZ: + for (rd_val = 0; (rs1_val & 1) == 0; rd_val++) { + rs1_val >>= 1; + if (rd_val == __riscv_xlen) + break; + } + break; + case INSN_MATCH_CPOP: + for (rd_val = 0; rs1_val != 0; rs1_val <<= 1) { + if ((long)rs1_val < 0) + rd_val++; + } + break; + case INSN_MATCH_ORC_B: + rd_val = 0; + for (ulong mask = 0xff; mask != 0; mask <<= 8) { + if (rs1_val & mask) + rd_val |= mask; + } + break; +#if __riscv_xlen == 64 + case INSN_MATCH_REV8_RV64: +#else + case INSN_MATCH_REV8_RV32: +#endif + rd_val = 0; + for (int i = sizeof(rs1_val) - 1; i >= 0; i--) { + rd_val <<= 8; + rd_val |= rs1_val & 0xff; + rs1_val >>= 8; + } + break; + case INSN_MATCH_SEXT_B: + rd_val = (long)(s8)rs1_val; + break; + case INSN_MATCH_SEXT_H: + rd_val = (long)(s16)rs1_val; + break; +#endif + default: + return truly_illegal_insn(insn, regs); + } + } + + SET_RD(insn, regs, rd_val); + + regs->mepc += 4; + + return 0; +} +#endif + +#if defined(CONFIG_EMU_ZBA) || defined(CONFIG_EMU_ZBB) || \ + defined(CONFIG_EMU_ZBC) || defined(CONFIG_EMU_ZBS) || \ + defined(CONFIG_EMU_ZICOND) +int sbi_insn_emu_op(ulong insn, struct sbi_trap_regs *regs) +{ + ulong rs1_val = GET_RS1(insn, regs); + ulong rs2_val = GET_RS2(insn, regs); + ulong rd_val; + + switch (insn & INSN_MASK_RTYPE_RD_RS1_RS2) { +#ifdef CONFIG_EMU_ZBS + /* Emulate Zbs register instructions */ + case INSN_MATCH_BCLR: + rd_val = rs1_val & ~(1ull << (rs2_val & MASK_SHAMT)); + break; + case INSN_MATCH_BEXT: + rd_val = (rs1_val >> (rs2_val & MASK_SHAMT)) & 1; + break; + case INSN_MATCH_BINV: + rd_val = rs1_val ^ (1ull << (rs2_val & MASK_SHAMT)); + break; + case INSN_MATCH_BSET: + rd_val = rs1_val | (1ull << (rs2_val & MASK_SHAMT)); + break; +#endif +#ifdef CONFIG_EMU_ZBB + /* Emulate Zbb register instructions */ + case INSN_MATCH_ANDN: + rd_val = rs1_val & ~rs2_val; + break; + case INSN_MATCH_MAX: + rd_val = (long)rs1_val > (long)rs2_val ? rs1_val : rs2_val; + break; + case INSN_MATCH_MAXU: + rd_val = rs1_val > rs2_val ? rs1_val : rs2_val; + break; + case INSN_MATCH_MIN: + rd_val = (long)rs1_val < (long)rs2_val ? rs1_val : rs2_val; + break; + case INSN_MATCH_MINU: + rd_val = rs1_val < rs2_val ? rs1_val : rs2_val; + break; + case INSN_MATCH_ORN: + rd_val = rs1_val | ~rs2_val; + break; + case INSN_MATCH_ROL: + rd_val = rs1_val << (rs2_val & MASK_SHAMT) | + rs1_val >> (__riscv_xlen - (rs2_val & MASK_SHAMT)); + break; + case INSN_MATCH_ROR: + rd_val = rs1_val >> (rs2_val & MASK_SHAMT) | + rs1_val << (__riscv_xlen - (rs2_val & MASK_SHAMT)); + break; + case INSN_MATCH_XNOR: + rd_val = ~(rs1_val ^ rs2_val); + break; +#endif +#ifdef CONFIG_EMU_ZBA + /* Emulate Zba register instructions */ + case INSN_MATCH_SH1ADD: + rd_val = rs2_val + (rs1_val << 1); + break; + case INSN_MATCH_SH2ADD: + rd_val = rs2_val + (rs1_val << 2); + break; + case INSN_MATCH_SH3ADD: + rd_val = rs2_val + (rs1_val << 3); + break; +#endif +#ifdef CONFIG_EMU_ZBC + /* Emulate Zbc instructions */ + case INSN_MATCH_CLMUL: + rd_val = 0; + for (int i = 0; i < __riscv_xlen; i++) { + if ((rs2_val >> i) & 1) + rd_val ^= rs1_val << i; + } + break; + case INSN_MATCH_CLMULH: + rd_val = 0; + for (int i = 1; i <= __riscv_xlen; i++) { + if ((rs2_val >> i) & 1) + rd_val ^= rs1_val >> (__riscv_xlen - i); + } + break; + case INSN_MATCH_CLMULR: + rd_val = 0; + for (int i = 0; i < __riscv_xlen; i++) { + if ((rs2_val >> i) & 1) + rd_val ^= rs1_val >> (__riscv_xlen - i - 1); + } + break; +#endif +#ifdef CONFIG_EMU_ZICOND + /* Emulate Zicond instructions */ + case INSN_MATCH_CZERO_EQZ: + rd_val = rs2_val ? rs1_val : 0; + break; + case INSN_MATCH_CZERO_NEZ: + rd_val = rs2_val ? 0 : rs1_val; + break; +#endif + default: + switch (insn & INSN_MASK_ITYPE_RD_RS) { +#if __riscv_xlen == 32 && defined(CONFIG_EMU_ZBB) + /* Emulate Zbb register instructions */ + case INSN_MATCH_ZEXT_H_RV32: + rd_val = (u16)rs1_val; + break; +#endif + default: + return truly_illegal_insn(insn, regs); + } + } + + SET_RD(insn, regs, rd_val); + + regs->mepc += 4; + + return 0; +} +#endif + +#if __riscv_xlen == 64 && (defined(CONFIG_EMU_ZBA) || defined(CONFIG_EMU_ZBB)) +int sbi_insn_emu_op_32(ulong insn, struct sbi_trap_regs *regs) +{ + ulong rs1_val = GET_RS1(insn, regs); + ulong rs2_val = GET_RS2(insn, regs); + ulong rd_val; + + switch (insn & INSN_MASK_RTYPE_RD_RS1_RS2) { +#ifdef CONFIG_EMU_ZBA + /* Emulate Zba register word instructions */ + case INSN_MATCH_ADD_UW: + rd_val = (rs1_val & 0xfffffffful) + rs2_val; + break; + case INSN_MATCH_SH1ADD_UW: + rd_val = rs2_val + ((rs1_val & 0xfffffffful) << 1); + break; + case INSN_MATCH_SH2ADD_UW: + rd_val = rs2_val + ((rs1_val & 0xfffffffful) << 2); + break; + case INSN_MATCH_SH3ADD_UW: + rd_val = rs2_val + ((rs1_val & 0xfffffffful) << 3); + break; +#endif +#ifdef CONFIG_EMU_ZBB + /* Emulate Zbb register word instructions */ + case INSN_MATCH_ROLW: + rd_val = (s64)(s32)((u32)rs1_val << (rs2_val & MASK_SHAMT32) | + (u32)rs1_val >> + (32 - (rs2_val & MASK_SHAMT32))); + break; + case INSN_MATCH_RORW: + rd_val = (s64)(s32)((u32)rs1_val >> (rs2_val & MASK_SHAMT32) | + (u32)rs1_val + << (32 - (rs2_val & MASK_SHAMT32))); + break; +#endif + default: + switch (insn & INSN_MASK_ITYPE_RD_RS) { +#ifdef CONFIG_EMU_ZBB + /* Emulate Zbb register word instructions */ + case INSN_MATCH_ZEXT_H_RV64: + rd_val = (u16)rs1_val; + break; +#endif + default: + return truly_illegal_insn(insn, regs); + } + } + + SET_RD(insn, regs, rd_val); + + regs->mepc += 4; + + return 0; +} +#endif + +#if __riscv_xlen == 64 && (defined(CONFIG_EMU_ZBA) || defined(CONFIG_EMU_ZBB)) +int sbi_insn_emu_op_imm_32(ulong insn, struct sbi_trap_regs *regs) +{ + ulong rs1_val = GET_RS1(insn, regs); + ulong rd_val; + + switch (insn & INSN_MASK_ITYPE_RD_RS) { +#ifdef CONFIG_EMU_ZBB + /* Emulate Zbb immediate word instructions */ + case INSN_MATCH_CLZW: + for (rd_val = 0; (s32)rs1_val >= 0; rd_val++) { + rs1_val = (long)(s32)((u32)rs1_val << 1); + if (rd_val == 32) + break; + } + break; + case INSN_MATCH_CTZW: + for (rd_val = 0; (rs1_val & 1) == 0; rd_val++) { + rs1_val >>= 1; + if (rd_val == 32) + break; + } + break; + case INSN_MATCH_CPOPW: + for (rd_val = 0; (s32)rs1_val != 0; + rs1_val = (long)(s32)(rs1_val << 1)) { + if ((s32)rs1_val < 0) + rd_val++; + } + break; +#endif + default: + switch (insn & INSN_MASK_SLLI_UW) { +#ifdef CONFIG_EMU_ZBA + /* Emulate Zba immediate word instructions */ + case INSN_MATCH_SLLI_UW: + rd_val = (ulong)(u32)rs1_val << GET_SHAMT(insn); + break; + case INSN_MATCH_RORIW: + rd_val = + (s64)(s32)((u32)rs1_val >> GET_SHAMT32(insn) | + (u32)rs1_val + << (32 - GET_SHAMT32(insn))); + break; +#endif + default: + return truly_illegal_insn(insn, regs); + } + } + + SET_RD(insn, regs, rd_val); + + regs->mepc += 4; + + return 0; +} +#endif + +#ifdef CONFIG_EMU_ZCB +int sbi_insn_emu_c_reserved(ulong insn, struct sbi_trap_regs *regs) +{ + ulong rs1_val = GET_RS1S(insn, regs); + struct sbi_trap_info uptrap; + ulong val; + + switch (insn & INSN_MASK_C_GENERIC_RXS_RXS) { + /* Emulate Zcb additional compressed instructions */ + case INSN_MATCH_C_LBU: + val = sbi_load_u8((void *)rs1_val, &uptrap); + if (uptrap.cause) + return sbi_trap_redirect(regs, &uptrap); + SET_RD2S(insn, regs, val); + break; + case INSN_MATCH_C_LBU + 0x40: + val = sbi_load_u8((void *)rs1_val + 1, &uptrap); + if (uptrap.cause) + return sbi_trap_redirect(regs, &uptrap); + SET_RD2S(insn, regs, val); + break; + case INSN_MATCH_C_LBU + 0x20: + val = sbi_load_u8((void *)rs1_val + 2, &uptrap); + if (uptrap.cause) + return sbi_trap_redirect(regs, &uptrap); + SET_RD2S(insn, regs, val); + break; + case INSN_MATCH_C_LBU + 0x60: + val = sbi_load_u8((void *)rs1_val + 3, &uptrap); + if (uptrap.cause) + return sbi_trap_redirect(regs, &uptrap); + SET_RD2S(insn, regs, val); + break; + case INSN_MATCH_C_LHU: + val = sbi_load_u16((void *)rs1_val, &uptrap); + if (uptrap.cause) + return sbi_trap_redirect(regs, &uptrap); + SET_RD2S(insn, regs, val); + break; + case INSN_MATCH_C_LHU + 0x20: + val = sbi_load_u16((void *)rs1_val + 2, &uptrap); + if (uptrap.cause) + return sbi_trap_redirect(regs, &uptrap); + SET_RD2S(insn, regs, val); + break; + case INSN_MATCH_C_LH: + val = sbi_load_s16((void *)rs1_val, &uptrap); + if (uptrap.cause) + return sbi_trap_redirect(regs, &uptrap); + SET_RD2S(insn, regs, val); + break; + case INSN_MATCH_C_LH + 0x20: + val = sbi_load_s16((void *)rs1_val + 2, &uptrap); + if (uptrap.cause) + return sbi_trap_redirect(regs, &uptrap); + SET_RD2S(insn, regs, val); + break; + case INSN_MATCH_C_SB: + val = GET_RS2S(insn, regs); + sbi_store_u8((void *)rs1_val, val, &uptrap); + if (uptrap.cause) + return sbi_trap_redirect(regs, &uptrap); + break; + case INSN_MATCH_C_SB + 0x40: + val = GET_RS2S(insn, regs); + sbi_store_u8((void *)rs1_val + 1, val, &uptrap); + if (uptrap.cause) + return sbi_trap_redirect(regs, &uptrap); + break; + case INSN_MATCH_C_SB + 0x20: + val = GET_RS2S(insn, regs); + sbi_store_u8((void *)rs1_val + 2, val, &uptrap); + if (uptrap.cause) + return sbi_trap_redirect(regs, &uptrap); + break; + case INSN_MATCH_C_SB + 0x60: + val = GET_RS2S(insn, regs); + sbi_store_u8((void *)rs1_val + 3, val, &uptrap); + if (uptrap.cause) + return sbi_trap_redirect(regs, &uptrap); + break; + case INSN_MATCH_C_SH: + val = GET_RS2S(insn, regs); + sbi_store_u16((void *)rs1_val, val, &uptrap); + if (uptrap.cause) + return sbi_trap_redirect(regs, &uptrap); + break; + case INSN_MATCH_C_SH + 0x20: + val = GET_RS2S(insn, regs); + sbi_store_u16((void *)rs1_val + 2, val, &uptrap); + if (uptrap.cause) + return sbi_trap_redirect(regs, &uptrap); + break; + default: + return truly_illegal_insn(insn, regs); + } + + regs->mepc += 2; + + return 0; +} +#endif + +#ifdef CONFIG_EMU_ZCMOP +int sbi_insn_emu_c_mop(ulong insn, struct sbi_trap_regs *regs) +{ + /* Emulate Zcmop compressed may-be operations */ + if ((insn & INSN_MASK_C_MOP_N) == INSN_MATCH_C_MOP_N) { + /* do nothing */ + regs->mepc += 2; + return 0; + } + + return truly_illegal_insn(insn, regs); +} +#endif + +#ifdef CONFIG_EMU_ZCB +int sbi_insn_emu_c_misc_alu(ulong insn, struct sbi_trap_regs *regs) +{ + ulong rs1_val = GET_RS1S(insn, regs); + + switch (insn & INSN_MASK_C_GENERIC_RXS) { + /* Emulate Zcb additional compressed instructions */ + case INSN_MATCH_C_ZEXT_B: + SET_RD1S(insn, regs, (u8)rs1_val); + break; + case INSN_MATCH_C_SEXT_B: + SET_RD1S(insn, regs, (long)(s8)rs1_val); + break; + case INSN_MATCH_C_ZEXT_H: + SET_RD1S(insn, regs, (u16)rs1_val); + break; + case INSN_MATCH_C_SEXT_H: + SET_RD1S(insn, regs, (long)(s16)rs1_val); + break; +#if __riscv_xlen == 64 + case INSN_MATCH_C_ZEXT_W: + SET_RD1S(insn, regs, (u32)rs1_val); + break; +#endif + case INSN_MATCH_C_NOT: + SET_RD1S(insn, regs, ~rs1_val); + break; + default: + switch (insn & INSN_MASK_C_GENERIC_RXS_RXS) { + case INSN_MATCH_C_MUL: + SET_RD1S(insn, regs, + (long)rs1_val * (long)GET_RS2S(insn, regs)); + break; + default: + return truly_illegal_insn(insn, regs); + } + } + + regs->mepc += 2; + + return 0; +} +#endif + +#if defined(CONFIG_EMU_ZICBOM) || defined(CONFIG_EMU_ZICBOZ) +static ulong read_senvcfg_or_emu(void) +{ + struct sbi_scratch *scratch = sbi_scratch_thishart_ptr(); + + /* Return actual CSR value or emulation */ + if (sbi_hart_has_csr(scratch, SBI_HART_CSR_SENVCFG)) + return csr_read(CSR_SENVCFG); + else + /* For the time being, assume that the menvcfg value + * for the logical AND is a suitable constant */ + return scratch->sw_senvcfg & + (ENVCFG_CBZE | ENVCFG_CBCFE | ENVCFG_CBIE); +} + +static ulong read_menvcfg_or_emu(void) +{ + struct sbi_scratch *scratch = sbi_scratch_thishart_ptr(); + + /* Return actual CSR value or emulation */ + if (sbi_hart_has_csr(scratch, SBI_HART_CSR_MENVCFG)) + return csr_read(CSR_MENVCFG); + else + /* For the time being, return a suitable constant */ + return ENVCFG_CBZE | ENVCFG_CBCFE | ENVCFG_CBIE; +} + +int sbi_insn_emu_zicbom_zicboz(ulong insn, struct sbi_trap_regs *regs) +{ + /* NOTE: Errata workarounds for fence instructions are handled in + * misc_mem_opcode_insn. */ + + /* Emulate Zicbom and Zicboz */ + switch (insn & INSN_MASK_CBO) { + case INSN_MATCH_CBO_ZERO: { + /* Check whether the instruction was even allowed */ + ulong prev_mode = sbi_mstatus_prev_mode(regs->mstatus); + if ((prev_mode == PRV_U && + !(read_senvcfg_or_emu() & ENVCFG_CBZE)) || + (prev_mode == PRV_S && + !(read_menvcfg_or_emu() & ENVCFG_CBZE))) + return truly_illegal_insn(insn, regs); + + u32 *addr = + (u32 *)(GET_RS1S(insn, regs) & 0xffffffffffffffc0ull); + struct sbi_trap_info uptrap; + /* Zero the 64 byte block */ + for (int i = 0; i < 16; i++) { + sbi_store_u32(addr + i, 0, &uptrap); + if (uptrap.cause) + return sbi_trap_redirect(regs, &uptrap); + } + break; + } + case INSN_MATCH_CBO_CLEAN: + case INSN_MATCH_CBO_FLUSH: { + /* Check whether the instruction was even allowed */ + ulong prev_mode = sbi_mstatus_prev_mode(regs->mstatus); + if ((prev_mode == PRV_U && + !(read_senvcfg_or_emu() & ENVCFG_CBCFE)) || + (prev_mode == PRV_S && + !(read_menvcfg_or_emu() & ENVCFG_CBCFE))) + return truly_illegal_insn(insn, regs); + + /* Simply flush the entire cache */ + fdt_cmo_flush_all(); + /* NOTE: Depending on cache architecture and use case, + * fdt_cmo_private_flc_flush_all might be sufficient. */ + + break; + } + case INSN_MATCH_CBO_INVAL: { + /* Check whether the instruction was even allowed */ + ulong prev_mode = sbi_mstatus_prev_mode(regs->mstatus); + if ((prev_mode == PRV_U && + !(read_senvcfg_or_emu() & ENVCFG_CBIE)) || + (prev_mode == PRV_S && + !(read_menvcfg_or_emu() & ENVCFG_CBIE))) + return truly_illegal_insn(insn, regs); + + /* Simply flush the entire cache */ + fdt_cmo_flush_all(); + /* NOTE: Depending on cache architecture and use case, + * fdt_cmo_private_flc_flush_all might be sufficient. */ + + break; + } + default: + return truly_illegal_insn(insn, regs); + } + + regs->mepc += 4; + + return 0; +} +#endif diff --git a/lib/sbi/sbi_insn_emu.conf b/lib/sbi/sbi_insn_emu.conf new file mode 100644 index 00000000000..bd35aa4afd2 --- /dev/null +++ b/lib/sbi/sbi_insn_emu.conf @@ -0,0 +1,175 @@ +# SPDX-License-Identifier: BSD-2-Clause + +if SBI_ISA_EXT_EMU + +config EMU_ALL + bool "All supported extensions" + default y + help + Emulates all supported ISA extensions, i.e RVA23U64 and beyond. + select EMU_RVA23 + select EMU_ZBC + +config EMU_RVA23 + bool "All supported mandatory extensions of RVA23U64" + default n + help + Emulates some ISA extensions required by the RVA23U64 profile, but not e.g. Zvfhmin. + This feature requires RVV 1.0 vector hardware. + select EMU_RVB23 + select EMU_ZVBB + select EMU_SUPM + +config EMU_RVB23 + bool "All supported mandatory extensions of RVB23U64" + default n + help + Emulates all ISA extensions required by the RVA22U64 profile on RVA20 CPUs. + select EMU_RVA22 + select EMU_ZICOND + select EMU_ZIMOP + select EMU_ZAWRS + select EMU_ZFA + select EMU_ZCB + select EMU_ZCMOP + +config EMU_RVA22 + bool "All supported mandatory extensions of RVA22U64" + default n + help + Emulates all ISA extensions required by the RVA22U64 profile on RVA20 CPUs. + select EMU_ZICBOM + select EMU_ZICBOZ + select EMU_ZFHMIN + select EMU_ZBA + select EMU_ZBB + select EMU_ZBS + +config EMU_ZICBOM + bool "Zicbom" + default n + help + Cache-block management instructions. + Zicbom emulation depends on OpenSBI's FDT CMO helper library. + depends on SBI_ISA_EXT_EMU + select FDT_CACHE + +config EMU_ZICBOZ + bool "Zicboz" + default n + help + Cache-block zero instructions. + depends on SBI_ISA_EXT_EMU + +config EMU_ZICOND + bool "Zicond" + default n + help + Integer conditional operations. + depends on SBI_ISA_EXT_EMU + +config EMU_ZIMOP + bool "Zimop" + default n + help + May-be-operations. + depends on SBI_ISA_EXT_EMU + +config EMU_ZAWRS + bool "Zawrs" + default n + help + Wait-on-reservation-set instructucions. + Loops containing WRS.STO or WRS.NTO will degrade to busy waiting. + depends on SBI_ISA_EXT_EMU + +config EMU_ZFA + bool "Zfa" + default n + help + Additional floating-point instructions. + depends on SBI_ISA_EXT_EMU + +config EMU_ZFHMIN + bool "Zfhmin" + default n + help + Half-precision floating-point. + depends on SBI_ISA_EXT_EMU + +config EMU_ZCB + bool "Zcb" + default n + help + Additional compressed instructions. + depends on SBI_ISA_EXT_EMU + +config EMU_ZCMOP + bool "Zcmop" + default n + help + Compressed may-be-operations. + depends on SBI_ISA_EXT_EMU + +config EMU_ZBA + bool "Zba" + default n + help + Address generation. + depends on SBI_ISA_EXT_EMU + +config EMU_ZBB + bool "Zbb" + default n + help + Basic bit-manipulation. + depends on SBI_ISA_EXT_EMU + +config EMU_ZBC + bool "Zbc" + default n + help + Carry-less multiplication. + depends on SBI_ISA_EXT_EMU + +config EMU_ZBS + bool "Zbs" + default n + help + Single-bit instructions. + depends on SBI_ISA_EXT_EMU + +config EMU_ZVBB + bool "Zvbb (needs RVV 1.0 and Zbb)" + default n + help + Vector basic bit-manipulation instructions. + Zvbb emulation strictly requires RVV 1.0 hardware. + It also uses (native or emulated) Zbb instructions. + depends on SBI_ISA_EXT_EMU + +if EMU_ZVBB +choice EMU_ZVBB_VLMAX_BYTES + prompt "Reserve buffers for VLEN of at most:" + default EMU_ZVBB_VLEN_256 + config EMU_ZVBB_VLEN_128 + bool "128 bits" + config EMU_ZVBB_VLEN_256 + bool "256 bits" + config EMU_ZVBB_VLEN_512 + bool "512 bits" + config EMU_ZVBB_VLEN_1024 + bool "1024 bits" +endchoice +endif + +config EMU_SUPM + bool "Supm" + default n + help + Pointer masking, with the execution environment providing a means to + select PMLEN=0 and PMLEN=7 at minimum. + Pointer masking needs to be set up via SBI and relies on page faults. + depends on SBI_ISA_EXT_EMU + +endif diff --git a/lib/sbi/sbi_insn_emu_fp.c b/lib/sbi/sbi_insn_emu_fp.c new file mode 100644 index 00000000000..48ee498db33 --- /dev/null +++ b/lib/sbi/sbi_insn_emu_fp.c @@ -0,0 +1,974 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 Benedikt Freisen. + * + * Authors: + * Benedikt Freisen + */ + +#include +#include +#include +#include +#include + +#ifdef CONFIG_EMU_ZFHMIN +int sbi_insn_emu_load_fp(ulong insn, struct sbi_trap_regs *regs) +{ + struct sbi_trap_context *tcntx = + container_of(regs, struct sbi_trap_context, regs); + + /* If floating point is available and insn is FLH, + * simply use the misaligned load handler */ + if ((regs->mstatus & MSTATUS_FS) != 0 && + (sbi_mstatus_prev_mode(regs->mstatus) != PRV_U || + (csr_read(CSR_SSTATUS) & SSTATUS_FS) != 0) && + (insn & INSN_MASK_FLH) == INSN_MATCH_FLH) { + tcntx->trap.cause = CAUSE_MISALIGNED_LOAD; + tcntx->trap.tval = GET_RS1(insn, regs) + IMM_I(insn); + return sbi_misaligned_load_handler(tcntx); + } + + return truly_illegal_insn(insn, regs); +} + +int sbi_insn_emu_store_fp(ulong insn, struct sbi_trap_regs *regs) +{ + struct sbi_trap_context *tcntx = + container_of(regs, struct sbi_trap_context, regs); + + /* If floating point is available and insn is FSH, + * simply use the misaligned store handler */ + if ((regs->mstatus & MSTATUS_FS) != 0 && + (sbi_mstatus_prev_mode(regs->mstatus) != PRV_U || + (csr_read(CSR_SSTATUS) & SSTATUS_FS) != 0) && + (insn & INSN_MASK_FSH) == INSN_MATCH_FSH) { + tcntx->trap.cause = CAUSE_MISALIGNED_LOAD; + tcntx->trap.tval = GET_RS1(insn, regs) + IMM_S(insn); + return sbi_misaligned_store_handler(tcntx); + } + + return truly_illegal_insn(insn, regs); +} + +#define RM_FIELD_RNE 0 +#define RM_FIELD_RTZ 1 +#define RM_FIELD_RDN 2 +#define RM_FIELD_RUP 3 +#define RM_FIELD_RMM 4 +#define RM_FIELD_DYN 7 + +#define FFLAG_INEXACT 0x01 +#define FFLAG_UNDERFLOW 0x02 +#define FFLAG_OVERFLOW 0x04 +#define FFLAG_DIVIDE_BY_ZERO 0x08 +#define FFLAG_INVALID_OPERATION 0x10 + +static u32 convert_f16_to_f32(u16 val, u32 *fcsr) +{ + /* special case: +/- zero */ + if ((val & 0x7fff) == 0) + return (u32)val << 16; + /* special case: +/- infinity */ + if ((val & 0x7fff) == 0x7c00) + return ((s32)(s16)val << 13) | 0x7f800000; + /* special case: NaN => output canonical NaN */ + if ((val & 0x7c00) == 0x7c00) { + /* handle signaling NaN */ + if ((val & 0x0200) == 0) + *fcsr |= FFLAG_INVALID_OPERATION; + /* always return canonical NaN */ + return 0x7fc00000; + } + /* generic case or denormalized */ + u32 result = (((s32)(s16)val << 13) & 0x8fffffff) + 0x38000000; + /* normalize denormalized */ + if ((val & 0x7c00) == 0) { + u32 signexp = result & 0xff800000; + result &= 0x007fffff; + while (!(result & 0x00800000)) { + signexp -= 0x00800000; + result <<= 1; + } + result = (signexp + 0x00800000) | (result & 0x007fffff); + } + return result; +} + +static u64 convert_f16_to_f64(u16 val, u32 *fcsr) +{ + /* special case: +/- zero */ + if ((val & 0x7fff) == 0) + return (u64)val << 48; + /* special case: +/- infinity */ + if ((val & 0x7fff) == 0x7c00) + return ((s64)(s16)val << 42) | 0x7ff0000000000000; + /* special case: NaN => output canonical NaN */ + if ((val & 0x7c00) == 0x7c00) { + /* handle signaling NaN */ + if ((val & 0x0200) == 0) + *fcsr |= FFLAG_INVALID_OPERATION; + /* always return canonical NaN */ + return 0x7ff8000000000000; + } + /* generic case or denormalized */ + u64 result = (((s64)(s16)val << 42) & 0x81ffffffffffffff) + + 0x3f00000000000000; + /* normalize denormalized */ + if ((val & 0x7c00) == 0) { + u64 signexp = result & 0xfff0000000000000; + result &= 0x000fffffffffffff; + while (!(result & 0x0010000000000000)) { + signexp -= 0x0010000000000000; + result <<= 1; + } + result = (signexp + 0x0010000000000000) | + (result & 0x000fffffffffffff); + } + return result; +} + +static u16 convert_f32_to_f16(u32 val, u32 *fcsr, int rm) +{ + /* rounding bias to be added below what will be the LSB: + * sign, future LSB, rounding mode */ + static const u32 rm_bias[2][2][5] = { + { { 0x0fffffff, 0, 0, 0x1fffffff, 0x10000000 }, + { 0x10000000, 0, 0, 0x1fffffff, 0x10000000 } }, + { { 0x0fffffff, 0, 0x1fffffff, 0, 0x10000000 }, + { 0x10000000, 0, 0x1fffffff, 0, 0x10000000 } } + }; + + /* values above the threshold (with masked sign) become infinity, + * unless the rounding mode says otherwise. + * sign, rounding mode */ + static const u32 inf_threshold[2][5] = { + { 0x477fefff, 0x477fffff, 0x477fffff, 0x477fe000, 0x477fefff }, + { 0x477fefff, 0x477fffff, 0x477fe000, 0x477fffff, 0x477fefff } + }; + + /* the "infinity" value to be used. + * sign, rounding mode */ + static const u16 inf_or_max[2][5] = { + { 0x7c00, 0x7bff, 0x7bff, 0x7c00, 0x7c00 }, + { 0xfc00, 0xfbff, 0xfc00, 0xfbff, 0xfc00 } + }; + + /* the "zero" value to be used. + * sign, rounding mode */ + static const u16 zero_or_one[2][5] = { + { 0x0000, 0x0000, 0x0000, 0x0001, 0x0000 }, + { 0x8000, 0x8000, 0x8001, 0x8000, 0x8000 } + }; + + /* values below the threshold (with masked sign) become denormalized. + * sign, rounding mode */ + static const u32 subnorm_threshold[2][5] = { + { 0x387fefff, 0x387fffff, 0x387fffff, 0x387fe000, 0x387fefff }, + { 0x387fefff, 0x387fffff, 0x387fe000, 0x387fffff, 0x387fefff } + }; + + int sign = val >> 31; + + /* special case: +/- zero */ + if ((val & 0x7fffffff) == 0) + return val >> 16; + /* special case: +/- infinity */ + if ((val & 0x7fffffff) == 0x7f800000) + return (val >> 16) & 0xfc00; + /* special case for NaN */ + if ((val & 0x7f800000) == 0x7f800000) { + /* handle signaling NaN */ + if ((val & 0x00400000) == 0) + *fcsr |= FFLAG_INVALID_OPERATION; + /* always return canonical NaN */ + return 0x7e00; + } + /* replace too small numbers with +/- 0 or +/- 1 */ + if ((val & 0x7f800000) < 0x31800000) { + *fcsr |= FFLAG_UNDERFLOW | FFLAG_INEXACT; + return zero_or_one[sign][rm]; + } + /* replace too big numbers with +/- infinity */ + if ((val & 0x7fffffff) > inf_threshold[sign][rm]) { + *fcsr |= FFLAG_OVERFLOW | FFLAG_INEXACT; + return inf_or_max[sign][rm]; + } + /* handle numbers that become denormalized */ + if ((val & 0x7fffffff) <= subnorm_threshold[sign][rm]) { + int shiftval = 113 - ((val >> 23) & 0xff); + u32 mant = (val & 0x007fffff) | 0x00800000; + /* set inexact flag if needed */ + if (mant & (0x07ffffff >> (14 - shiftval))) + *fcsr |= FFLAG_UNDERFLOW | FFLAG_INEXACT; + return (sign << 15) | + ((mant + + (rm_bias[sign][(mant >> (13 + shiftval)) & 1][rm] >> + (16 - shiftval))) >> + (13 + shiftval)); + } + /* no special case */ + if (val & 0x1fff) + *fcsr |= FFLAG_INEXACT; + return (sign << 15) | ((((val & 0x7f800000) - 0x38000000) >> 13) + + (((val & 0x007fffff) + + (rm_bias[sign][(val >> 13) & 1][rm] >> 16)) >> + 13)); +} + +static u16 convert_f64_to_f16(u64 val, u32 *fcsr, int rm) +{ + /* rounding bias to be added below what will be the LSB: + * sign, future LSB, rounding mode */ + static const u64 rm_bias[2][2][5] = { + { { 0x1ffffffffffffff, 0, 0, 0x3ffffffffffffff, + 0x200000000000000 }, + { 0x200000000000000, 0, 0, 0x3ffffffffffffff, + 0x200000000000000 } }, + { { 0x1ffffffffffffff, 0, 0x3ffffffffffffff, 0, + 0x200000000000000 }, + { 0x200000000000000, 0, 0x3ffffffffffffff, 0, + 0x200000000000000 } } + }; + + /* values above the threshold (with masked sign) become infinity, + * unless the rounding mode says otherwise. + * sign, rounding mode */ + static const u64 inf_threshold[2][5] = { + { 0x40effdffffffffff, 0x40efffffffffffff, 0x40efffffffffffff, + 0x40effc0000000000, 0x40effdffffffffff }, + { 0x40effdffffffffff, 0x40efffffffffffff, 0x40effc0000000000, + 0x40efffffffffffff, 0x40effdffffffffff } + }; + + /* the "infinity" value to be used. + * sign, rounding mode */ + static const u16 inf_or_max[2][5] = { + { 0x7c00, 0x7bff, 0x7bff, 0x7c00, 0x7c00 }, + { 0xfc00, 0xfbff, 0xfc00, 0xfbff, 0xfc00 } + }; + + /* the "zero" value to be used. + * sign, rounding mode */ + static const u16 zero_or_one[2][5] = { + { 0x0000, 0x0000, 0x0000, 0x0001, 0x0000 }, + { 0x8000, 0x8000, 0x8001, 0x8000, 0x8000 } + }; + + /* values below the threshold (with masked sign) become denormalized. + * sign, rounding mode */ + static const u64 subnorm_threshold[2][5] = { + { 0x3f0ffdffffffffff, 0x3f0fffffffffffff, 0x3f0fffffffffffff, + 0x3f0ffc0000000000, 0x3f0ffdffffffffff }, + { 0x3f0ffdffffffffff, 0x3f0fffffffffffff, 0x3f0ffc0000000000, + 0x3f0fffffffffffff, 0x3f0ffdffffffffff } + }; + + int sign = val >> 63; + + /* special case: +/- zero */ + if ((val & 0x7fffffffffffffff) == 0) + return val >> 48; + /* special case: +/- infinity */ + if ((val & 0x7fffffffffffffff) == 0x7ff0000000000000) + return (val >> 48) & 0xfc00; + /* special case for NaN */ + if ((val & 0x7ff0000000000000) == 0x7ff0000000000000) { + /* handle signaling NaN */ + if ((val & 0x0008000000000000) == 0) + *fcsr |= FFLAG_INVALID_OPERATION; + /* always return canonical NaN */ + return 0x7e00; + } + /* replace too small numbers with +/- 0 or +/- 1 */ + if ((val & 0x7ff0000000000000) < 0x3e30000000000000) { + *fcsr |= FFLAG_UNDERFLOW | FFLAG_INEXACT; + return zero_or_one[sign][rm]; + } + /* replace too big numbers with +/- infinity */ + if ((val & 0x7fffffffffffffff) > inf_threshold[sign][rm]) { + *fcsr |= FFLAG_OVERFLOW | FFLAG_INEXACT; + return inf_or_max[sign][rm]; + } + /* handle numbers that become denormalized */ + if ((val & 0x7fffffffffffffff) <= subnorm_threshold[sign][rm]) { + unsigned shiftval = 1009 - ((val >> 52) & 0x7ff); + u64 mant = (val & 0x000fffffffffffff) | 0x0010000000000000; + /* set inexact flag if needed */ + if (mant & (0x00ffffffffffffff >> (14 - shiftval))) + *fcsr |= FFLAG_UNDERFLOW | FFLAG_INEXACT; + return (sign << 15) | + ((mant + + (rm_bias[sign][(mant >> (42 + shiftval)) & 1][rm] >> + (16 - shiftval))) >> + (42 + shiftval)); + } + /* no special case */ + if (val & 0x3ffffffffff) + *fcsr |= FFLAG_INEXACT; + return (sign << 15) | + ((((val & 0x7ff0000000000000) - 0x3f00000000000000) >> 42) + + (((val & 0x000fffffffffffff) + + (rm_bias[sign][(val >> 42) & 1][rm] >> 16)) >> + 42)); +} +#endif + +#ifdef CONFIG_EMU_ZFA +static const u16 f16_imm_lut[32] = { + 0xbc00, 0x0400, 0x0100, 0x0200, 0x1c00, 0x2000, 0x2c00, 0x3000, + 0x3400, 0x3500, 0x3600, 0x3700, 0x3800, 0x3900, 0x3a00, 0x3b00, + 0x3c00, 0x3d00, 0x3e00, 0x3f00, 0x4000, 0x4100, 0x4200, 0x4400, + 0x4800, 0x4c00, 0x5800, 0x5c00, 0x7800, 0x7c00, 0x7c00, 0x7e00 +}; + +static const u32 f32_imm_lut[32] = { + 0xbf800000, 0x00800000, 0x37800000, 0x38000000, 0x3b800000, 0x3c000000, + 0x3d800000, 0x3e000000, 0x3e800000, 0x3ea00000, 0x3ec00000, 0x3ee00000, + 0x3f000000, 0x3f200000, 0x3f400000, 0x3f600000, 0x3f800000, 0x3fa00000, + 0x3fc00000, 0x3fe00000, 0x40000000, 0x40200000, 0x40400000, 0x40800000, + 0x41000000, 0x41800000, 0x43000000, 0x43800000, 0x47000000, 0x47800000, + 0x7f800000, 0x7fc00000 +}; + +static const u64 f64_imm_lut[32] = { + 0xbc00000000000000, 0x0010000000000000, 0x3ef0000000000000, + 0x3f00000000000000, 0x3f70000000000000, 0x3f80000000000000, + 0x3fb0000000000000, 0x3fc0000000000000, 0x3fd0000000000000, + 0x3fd4000000000000, 0x3fd8000000000000, 0x3fdc000000000000, + 0x3fe0000000000000, 0x3fe4000000000000, 0x3fe8000000000000, + 0x3fec000000000000, 0x3ff0000000000000, 0x3ff4000000000000, + 0x3ff8000000000000, 0x3ffc000000000000, 0x4000000000000000, + 0x4004000000000000, 0x4008000000000000, 0x4010000000000000, + 0x4020000000000000, 0x4030000000000000, 0x4060000000000000, + 0x4070000000000000, 0x40e0000000000000, 0x40f0000000000000, + 0x7ff0000000000000, 0x7ff8000000000000 +}; + +static u32 round_f32(u32 val, u32 *fcsr, int rm, bool set_nx) +{ + /* rounding bias to be added below what will be the LSB: + * sign, future LSB, rounding mode */ + static const u32 rm_bias[2][2][5] = { + { { 0x3fffff, 0x000000, 0x000000, 0x7fffff, 0x400000 }, + { 0x400000, 0x000000, 0x000000, 0x7fffff, 0x400000 } }, + { { 0x3fffff, 0x000000, 0x7fffff, 0x000000, 0x400000 }, + { 0x400000, 0x000000, 0x7fffff, 0x000000, 0x400000 } } + }; + + /* values >= this (with masked sign) become at least +/- 1 + * sign, rounding mode */ + static const u32 one_threshold[2][5] = { + { 0x3effffff, 0x3f800000, 0x3f800000, 1, 0x3f000000 }, + { 0x3effffff, 0x3f800000, 1, 0x3f800000, 0x3f000000 } + }; + + /* handle +/- zero */ + if ((val & 0x7fffffff) == 0) + return val; + /* handle NaNs */ + if ((val & 0x7fffffff) > 0x7f800000) { + /* check for signaling NaN */ + if (!(val & 0x00400000)) + *fcsr |= FFLAG_INVALID_OPERATION; + /* return canonical NaN */ + return 0x7fc00000; + } + /* handle values too big to have a fractional part */ + if ((val & 0x7f800000) >= 0x4b000000) + return val; + /* handle values that can only yield 0 or 1 */ + if ((val & 0x7fffffff) < 0x3f800000) { + if (set_nx) + *fcsr |= FFLAG_INEXACT; + if ((val & 0x7f800000) >= one_threshold[val >> 31][rm]) + return (val & 0x80000000) | 0x3f800000; + return val & 0x80000000; + } + /* handle all other values */ + unsigned sh = ((val & 0x7f800000) >> 23) - 127; + u32 new_val = (val & 0x7fffff) | 0x800000; + new_val += rm_bias[val >> 31][(new_val >> (23 - sh)) & 1][rm] >> sh; + new_val &= ~(0x7fffff >> sh); + if (new_val >= 0x1000000) { + new_val >>= 1; + new_val &= 0x7fffff; + new_val |= (val & 0x7f800000) + 0x00800000; + } else { + new_val &= 0x7fffff; + new_val |= val & 0x7f800000; + } + new_val |= val & 0x80000000; + if (set_nx && new_val != val) + *fcsr |= FFLAG_INEXACT; + return new_val; +} + +static u64 round_f64(u64 val, u32 *fcsr, int rm, bool set_nx) +{ + /* rounding bias to be added below what will be the LSB: + * sign, future LSB, rounding mode */ + static const u64 rm_bias[2][2][5] = { + { { 0x7ffffffffffff, 0, 0, 0xfffffffffffff, 0x8000000000000 }, + { 0x8000000000000, 0, 0, 0xfffffffffffff, 0x8000000000000 } }, + { { 0x7ffffffffffff, 0, 0xfffffffffffff, 0, 0x8000000000000 }, + { 0x8000000000000, 0, 0xfffffffffffff, 0, 0x8000000000000 } } + }; + + /* values >= this (with masked sign) become at least +/- 1 + * sign, rounding mode */ + static const u64 one_threshold[2][5] = { + { 0x3fdfffffffffffff, 0x3ff0000000000000, 0x3ff0000000000000, 1, + 0x3fe0000000000000 }, + { 0x3fdfffffffffffff, 0x3ff0000000000000, 1, 0x3ff0000000000000, + 0x3fe0000000000000 } + }; + + /* handle +/- zero */ + if ((val & 0x7fffffffffffffff) == 0) + return val; + /* handle NaNs */ + if ((val & 0x7fffffffffffffff) > 0x7ff0000000000000) { + /* check for signaling NaN */ + if (!(val & 0x0008000000000000)) + *fcsr |= FFLAG_INVALID_OPERATION; + /* return canonical NaN */ + return 0x7ff8000000000000; + } + /* handle values too big to have a fractional part */ + if ((val & 0x7ff0000000000000) >= 0x4330000000000000) + return val; + /* handle values that can only yield 0 or 1 */ + if ((val & 0x7fffffffffffffff) < 0x3ff0000000000000) { + if (set_nx) + *fcsr |= FFLAG_INEXACT; + if ((val & 0x7ff0000000000000) >= one_threshold[val >> 63][rm]) + return (val & 0x8000000000000000) | 0x3ff0000000000000; + return val & 0x8000000000000000; + } + /* handle all other values */ + unsigned sh = ((val & 0x7ff0000000000000) >> 52) - 1023; + u64 new_val = (val & 0x000fffffffffffff) | 0x0010000000000000; + new_val += rm_bias[val >> 63][(new_val >> (52 - sh)) & 1][rm] >> sh; + new_val &= ~(0x000fffffffffffff >> sh); + if (new_val >= 0x0020000000000000) { + new_val >>= 1; + new_val &= 0x000fffffffffffff; + new_val |= (val & 0x7ff0000000000000) + 0x0010000000000000; + } else { + new_val &= 0x000fffffffffffff; + new_val |= val & 0x7ff0000000000000; + } + new_val |= val & 0x8000000000000000; + if (set_nx && new_val != val) + *fcsr |= FFLAG_INEXACT; + return new_val; +} + +static u16 round_f16(u16 val, u32 *fcsr, int rm, bool set_nx) +{ + /* rounding bias to be added below what will be the LSB: + * sign, future LSB, rounding mode */ + static const u16 rm_bias[2][2][5] = { + { { 0x1ff, 0x000, 0x000, 0x3ff, 0x200 }, + { 0x200, 0x000, 0x000, 0x3ff, 0x200 } }, + { { 0x1ff, 0x000, 0x3ff, 0x000, 0x200 }, + { 0x200, 0x000, 0x3ff, 0x000, 0x200 } } + }; + + /* values >= this (with masked sign) become at least +/- 1 + * sign, rounding mode */ + static const u16 one_threshold[2][5] = { + { 0x37ff, 0x3c00, 0x3c00, 0x0001, 0x3800 }, + { 0x37ff, 0x3c00, 0x0001, 0x3c00, 0x3800 } + }; + + /* handle +/- zero */ + if ((val & 0x7fff) == 0) + return val; + /* handle NaNs */ + if ((val & 0x7fff) > 0x7c00) { + /* check for signaling NaN */ + if (!(val & 0x0200)) + *fcsr |= FFLAG_INVALID_OPERATION; + /* return canonical NaN */ + return 0x7e00; + } + /* handle values too big to have a fractional part */ + if ((val & 0x7c00) >= 0x6400) + return val; + /* handle values that can only yield 0 or 1 */ + if ((val & 0x7fff) < 0x3c00) { + if (set_nx) + *fcsr |= FFLAG_INEXACT; + if ((val & 0x7fff) >= one_threshold[val >> 15][rm]) + return (val & 0x8000) | 0x3c00; + return val & 0x8000; + } + /* handle all other values */ + unsigned sh = ((val & 0x7c00) >> 10) - 15; + u16 new_val = (val & 0x3ff) | 0x400; + new_val += rm_bias[val >> 15][(new_val >> (10 - sh)) & 1][rm] >> sh; + new_val &= ~(0x3ff >> sh); + if (new_val >= 0x800) { + new_val >>= 1; + new_val &= 0x3ff; + new_val |= (val & 0x7c00) + 0x0400; + } else { + new_val &= 0x3ff; + new_val |= val & 0x7c00; + } + new_val |= val & 0x8000; + if (set_nx && new_val != val) + *fcsr |= FFLAG_INEXACT; + return new_val; +} + +static s32 fcvtmod_f64(u64 val, u32 *fcsr) +{ + bool sign = val >> 63; + val &= 0x7fffffffffffffff; + + /* handle +/- zero */ + if (val == 0) + return 0; + + int exp = ((val >> 52) & 0x7ff) - 1023; + /* handle values that become zero */ + if (exp < 0) { + *fcsr |= FFLAG_INEXACT; + return 0; + } + /* handle all bigger values */ + /* handle overflow */ + if (exp > 31) + *fcsr |= FFLAG_INVALID_OPERATION; + /* handle values so big that all relevant lower bits are 0 */ + if (exp > 52 + 31) + return 0; + + u64 mant = (val & 0x000fffffffffffff) | 0x0010000000000000; + + /* handle all other values */ + if (exp >= 52) { + mant = mant << (exp - 52); + } else { + if ((mant & (0x000fffffffffffff >> exp)) != 0) + *fcsr |= FFLAG_INEXACT; + mant = mant >> (52 - exp); + } + mant &= 0x7fffffff; + return sign ? -mant : mant; +} + +static u32 f32_handle_and_signal_nans(u32 rs1, u32 rs2) +{ + u32 val = 0; + /* check first and second operand for NaN */ + if ((rs1 & 0x7fffffff) > 0x7f800000) { + /* set canonical NaN */ + val = 0x7fc00000; + /* check for signaling NaN */ + if (!(rs1 & 0x00400000)) + SET_FCSR(GET_FCSR() | FFLAG_INVALID_OPERATION); + } else if ((rs2 & 0x7fffffff) > 0x7f800000) { + /* set canonical NaN */ + val = 0x7fc00000; + /* check for signaling NaN */ + if (!(rs2 & 0x00400000)) + SET_FCSR(GET_FCSR() | FFLAG_INVALID_OPERATION); + } + return val; +} + +static u64 f64_handle_and_signal_nans(u64 rs1, u64 rs2) +{ + u64 val = 0; + /* check first and second operand for NaN */ + if ((rs1 & 0x7fffffffffffffff) > 0x7ff0000000000000) { + /* set canonical NaN */ + val = 0x7ff8000000000000; + /* check for signaling NaN */ + if (!(rs1 & 0x0008000000000000)) + SET_FCSR(GET_FCSR() | FFLAG_INVALID_OPERATION); + } else if ((rs2 & 0x7fffffffffffffff) > 0x7ff0000000000000) { + /* set canonical NaN */ + val = 0x7ff8000000000000; + /* check for signaling NaN */ + if (!(rs2 & 0x0008000000000000)) + SET_FCSR(GET_FCSR() | FFLAG_INVALID_OPERATION); + } + return val; +} + +static u16 f16_handle_and_signal_nans(u16 rs1, u16 rs2) +{ + u16 val = 0; + /* check first and second operand for NaN */ + if ((rs1 & 0x7fff) > 0x7c00) { + /* set canonical NaN */ + val = 0x7e00; + /* check for signaling NaN */ + if (!(rs1 & 0x0200)) + SET_FCSR(GET_FCSR() | FFLAG_INVALID_OPERATION); + } else if ((rs2 & 0x7fff) > 0x7c00) { + /* set canonical NaN */ + val = 0x7e00; + /* check for signaling NaN */ + if (!(rs2 & 0x0200)) + SET_FCSR(GET_FCSR() | FFLAG_INVALID_OPERATION); + } + return val; +} +#endif + +#if defined(CONFIG_EMU_ZFHMIN) || defined(CONFIG_EMU_ZFA) +int sbi_insn_emu_op_fp(ulong insn, struct sbi_trap_regs *regs) +{ + u64 val; + u32 fcsr; + + /* do not emulate floating point instructions when disabled */ + if ((regs->mstatus & MSTATUS_FS) == 0 || + (sbi_mstatus_prev_mode(regs->mstatus) == PRV_U && + (csr_read(CSR_SSTATUS) & SSTATUS_FS) == 0)) + return truly_illegal_insn(insn, regs); + + switch (insn & INSN_MASK_ITYPE_RD_RS) { +#ifdef CONFIG_EMU_ZFHMIN + /* Emulate Zfhmin instructions */ + case INSN_MATCH_FCVT_S_H | (RM_FIELD_RNE << 12): + case INSN_MATCH_FCVT_S_H | (RM_FIELD_RTZ << 12): + case INSN_MATCH_FCVT_S_H | (RM_FIELD_RDN << 12): + case INSN_MATCH_FCVT_S_H | (RM_FIELD_RUP << 12): + case INSN_MATCH_FCVT_S_H | (RM_FIELD_RMM << 12): + case INSN_MATCH_FCVT_S_H | (RM_FIELD_DYN << 12): + fcsr = GET_FCSR(); + val = GET_F16_RS1_OR_NAN(insn, regs); + val = convert_f16_to_f32(val, &fcsr); + SET_F32_RD(insn, regs, val); + SET_FCSR(fcsr); + break; + case INSN_MATCH_FCVT_H_S | (RM_FIELD_RNE << 12): + case INSN_MATCH_FCVT_H_S | (RM_FIELD_RTZ << 12): + case INSN_MATCH_FCVT_H_S | (RM_FIELD_RDN << 12): + case INSN_MATCH_FCVT_H_S | (RM_FIELD_RUP << 12): + case INSN_MATCH_FCVT_H_S | (RM_FIELD_RMM << 12): + fcsr = GET_FCSR(); + val = GET_F32_RS1_OR_NAN(insn, regs); + val = convert_f32_to_f16(val, &fcsr, GET_RM(insn)); + SET_F16_RD(insn, regs, val); + SET_FCSR(fcsr); + break; + case INSN_MATCH_FCVT_H_S | (RM_FIELD_DYN << 12): + fcsr = GET_FCSR(); + if ((fcsr & 0xe0) == 0xa0 || (fcsr & 0xe0) == 0xc0) + return truly_illegal_insn(insn, regs); + val = GET_F32_RS1_OR_NAN(insn, regs); + val = convert_f32_to_f16(val, &fcsr, (fcsr >> 5) & 7); + SET_F16_RD(insn, regs, val); + SET_FCSR(fcsr); + break; + case INSN_MATCH_FCVT_D_H | (RM_FIELD_RNE << 12): + case INSN_MATCH_FCVT_D_H | (RM_FIELD_RTZ << 12): + case INSN_MATCH_FCVT_D_H | (RM_FIELD_RDN << 12): + case INSN_MATCH_FCVT_D_H | (RM_FIELD_RUP << 12): + case INSN_MATCH_FCVT_D_H | (RM_FIELD_RMM << 12): + case INSN_MATCH_FCVT_D_H | (RM_FIELD_DYN << 12): + fcsr = GET_FCSR(); + val = GET_F16_RS1_OR_NAN(insn, regs); + val = convert_f16_to_f64(val, &fcsr); + SET_F64_RD(insn, regs, val); + SET_FCSR(fcsr); + break; + case INSN_MATCH_FCVT_H_D | (RM_FIELD_RNE << 12): + case INSN_MATCH_FCVT_H_D | (RM_FIELD_RTZ << 12): + case INSN_MATCH_FCVT_H_D | (RM_FIELD_RDN << 12): + case INSN_MATCH_FCVT_H_D | (RM_FIELD_RUP << 12): + case INSN_MATCH_FCVT_H_D | (RM_FIELD_RMM << 12): + fcsr = GET_FCSR(); + val = GET_F64_RS1_OR_NAN(insn, regs); + val = convert_f64_to_f16(val, &fcsr, GET_RM(insn)); + SET_F16_RD(insn, regs, val); + SET_FCSR(fcsr); + break; + case INSN_MATCH_FCVT_H_D | (RM_FIELD_DYN << 12): + fcsr = GET_FCSR(); + if ((fcsr & 0xe0) == 0xa0 || (fcsr & 0xe0) == 0xc0) + return truly_illegal_insn(insn, regs); + val = GET_F64_RS1_OR_NAN(insn, regs); + val = convert_f64_to_f16(val, &fcsr, (fcsr >> 5) & 7); + SET_F16_RD(insn, regs, val); + SET_FCSR(fcsr); + break; + case INSN_MATCH_FMV_X_H: + val = GET_F16_RS1(insn, regs); + SET_RD(insn, regs, (ulong)(long)(s16)(u16)val); + break; + case INSN_MATCH_FMV_H_X: + val = GET_RS1(insn, regs); + SET_F16_RD(insn, regs, val); + break; +#endif +#ifdef CONFIG_EMU_ZFA + /* Emulate Zfa instructions */ + case INSN_MATCH_FLI_H: + val = GET_RS1_NUM(insn); + val = f16_imm_lut[val]; + SET_F16_RD(insn, regs, val); + break; + case INSN_MATCH_FLI_S: + val = GET_RS1_NUM(insn); + val = f32_imm_lut[val]; + SET_F32_RD(insn, regs, val); + break; + case INSN_MATCH_FLI_D: + val = GET_RS1_NUM(insn); + val = f64_imm_lut[val]; + SET_F64_RD(insn, regs, val); + break; + case INSN_MATCH_FROUND_S | (RM_FIELD_RNE << 12): + case INSN_MATCH_FROUND_S | (RM_FIELD_RTZ << 12): + case INSN_MATCH_FROUND_S | (RM_FIELD_RDN << 12): + case INSN_MATCH_FROUND_S | (RM_FIELD_RUP << 12): + case INSN_MATCH_FROUND_S | (RM_FIELD_RMM << 12): + fcsr = GET_FCSR(); + val = GET_F32_RS1_OR_NAN(insn, regs); + val = round_f32(val, &fcsr, GET_RM(insn), false); + SET_F32_RD(insn, regs, val); + SET_FCSR(fcsr); + break; + case INSN_MATCH_FROUND_S | (RM_FIELD_DYN << 12): + fcsr = GET_FCSR(); + if ((fcsr & 0xe0) == 0xa0 || (fcsr & 0xe0) == 0xc0) + return truly_illegal_insn(insn, regs); + val = GET_F32_RS1_OR_NAN(insn, regs); + val = round_f32(val, &fcsr, (fcsr >> 5) & 7, false); + SET_F32_RD(insn, regs, val); + SET_FCSR(fcsr); + break; + case INSN_MATCH_FROUNDNX_S | (RM_FIELD_RNE << 12): + case INSN_MATCH_FROUNDNX_S | (RM_FIELD_RTZ << 12): + case INSN_MATCH_FROUNDNX_S | (RM_FIELD_RDN << 12): + case INSN_MATCH_FROUNDNX_S | (RM_FIELD_RUP << 12): + case INSN_MATCH_FROUNDNX_S | (RM_FIELD_RMM << 12): + fcsr = GET_FCSR(); + val = GET_F32_RS1_OR_NAN(insn, regs); + val = round_f32(val, &fcsr, GET_RM(insn), true); + SET_F32_RD(insn, regs, val); + SET_FCSR(fcsr); + break; + case INSN_MATCH_FROUNDNX_S | (RM_FIELD_DYN << 12): + fcsr = GET_FCSR(); + if ((fcsr & 0xe0) == 0xa0 || (fcsr & 0xe0) == 0xc0) + return truly_illegal_insn(insn, regs); + val = GET_F32_RS1_OR_NAN(insn, regs); + val = round_f32(val, &fcsr, (fcsr >> 5) & 7, true); + SET_F32_RD(insn, regs, val); + SET_FCSR(fcsr); + break; + case INSN_MATCH_FROUND_D | (RM_FIELD_RNE << 12): + case INSN_MATCH_FROUND_D | (RM_FIELD_RTZ << 12): + case INSN_MATCH_FROUND_D | (RM_FIELD_RDN << 12): + case INSN_MATCH_FROUND_D | (RM_FIELD_RUP << 12): + case INSN_MATCH_FROUND_D | (RM_FIELD_RMM << 12): + fcsr = GET_FCSR(); + val = GET_F64_RS1_OR_NAN(insn, regs); + val = round_f64(val, &fcsr, GET_RM(insn), false); + SET_F64_RD(insn, regs, val); + SET_FCSR(fcsr); + break; + case INSN_MATCH_FROUND_D | (RM_FIELD_DYN << 12): + fcsr = GET_FCSR(); + if ((fcsr & 0xe0) == 0xa0 || (fcsr & 0xe0) == 0xc0) + return truly_illegal_insn(insn, regs); + val = GET_F64_RS1_OR_NAN(insn, regs); + val = round_f64(val, &fcsr, (fcsr >> 5) & 7, false); + SET_F64_RD(insn, regs, val); + SET_FCSR(fcsr); + break; + case INSN_MATCH_FROUNDNX_D | (RM_FIELD_RNE << 12): + case INSN_MATCH_FROUNDNX_D | (RM_FIELD_RTZ << 12): + case INSN_MATCH_FROUNDNX_D | (RM_FIELD_RDN << 12): + case INSN_MATCH_FROUNDNX_D | (RM_FIELD_RUP << 12): + case INSN_MATCH_FROUNDNX_D | (RM_FIELD_RMM << 12): + fcsr = GET_FCSR(); + val = GET_F64_RS1_OR_NAN(insn, regs); + val = round_f64(val, &fcsr, GET_RM(insn), true); + SET_F64_RD(insn, regs, val); + SET_FCSR(fcsr); + break; + case INSN_MATCH_FROUNDNX_D | (RM_FIELD_DYN << 12): + fcsr = GET_FCSR(); + if ((fcsr & 0xe0) == 0xa0 || (fcsr & 0xe0) == 0xc0) + return truly_illegal_insn(insn, regs); + val = GET_F64_RS1_OR_NAN(insn, regs); + val = round_f64(val, &fcsr, (fcsr >> 5) & 7, true); + SET_F64_RD(insn, regs, val); + SET_FCSR(fcsr); + break; + case INSN_MATCH_FROUND_H | (RM_FIELD_RNE << 12): + case INSN_MATCH_FROUND_H | (RM_FIELD_RTZ << 12): + case INSN_MATCH_FROUND_H | (RM_FIELD_RDN << 12): + case INSN_MATCH_FROUND_H | (RM_FIELD_RUP << 12): + case INSN_MATCH_FROUND_H | (RM_FIELD_RMM << 12): + fcsr = GET_FCSR(); + val = GET_F16_RS1_OR_NAN(insn, regs); + val = round_f16(val, &fcsr, GET_RM(insn), false); + SET_F16_RD(insn, regs, val); + SET_FCSR(fcsr); + break; + case INSN_MATCH_FROUND_H | (RM_FIELD_DYN << 12): + fcsr = GET_FCSR(); + if ((fcsr & 0xe0) == 0xa0 || (fcsr & 0xe0) == 0xc0) + return truly_illegal_insn(insn, regs); + val = GET_F16_RS1_OR_NAN(insn, regs); + val = round_f16(val, &fcsr, (fcsr >> 5) & 7, false); + SET_F16_RD(insn, regs, val); + SET_FCSR(fcsr); + break; + case INSN_MATCH_FROUNDNX_H | (RM_FIELD_RNE << 12): + case INSN_MATCH_FROUNDNX_H | (RM_FIELD_RTZ << 12): + case INSN_MATCH_FROUNDNX_H | (RM_FIELD_RDN << 12): + case INSN_MATCH_FROUNDNX_H | (RM_FIELD_RUP << 12): + case INSN_MATCH_FROUNDNX_H | (RM_FIELD_RMM << 12): + fcsr = GET_FCSR(); + val = GET_F16_RS1_OR_NAN(insn, regs); + val = round_f16(val, &fcsr, GET_RM(insn), true); + SET_F16_RD(insn, regs, val); + SET_FCSR(fcsr); + break; + case INSN_MATCH_FROUNDNX_H | (RM_FIELD_DYN << 12): + fcsr = GET_FCSR(); + if ((fcsr & 0xe0) == 0xa0 || (fcsr & 0xe0) == 0xc0) + return truly_illegal_insn(insn, regs); + val = GET_F16_RS1_OR_NAN(insn, regs); + val = round_f16(val, &fcsr, (fcsr >> 5) & 7, true); + SET_F16_RD(insn, regs, val); + SET_FCSR(fcsr); + break; + case INSN_MATCH_FCVTMOD_W_D: + fcsr = GET_FCSR(); + val = GET_F64_RS1_OR_NAN(insn, regs); + val = (s64)fcvtmod_f64(val, &fcsr); + SET_RD(insn, regs, val); + SET_FCSR(fcsr); + break; +#endif + default: + switch (insn & INSN_MASK_RTYPE_RD_RS1_RS2) { +#ifdef CONFIG_EMU_ZFA + case INSN_MATCH_FMINM_H: { + u16 rs1 = GET_F16_RS1_OR_NAN(insn, regs); + u16 rs2 = GET_F16_RS2_OR_NAN(insn, regs); + if (!(val = f16_handle_and_signal_nans(rs1, rs2))) + val = ((rs1 < rs2) ^ ((rs1 | rs2) >> 15)) ? rs1 + : rs2; + SET_F16_RD(insn, regs, val); + break; + } + case INSN_MATCH_FMAXM_H: { + u16 rs1 = GET_F16_RS1_OR_NAN(insn, regs); + u16 rs2 = GET_F16_RS2_OR_NAN(insn, regs); + if (!(val = f16_handle_and_signal_nans(rs1, rs2))) + val = ((rs1 > rs2) ^ ((rs1 | rs2) >> 15)) ? rs1 + : rs2; + SET_F16_RD(insn, regs, val); + break; + } + case INSN_MATCH_FMINM_S: { + u32 rs1 = GET_F32_RS1_OR_NAN(insn, regs); + u32 rs2 = GET_F32_RS2_OR_NAN(insn, regs); + if (!(val = f32_handle_and_signal_nans(rs1, rs2))) + val = ((rs1 < rs2) ^ ((rs1 | rs2) >> 31)) ? rs1 + : rs2; + SET_F32_RD(insn, regs, val); + break; + } + case INSN_MATCH_FMAXM_S: { + u32 rs1 = GET_F32_RS1_OR_NAN(insn, regs); + u32 rs2 = GET_F32_RS2_OR_NAN(insn, regs); + if (!(val = f32_handle_and_signal_nans(rs1, rs2))) + val = ((rs1 > rs2) ^ ((rs1 | rs2) >> 31)) ? rs1 + : rs2; + SET_F32_RD(insn, regs, val); + break; + } + case INSN_MATCH_FMINM_D: { + u64 rs1 = GET_F64_RS1_OR_NAN(insn, regs); + u64 rs2 = GET_F64_RS2_OR_NAN(insn, regs); + if (!(val = f64_handle_and_signal_nans(rs1, rs2))) + val = ((rs1 < rs2) ^ ((rs1 | rs2) >> 63)) ? rs1 + : rs2; + SET_F64_RD(insn, regs, val); + break; + } + case INSN_MATCH_FMAXM_D: { + u64 rs1 = GET_F64_RS1_OR_NAN(insn, regs); + u64 rs2 = GET_F64_RS2_OR_NAN(insn, regs); + if (!(val = f64_handle_and_signal_nans(rs1, rs2))) + val = ((rs1 > rs2) ^ ((rs1 | rs2) >> 63)) ? rs1 + : rs2; + SET_F64_RD(insn, regs, val); + break; + } + case INSN_MATCH_FLTQ_H: { + u16 rs1 = GET_F16_RS1_OR_NAN(insn, regs); + u16 rs2 = GET_F16_RS2_OR_NAN(insn, regs); + if ((val = !f16_handle_and_signal_nans(rs1, rs2))) + val = (rs1 < rs2) ^ ((rs1 | rs2) >> 15); + SET_RD(insn, regs, val); + break; + } + case INSN_MATCH_FLEQ_H: { + u16 rs1 = GET_F16_RS1_OR_NAN(insn, regs); + u16 rs2 = GET_F16_RS2_OR_NAN(insn, regs); + if ((val = !f16_handle_and_signal_nans(rs1, rs2))) + val = !((rs1 > rs2) ^ ((rs1 | rs2) >> 15)); + SET_RD(insn, regs, val); + break; + } + case INSN_MATCH_FLTQ_S: { + u32 rs1 = GET_F32_RS1_OR_NAN(insn, regs); + u32 rs2 = GET_F32_RS2_OR_NAN(insn, regs); + if ((val = !f32_handle_and_signal_nans(rs1, rs2))) + val = (rs1 < rs2) ^ ((rs1 | rs2) >> 31); + SET_RD(insn, regs, val); + break; + } + case INSN_MATCH_FLEQ_S: { + u32 rs1 = GET_F32_RS1_OR_NAN(insn, regs); + u32 rs2 = GET_F32_RS2_OR_NAN(insn, regs); + if ((val = !f32_handle_and_signal_nans(rs1, rs2))) + val = !((rs1 > rs2) ^ ((rs1 | rs2) >> 31)); + SET_RD(insn, regs, val); + break; + } + case INSN_MATCH_FLTQ_D: { + u64 rs1 = GET_F64_RS1_OR_NAN(insn, regs); + u64 rs2 = GET_F64_RS2_OR_NAN(insn, regs); + if ((val = !f64_handle_and_signal_nans(rs1, rs2))) + val = (rs1 < rs2) ^ ((rs1 | rs2) >> 63); + SET_RD(insn, regs, val); + break; + } + case INSN_MATCH_FLEQ_D: { + u64 rs1 = GET_F64_RS1_OR_NAN(insn, regs); + u64 rs2 = GET_F64_RS2_OR_NAN(insn, regs); + if ((val = !f64_handle_and_signal_nans(rs1, rs2))) + val = !((rs1 > rs2) ^ ((rs1 | rs2) >> 63)); + SET_RD(insn, regs, val); + break; + } +#endif + default: + return truly_illegal_insn(insn, regs); + } + } + + regs->mepc += 4; + + return 0; +} +#endif diff --git a/lib/sbi/sbi_insn_emu_v.c b/lib/sbi/sbi_insn_emu_v.c new file mode 100644 index 00000000000..dedb1334b8a --- /dev/null +++ b/lib/sbi/sbi_insn_emu_v.c @@ -0,0 +1,1150 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 Benedikt Freisen. + * + * Authors: + * Benedikt Freisen + */ + +#if __riscv_xlen == 64 && CONFIG_EMU_ZVBB + +#ifndef OPENSBI_CC_SUPPORT_VECTOR +#error "Zvbb emulation requires a toolchain that supports the V extension!" +#endif + +#include +#include +#include + +/* we need a buffer size of eight times VLEN bits */ +#if CONFIG_EMU_ZVBB_VLEN_128 +#define VLMAX_BYTES (8 * 16) +#endif + +#if CONFIG_EMU_ZVBB_VLEN_256 +#define VLMAX_BYTES (8 * 32) +#endif + +#if CONFIG_EMU_ZVBB_VLEN_512 +#define VLMAX_BYTES (8 * 64) +#endif + +#if CONFIG_EMU_ZVBB_VLEN_1024 +#define VLMAX_BYTES (8 * 128) +#endif + +typedef union { + u8 u8[VLMAX_BYTES]; + u16 u16[VLMAX_BYTES / 2]; + u32 u32[VLMAX_BYTES / 4]; + u64 u64[VLMAX_BYTES / 8]; +} sbi_vector_data; + +#define INLINE_VSE8(nstr, dest) \ + asm volatile(".option push\n\t" \ + ".option arch, +v\n\t" \ + "vse8.v " nstr ", (%0)\n\t" \ + ".option pop\n\t" ::"r"(dest) \ + : "memory"); + +#define CASE_N_INLINE_VSE8(n, nstr, dest) \ + case n: \ + INLINE_VSE8(nstr, dest); \ + break; + +#define INLINE_VSE16(nstr, dest) \ + asm volatile(".option push\n\t" \ + ".option arch, +v\n\t" \ + "vse16.v " nstr ", (%0)\n\t" \ + ".option pop\n\t" ::"r"(dest) \ + : "memory"); + +#define CASE_N_INLINE_VSE16(n, nstr, dest) \ + case n: \ + INLINE_VSE16(nstr, dest); \ + break; + +#define INLINE_VSE32(nstr, dest) \ + asm volatile(".option push\n\t" \ + ".option arch, +v\n\t" \ + "vse32.v " nstr ", (%0)\n\t" \ + ".option pop\n\t" ::"r"(dest) \ + : "memory"); + +#define CASE_N_INLINE_VSE32(n, nstr, dest) \ + case n: \ + INLINE_VSE32(nstr, dest); \ + break; + +#define INLINE_VSE64(nstr, dest) \ + asm volatile(".option push\n\t" \ + ".option arch, +v\n\t" \ + "vse64.v " nstr ", (%0)\n\t" \ + ".option pop\n\t" ::"r"(dest) \ + : "memory"); + +#define CASE_N_INLINE_VSE64(n, nstr, dest) \ + case n: \ + INLINE_VSE64(nstr, dest); \ + break; + +static inline void get_vector_as_array_u8(int n, sbi_vector_data *dest) +{ + switch (n) { + CASE_N_INLINE_VSE8(0, "v0", dest); + CASE_N_INLINE_VSE8(1, "v1", dest); + CASE_N_INLINE_VSE8(2, "v2", dest); + CASE_N_INLINE_VSE8(3, "v3", dest); + CASE_N_INLINE_VSE8(4, "v4", dest); + CASE_N_INLINE_VSE8(5, "v5", dest); + CASE_N_INLINE_VSE8(6, "v6", dest); + CASE_N_INLINE_VSE8(7, "v7", dest); + CASE_N_INLINE_VSE8(8, "v8", dest); + CASE_N_INLINE_VSE8(9, "v9", dest); + CASE_N_INLINE_VSE8(10, "v10", dest); + CASE_N_INLINE_VSE8(11, "v11", dest); + CASE_N_INLINE_VSE8(12, "v12", dest); + CASE_N_INLINE_VSE8(13, "v13", dest); + CASE_N_INLINE_VSE8(14, "v14", dest); + CASE_N_INLINE_VSE8(15, "v15", dest); + CASE_N_INLINE_VSE8(16, "v16", dest); + CASE_N_INLINE_VSE8(17, "v17", dest); + CASE_N_INLINE_VSE8(18, "v18", dest); + CASE_N_INLINE_VSE8(19, "v19", dest); + CASE_N_INLINE_VSE8(20, "v20", dest); + CASE_N_INLINE_VSE8(21, "v21", dest); + CASE_N_INLINE_VSE8(22, "v22", dest); + CASE_N_INLINE_VSE8(23, "v23", dest); + CASE_N_INLINE_VSE8(24, "v24", dest); + CASE_N_INLINE_VSE8(25, "v25", dest); + CASE_N_INLINE_VSE8(26, "v26", dest); + CASE_N_INLINE_VSE8(27, "v27", dest); + CASE_N_INLINE_VSE8(28, "v28", dest); + CASE_N_INLINE_VSE8(29, "v29", dest); + CASE_N_INLINE_VSE8(30, "v30", dest); + CASE_N_INLINE_VSE8(31, "v31", dest); + } +} + +static inline void get_vector_as_array_u16(int n, sbi_vector_data *dest) +{ + switch (n) { + CASE_N_INLINE_VSE16(0, "v0", dest); + CASE_N_INLINE_VSE16(1, "v1", dest); + CASE_N_INLINE_VSE16(2, "v2", dest); + CASE_N_INLINE_VSE16(3, "v3", dest); + CASE_N_INLINE_VSE16(4, "v4", dest); + CASE_N_INLINE_VSE16(5, "v5", dest); + CASE_N_INLINE_VSE16(6, "v6", dest); + CASE_N_INLINE_VSE16(7, "v7", dest); + CASE_N_INLINE_VSE16(8, "v8", dest); + CASE_N_INLINE_VSE16(9, "v9", dest); + CASE_N_INLINE_VSE16(10, "v10", dest); + CASE_N_INLINE_VSE16(11, "v11", dest); + CASE_N_INLINE_VSE16(12, "v12", dest); + CASE_N_INLINE_VSE16(13, "v13", dest); + CASE_N_INLINE_VSE16(14, "v14", dest); + CASE_N_INLINE_VSE16(15, "v15", dest); + CASE_N_INLINE_VSE16(16, "v16", dest); + CASE_N_INLINE_VSE16(17, "v17", dest); + CASE_N_INLINE_VSE16(18, "v18", dest); + CASE_N_INLINE_VSE16(19, "v19", dest); + CASE_N_INLINE_VSE16(20, "v20", dest); + CASE_N_INLINE_VSE16(21, "v21", dest); + CASE_N_INLINE_VSE16(22, "v22", dest); + CASE_N_INLINE_VSE16(23, "v23", dest); + CASE_N_INLINE_VSE16(24, "v24", dest); + CASE_N_INLINE_VSE16(25, "v25", dest); + CASE_N_INLINE_VSE16(26, "v26", dest); + CASE_N_INLINE_VSE16(27, "v27", dest); + CASE_N_INLINE_VSE16(28, "v28", dest); + CASE_N_INLINE_VSE16(29, "v29", dest); + CASE_N_INLINE_VSE16(30, "v30", dest); + CASE_N_INLINE_VSE16(31, "v31", dest); + } +} + +static inline void get_vector_as_array_u32(int n, sbi_vector_data *dest) +{ + switch (n) { + CASE_N_INLINE_VSE32(0, "v0", dest); + CASE_N_INLINE_VSE32(1, "v1", dest); + CASE_N_INLINE_VSE32(2, "v2", dest); + CASE_N_INLINE_VSE32(3, "v3", dest); + CASE_N_INLINE_VSE32(4, "v4", dest); + CASE_N_INLINE_VSE32(5, "v5", dest); + CASE_N_INLINE_VSE32(6, "v6", dest); + CASE_N_INLINE_VSE32(7, "v7", dest); + CASE_N_INLINE_VSE32(8, "v8", dest); + CASE_N_INLINE_VSE32(9, "v9", dest); + CASE_N_INLINE_VSE32(10, "v10", dest); + CASE_N_INLINE_VSE32(11, "v11", dest); + CASE_N_INLINE_VSE32(12, "v12", dest); + CASE_N_INLINE_VSE32(13, "v13", dest); + CASE_N_INLINE_VSE32(14, "v14", dest); + CASE_N_INLINE_VSE32(15, "v15", dest); + CASE_N_INLINE_VSE32(16, "v16", dest); + CASE_N_INLINE_VSE32(17, "v17", dest); + CASE_N_INLINE_VSE32(18, "v18", dest); + CASE_N_INLINE_VSE32(19, "v19", dest); + CASE_N_INLINE_VSE32(20, "v20", dest); + CASE_N_INLINE_VSE32(21, "v21", dest); + CASE_N_INLINE_VSE32(22, "v22", dest); + CASE_N_INLINE_VSE32(23, "v23", dest); + CASE_N_INLINE_VSE32(24, "v24", dest); + CASE_N_INLINE_VSE32(25, "v25", dest); + CASE_N_INLINE_VSE32(26, "v26", dest); + CASE_N_INLINE_VSE32(27, "v27", dest); + CASE_N_INLINE_VSE32(28, "v28", dest); + CASE_N_INLINE_VSE32(29, "v29", dest); + CASE_N_INLINE_VSE32(30, "v30", dest); + CASE_N_INLINE_VSE32(31, "v31", dest); + } +} + +static inline void get_vector_as_array_u64(int n, sbi_vector_data *dest) +{ + switch (n) { + CASE_N_INLINE_VSE64(0, "v0", dest); + CASE_N_INLINE_VSE64(1, "v1", dest); + CASE_N_INLINE_VSE64(2, "v2", dest); + CASE_N_INLINE_VSE64(3, "v3", dest); + CASE_N_INLINE_VSE64(4, "v4", dest); + CASE_N_INLINE_VSE64(5, "v5", dest); + CASE_N_INLINE_VSE64(6, "v6", dest); + CASE_N_INLINE_VSE64(7, "v7", dest); + CASE_N_INLINE_VSE64(8, "v8", dest); + CASE_N_INLINE_VSE64(9, "v9", dest); + CASE_N_INLINE_VSE64(10, "v10", dest); + CASE_N_INLINE_VSE64(11, "v11", dest); + CASE_N_INLINE_VSE64(12, "v12", dest); + CASE_N_INLINE_VSE64(13, "v13", dest); + CASE_N_INLINE_VSE64(14, "v14", dest); + CASE_N_INLINE_VSE64(15, "v15", dest); + CASE_N_INLINE_VSE64(16, "v16", dest); + CASE_N_INLINE_VSE64(17, "v17", dest); + CASE_N_INLINE_VSE64(18, "v18", dest); + CASE_N_INLINE_VSE64(19, "v19", dest); + CASE_N_INLINE_VSE64(20, "v20", dest); + CASE_N_INLINE_VSE64(21, "v21", dest); + CASE_N_INLINE_VSE64(22, "v22", dest); + CASE_N_INLINE_VSE64(23, "v23", dest); + CASE_N_INLINE_VSE64(24, "v24", dest); + CASE_N_INLINE_VSE64(25, "v25", dest); + CASE_N_INLINE_VSE64(26, "v26", dest); + CASE_N_INLINE_VSE64(27, "v27", dest); + CASE_N_INLINE_VSE64(28, "v28", dest); + CASE_N_INLINE_VSE64(29, "v29", dest); + CASE_N_INLINE_VSE64(30, "v30", dest); + CASE_N_INLINE_VSE64(31, "v31", dest); + } +} + +#define INLINE_VLE8(nstr, src) \ + asm volatile(".option push\n\t" \ + ".option arch, +v\n\t" \ + "vle8.v " nstr ", (%0)\n\t" \ + ".option pop\n\t" ::"r"(src) \ + : "memory"); + +#define CASE_N_INLINE_VLE8(n, nstr, src) \ + case n: \ + INLINE_VLE8(nstr, src); \ + break; + +#define INLINE_VLE16(nstr, src) \ + asm volatile(".option push\n\t" \ + ".option arch, +v\n\t" \ + "vle16.v " nstr ", (%0)\n\t" \ + ".option pop\n\t" ::"r"(src) \ + : "memory"); + +#define CASE_N_INLINE_VLE16(n, nstr, src) \ + case n: \ + INLINE_VLE16(nstr, src); \ + break; + +#define INLINE_VLE32(nstr, src) \ + asm volatile(".option push\n\t" \ + ".option arch, +v\n\t" \ + "vle32.v " nstr ", (%0)\n\t" \ + ".option pop\n\t" ::"r"(src) \ + : "memory"); + +#define CASE_N_INLINE_VLE32(n, nstr, src) \ + case n: \ + INLINE_VLE32(nstr, src); \ + break; + +#define INLINE_VLE64(nstr, src) \ + asm volatile(".option push\n\t" \ + ".option arch, +v\n\t" \ + "vle64.v " nstr ", (%0)\n\t" \ + ".option pop\n\t" ::"r"(src) \ + : "memory"); + +#define CASE_N_INLINE_VLE64(n, nstr, src) \ + case n: \ + INLINE_VLE64(nstr, src); \ + break; + +static inline void set_vector_from_array_u8(int n, sbi_vector_data *src) +{ + switch (n) { + CASE_N_INLINE_VLE8(0, "v0", src); + CASE_N_INLINE_VLE8(1, "v1", src); + CASE_N_INLINE_VLE8(2, "v2", src); + CASE_N_INLINE_VLE8(3, "v3", src); + CASE_N_INLINE_VLE8(4, "v4", src); + CASE_N_INLINE_VLE8(5, "v5", src); + CASE_N_INLINE_VLE8(6, "v6", src); + CASE_N_INLINE_VLE8(7, "v7", src); + CASE_N_INLINE_VLE8(8, "v8", src); + CASE_N_INLINE_VLE8(9, "v9", src); + CASE_N_INLINE_VLE8(10, "v10", src); + CASE_N_INLINE_VLE8(11, "v11", src); + CASE_N_INLINE_VLE8(12, "v12", src); + CASE_N_INLINE_VLE8(13, "v13", src); + CASE_N_INLINE_VLE8(14, "v14", src); + CASE_N_INLINE_VLE8(15, "v15", src); + CASE_N_INLINE_VLE8(16, "v16", src); + CASE_N_INLINE_VLE8(17, "v17", src); + CASE_N_INLINE_VLE8(18, "v18", src); + CASE_N_INLINE_VLE8(19, "v19", src); + CASE_N_INLINE_VLE8(20, "v20", src); + CASE_N_INLINE_VLE8(21, "v21", src); + CASE_N_INLINE_VLE8(22, "v22", src); + CASE_N_INLINE_VLE8(23, "v23", src); + CASE_N_INLINE_VLE8(24, "v24", src); + CASE_N_INLINE_VLE8(25, "v25", src); + CASE_N_INLINE_VLE8(26, "v26", src); + CASE_N_INLINE_VLE8(27, "v27", src); + CASE_N_INLINE_VLE8(28, "v28", src); + CASE_N_INLINE_VLE8(29, "v29", src); + CASE_N_INLINE_VLE8(30, "v30", src); + CASE_N_INLINE_VLE8(31, "v31", src); + } +} + +static inline void set_vector_from_array_u16(int n, sbi_vector_data *src) +{ + switch (n) { + CASE_N_INLINE_VLE16(0, "v0", src); + CASE_N_INLINE_VLE16(1, "v1", src); + CASE_N_INLINE_VLE16(2, "v2", src); + CASE_N_INLINE_VLE16(3, "v3", src); + CASE_N_INLINE_VLE16(4, "v4", src); + CASE_N_INLINE_VLE16(5, "v5", src); + CASE_N_INLINE_VLE16(6, "v6", src); + CASE_N_INLINE_VLE16(7, "v7", src); + CASE_N_INLINE_VLE16(8, "v8", src); + CASE_N_INLINE_VLE16(9, "v9", src); + CASE_N_INLINE_VLE16(10, "v10", src); + CASE_N_INLINE_VLE16(11, "v11", src); + CASE_N_INLINE_VLE16(12, "v12", src); + CASE_N_INLINE_VLE16(13, "v13", src); + CASE_N_INLINE_VLE16(14, "v14", src); + CASE_N_INLINE_VLE16(15, "v15", src); + CASE_N_INLINE_VLE16(16, "v16", src); + CASE_N_INLINE_VLE16(17, "v17", src); + CASE_N_INLINE_VLE16(18, "v18", src); + CASE_N_INLINE_VLE16(19, "v19", src); + CASE_N_INLINE_VLE16(20, "v20", src); + CASE_N_INLINE_VLE16(21, "v21", src); + CASE_N_INLINE_VLE16(22, "v22", src); + CASE_N_INLINE_VLE16(23, "v23", src); + CASE_N_INLINE_VLE16(24, "v24", src); + CASE_N_INLINE_VLE16(25, "v25", src); + CASE_N_INLINE_VLE16(26, "v26", src); + CASE_N_INLINE_VLE16(27, "v27", src); + CASE_N_INLINE_VLE16(28, "v28", src); + CASE_N_INLINE_VLE16(29, "v29", src); + CASE_N_INLINE_VLE16(30, "v30", src); + CASE_N_INLINE_VLE16(31, "v31", src); + } +} + +static inline void set_vector_from_array_u32(int n, sbi_vector_data *src) +{ + switch (n) { + CASE_N_INLINE_VLE32(0, "v0", src); + CASE_N_INLINE_VLE32(1, "v1", src); + CASE_N_INLINE_VLE32(2, "v2", src); + CASE_N_INLINE_VLE32(3, "v3", src); + CASE_N_INLINE_VLE32(4, "v4", src); + CASE_N_INLINE_VLE32(5, "v5", src); + CASE_N_INLINE_VLE32(6, "v6", src); + CASE_N_INLINE_VLE32(7, "v7", src); + CASE_N_INLINE_VLE32(8, "v8", src); + CASE_N_INLINE_VLE32(9, "v9", src); + CASE_N_INLINE_VLE32(10, "v10", src); + CASE_N_INLINE_VLE32(11, "v11", src); + CASE_N_INLINE_VLE32(12, "v12", src); + CASE_N_INLINE_VLE32(13, "v13", src); + CASE_N_INLINE_VLE32(14, "v14", src); + CASE_N_INLINE_VLE32(15, "v15", src); + CASE_N_INLINE_VLE32(16, "v16", src); + CASE_N_INLINE_VLE32(17, "v17", src); + CASE_N_INLINE_VLE32(18, "v18", src); + CASE_N_INLINE_VLE32(19, "v19", src); + CASE_N_INLINE_VLE32(20, "v20", src); + CASE_N_INLINE_VLE32(21, "v21", src); + CASE_N_INLINE_VLE32(22, "v22", src); + CASE_N_INLINE_VLE32(23, "v23", src); + CASE_N_INLINE_VLE32(24, "v24", src); + CASE_N_INLINE_VLE32(25, "v25", src); + CASE_N_INLINE_VLE32(26, "v26", src); + CASE_N_INLINE_VLE32(27, "v27", src); + CASE_N_INLINE_VLE32(28, "v28", src); + CASE_N_INLINE_VLE32(29, "v29", src); + CASE_N_INLINE_VLE32(30, "v30", src); + CASE_N_INLINE_VLE32(31, "v31", src); + } +} + +static inline void set_vector_from_array_u64(int n, sbi_vector_data *src) +{ + switch (n) { + CASE_N_INLINE_VLE64(0, "v0", src); + CASE_N_INLINE_VLE64(1, "v1", src); + CASE_N_INLINE_VLE64(2, "v2", src); + CASE_N_INLINE_VLE64(3, "v3", src); + CASE_N_INLINE_VLE64(4, "v4", src); + CASE_N_INLINE_VLE64(5, "v5", src); + CASE_N_INLINE_VLE64(6, "v6", src); + CASE_N_INLINE_VLE64(7, "v7", src); + CASE_N_INLINE_VLE64(8, "v8", src); + CASE_N_INLINE_VLE64(9, "v9", src); + CASE_N_INLINE_VLE64(10, "v10", src); + CASE_N_INLINE_VLE64(11, "v11", src); + CASE_N_INLINE_VLE64(12, "v12", src); + CASE_N_INLINE_VLE64(13, "v13", src); + CASE_N_INLINE_VLE64(14, "v14", src); + CASE_N_INLINE_VLE64(15, "v15", src); + CASE_N_INLINE_VLE64(16, "v16", src); + CASE_N_INLINE_VLE64(17, "v17", src); + CASE_N_INLINE_VLE64(18, "v18", src); + CASE_N_INLINE_VLE64(19, "v19", src); + CASE_N_INLINE_VLE64(20, "v20", src); + CASE_N_INLINE_VLE64(21, "v21", src); + CASE_N_INLINE_VLE64(22, "v22", src); + CASE_N_INLINE_VLE64(23, "v23", src); + CASE_N_INLINE_VLE64(24, "v24", src); + CASE_N_INLINE_VLE64(25, "v25", src); + CASE_N_INLINE_VLE64(26, "v26", src); + CASE_N_INLINE_VLE64(27, "v27", src); + CASE_N_INLINE_VLE64(28, "v28", src); + CASE_N_INLINE_VLE64(29, "v29", src); + CASE_N_INLINE_VLE64(30, "v30", src); + CASE_N_INLINE_VLE64(31, "v31", src); + } +} + +#define INLINE_VLE8_M(nstr, src) \ + asm volatile(".option push\n\t" \ + ".option arch, +v\n\t" \ + "vle8.v " nstr ", (%0), v0.t\n\t" \ + ".option pop\n\t" ::"r"(src) \ + : "memory"); + +#define CASE_N_INLINE_VLE8_M(n, nstr, src) \ + case n: \ + INLINE_VLE8_M(nstr, src); \ + break; + +#define INLINE_VLE16_M(nstr, src) \ + asm volatile(".option push\n\t" \ + ".option arch, +v\n\t" \ + "vle16.v " nstr ", (%0), v0.t\n\t" \ + ".option pop\n\t" ::"r"(src) \ + : "memory"); + +#define CASE_N_INLINE_VLE16_M(n, nstr, src) \ + case n: \ + INLINE_VLE16_M(nstr, src); \ + break; + +#define INLINE_VLE32_M(nstr, src) \ + asm volatile(".option push\n\t" \ + ".option arch, +v\n\t" \ + "vle32.v " nstr ", (%0), v0.t\n\t" \ + ".option pop\n\t" ::"r"(src) \ + : "memory"); + +#define CASE_N_INLINE_VLE32_M(n, nstr, src) \ + case n: \ + INLINE_VLE32_M(nstr, src); \ + break; + +#define INLINE_VLE64_M(nstr, src) \ + asm volatile(".option push\n\t" \ + ".option arch, +v\n\t" \ + "vle64.v " nstr ", (%0), v0.t\n\t" \ + ".option pop\n\t" ::"r"(src) \ + : "memory"); + +#define CASE_N_INLINE_VLE64_M(n, nstr, src) \ + case n: \ + INLINE_VLE64_M(nstr, src); \ + break; + +static inline void set_masked_vector_from_array_u8(int n, sbi_vector_data *src) +{ + switch (n) { + /* NOTE: v0 contains the mask */ + CASE_N_INLINE_VLE8_M(1, "v1", src); + CASE_N_INLINE_VLE8_M(2, "v2", src); + CASE_N_INLINE_VLE8_M(3, "v3", src); + CASE_N_INLINE_VLE8_M(4, "v4", src); + CASE_N_INLINE_VLE8_M(5, "v5", src); + CASE_N_INLINE_VLE8_M(6, "v6", src); + CASE_N_INLINE_VLE8_M(7, "v7", src); + CASE_N_INLINE_VLE8_M(8, "v8", src); + CASE_N_INLINE_VLE8_M(9, "v9", src); + CASE_N_INLINE_VLE8_M(10, "v10", src); + CASE_N_INLINE_VLE8_M(11, "v11", src); + CASE_N_INLINE_VLE8_M(12, "v12", src); + CASE_N_INLINE_VLE8_M(13, "v13", src); + CASE_N_INLINE_VLE8_M(14, "v14", src); + CASE_N_INLINE_VLE8_M(15, "v15", src); + CASE_N_INLINE_VLE8_M(16, "v16", src); + CASE_N_INLINE_VLE8_M(17, "v17", src); + CASE_N_INLINE_VLE8_M(18, "v18", src); + CASE_N_INLINE_VLE8_M(19, "v19", src); + CASE_N_INLINE_VLE8_M(20, "v20", src); + CASE_N_INLINE_VLE8_M(21, "v21", src); + CASE_N_INLINE_VLE8_M(22, "v22", src); + CASE_N_INLINE_VLE8_M(23, "v23", src); + CASE_N_INLINE_VLE8_M(24, "v24", src); + CASE_N_INLINE_VLE8_M(25, "v25", src); + CASE_N_INLINE_VLE8_M(26, "v26", src); + CASE_N_INLINE_VLE8_M(27, "v27", src); + CASE_N_INLINE_VLE8_M(28, "v28", src); + CASE_N_INLINE_VLE8_M(29, "v29", src); + CASE_N_INLINE_VLE8_M(30, "v30", src); + CASE_N_INLINE_VLE8_M(31, "v31", src); + } +} + +static inline void set_masked_vector_from_array_u16(int n, sbi_vector_data *src) +{ + switch (n) { + /* NOTE: v0 contains the mask */ + CASE_N_INLINE_VLE16_M(1, "v1", src); + CASE_N_INLINE_VLE16_M(2, "v2", src); + CASE_N_INLINE_VLE16_M(3, "v3", src); + CASE_N_INLINE_VLE16_M(4, "v4", src); + CASE_N_INLINE_VLE16_M(5, "v5", src); + CASE_N_INLINE_VLE16_M(6, "v6", src); + CASE_N_INLINE_VLE16_M(7, "v7", src); + CASE_N_INLINE_VLE16_M(8, "v8", src); + CASE_N_INLINE_VLE16_M(9, "v9", src); + CASE_N_INLINE_VLE16_M(10, "v10", src); + CASE_N_INLINE_VLE16_M(11, "v11", src); + CASE_N_INLINE_VLE16_M(12, "v12", src); + CASE_N_INLINE_VLE16_M(13, "v13", src); + CASE_N_INLINE_VLE16_M(14, "v14", src); + CASE_N_INLINE_VLE16_M(15, "v15", src); + CASE_N_INLINE_VLE16_M(16, "v16", src); + CASE_N_INLINE_VLE16_M(17, "v17", src); + CASE_N_INLINE_VLE16_M(18, "v18", src); + CASE_N_INLINE_VLE16_M(19, "v19", src); + CASE_N_INLINE_VLE16_M(20, "v20", src); + CASE_N_INLINE_VLE16_M(21, "v21", src); + CASE_N_INLINE_VLE16_M(22, "v22", src); + CASE_N_INLINE_VLE16_M(23, "v23", src); + CASE_N_INLINE_VLE16_M(24, "v24", src); + CASE_N_INLINE_VLE16_M(25, "v25", src); + CASE_N_INLINE_VLE16_M(26, "v26", src); + CASE_N_INLINE_VLE16_M(27, "v27", src); + CASE_N_INLINE_VLE16_M(28, "v28", src); + CASE_N_INLINE_VLE16_M(29, "v29", src); + CASE_N_INLINE_VLE16_M(30, "v30", src); + CASE_N_INLINE_VLE16_M(31, "v31", src); + } +} + +static inline void set_masked_vector_from_array_u32(int n, sbi_vector_data *src) +{ + switch (n) { + /* NOTE: v0 contains the mask */ + CASE_N_INLINE_VLE32_M(1, "v1", src); + CASE_N_INLINE_VLE32_M(2, "v2", src); + CASE_N_INLINE_VLE32_M(3, "v3", src); + CASE_N_INLINE_VLE32_M(4, "v4", src); + CASE_N_INLINE_VLE32_M(5, "v5", src); + CASE_N_INLINE_VLE32_M(6, "v6", src); + CASE_N_INLINE_VLE32_M(7, "v7", src); + CASE_N_INLINE_VLE32_M(8, "v8", src); + CASE_N_INLINE_VLE32_M(9, "v9", src); + CASE_N_INLINE_VLE32_M(10, "v10", src); + CASE_N_INLINE_VLE32_M(11, "v11", src); + CASE_N_INLINE_VLE32_M(12, "v12", src); + CASE_N_INLINE_VLE32_M(13, "v13", src); + CASE_N_INLINE_VLE32_M(14, "v14", src); + CASE_N_INLINE_VLE32_M(15, "v15", src); + CASE_N_INLINE_VLE32_M(16, "v16", src); + CASE_N_INLINE_VLE32_M(17, "v17", src); + CASE_N_INLINE_VLE32_M(18, "v18", src); + CASE_N_INLINE_VLE32_M(19, "v19", src); + CASE_N_INLINE_VLE32_M(20, "v20", src); + CASE_N_INLINE_VLE32_M(21, "v21", src); + CASE_N_INLINE_VLE32_M(22, "v22", src); + CASE_N_INLINE_VLE32_M(23, "v23", src); + CASE_N_INLINE_VLE32_M(24, "v24", src); + CASE_N_INLINE_VLE32_M(25, "v25", src); + CASE_N_INLINE_VLE32_M(26, "v26", src); + CASE_N_INLINE_VLE32_M(27, "v27", src); + CASE_N_INLINE_VLE32_M(28, "v28", src); + CASE_N_INLINE_VLE32_M(29, "v29", src); + CASE_N_INLINE_VLE32_M(30, "v30", src); + CASE_N_INLINE_VLE32_M(31, "v31", src); + } +} + +static inline void set_masked_vector_from_array_u64(int n, sbi_vector_data *src) +{ + switch (n) { + /* NOTE: v0 contains the mask */ + CASE_N_INLINE_VLE64_M(1, "v1", src); + CASE_N_INLINE_VLE64_M(2, "v2", src); + CASE_N_INLINE_VLE64_M(3, "v3", src); + CASE_N_INLINE_VLE64_M(4, "v4", src); + CASE_N_INLINE_VLE64_M(5, "v5", src); + CASE_N_INLINE_VLE64_M(6, "v6", src); + CASE_N_INLINE_VLE64_M(7, "v7", src); + CASE_N_INLINE_VLE64_M(8, "v8", src); + CASE_N_INLINE_VLE64_M(9, "v9", src); + CASE_N_INLINE_VLE64_M(10, "v10", src); + CASE_N_INLINE_VLE64_M(11, "v11", src); + CASE_N_INLINE_VLE64_M(12, "v12", src); + CASE_N_INLINE_VLE64_M(13, "v13", src); + CASE_N_INLINE_VLE64_M(14, "v14", src); + CASE_N_INLINE_VLE64_M(15, "v15", src); + CASE_N_INLINE_VLE64_M(16, "v16", src); + CASE_N_INLINE_VLE64_M(17, "v17", src); + CASE_N_INLINE_VLE64_M(18, "v18", src); + CASE_N_INLINE_VLE64_M(19, "v19", src); + CASE_N_INLINE_VLE64_M(20, "v20", src); + CASE_N_INLINE_VLE64_M(21, "v21", src); + CASE_N_INLINE_VLE64_M(22, "v22", src); + CASE_N_INLINE_VLE64_M(23, "v23", src); + CASE_N_INLINE_VLE64_M(24, "v24", src); + CASE_N_INLINE_VLE64_M(25, "v25", src); + CASE_N_INLINE_VLE64_M(26, "v26", src); + CASE_N_INLINE_VLE64_M(27, "v27", src); + CASE_N_INLINE_VLE64_M(28, "v28", src); + CASE_N_INLINE_VLE64_M(29, "v29", src); + CASE_N_INLINE_VLE64_M(30, "v30", src); + CASE_N_INLINE_VLE64_M(31, "v31", src); + } +} + +static inline void foreach_velem_vv(int vl, int sew, bool masked, int vd, + int vs1, int vs2, u64 op(u64, u64)) +{ + sbi_vector_data vs1_data; + sbi_vector_data vs2_data; + sbi_vector_data vd_data; + + /* treat as no-op if VL is 0 */ + if (vl == 0) + return; + + switch (sew) { + case 0: + get_vector_as_array_u8(vs1, &vs1_data); + get_vector_as_array_u8(vs2, &vs2_data); + for (int i = 0; i < vl; i++) + vd_data.u8[i] = op(vs1_data.u8[i], vs2_data.u8[i]); + if (masked) + set_masked_vector_from_array_u8(vd, &vd_data); + else + set_vector_from_array_u8(vd, &vd_data); + break; + case 1: + get_vector_as_array_u16(vs1, &vs1_data); + get_vector_as_array_u16(vs2, &vs2_data); + for (int i = 0; i < vl; i++) + vd_data.u16[i] = op(vs1_data.u16[i], vs2_data.u16[i]); + if (masked) + set_masked_vector_from_array_u16(vd, &vd_data); + else + set_vector_from_array_u16(vd, &vd_data); + break; + case 2: + get_vector_as_array_u32(vs1, &vs1_data); + get_vector_as_array_u32(vs2, &vs2_data); + for (int i = 0; i < vl; i++) + vd_data.u32[i] = op(vs1_data.u32[i], vs2_data.u32[i]); + if (masked) + set_masked_vector_from_array_u32(vd, &vd_data); + else + set_vector_from_array_u32(vd, &vd_data); + break; + case 3: + get_vector_as_array_u64(vs1, &vs1_data); + get_vector_as_array_u64(vs2, &vs2_data); + for (int i = 0; i < vl; i++) + vd_data.u64[i] = op(vs1_data.u64[i], vs2_data.u64[i]); + if (masked) + set_masked_vector_from_array_u64(vd, &vd_data); + else + set_vector_from_array_u64(vd, &vd_data); + break; + } +} + +static inline void foreach_velem_vi(int vl, int sew, bool masked, int vd, + u64 imm, int vs2, u64 op(u64, u64)) +{ + sbi_vector_data vs2_data; + sbi_vector_data vd_data; + + /* treat as no-op if VL is 0 */ + if (vl == 0) + return; + + switch (sew) { + case 0: + get_vector_as_array_u8(vs2, &vs2_data); + for (int i = 0; i < vl; i++) + vd_data.u8[i] = op(imm, vs2_data.u8[i]); + if (masked) + set_masked_vector_from_array_u8(vd, &vd_data); + else + set_vector_from_array_u8(vd, &vd_data); + break; + case 1: + get_vector_as_array_u16(vs2, &vs2_data); + for (int i = 0; i < vl; i++) + vd_data.u16[i] = op(imm, vs2_data.u16[i]); + if (masked) + set_masked_vector_from_array_u16(vd, &vd_data); + else + set_vector_from_array_u16(vd, &vd_data); + break; + case 2: + get_vector_as_array_u32(vs2, &vs2_data); + for (int i = 0; i < vl; i++) + vd_data.u32[i] = op(imm, vs2_data.u32[i]); + if (masked) + set_masked_vector_from_array_u32(vd, &vd_data); + else + set_vector_from_array_u32(vd, &vd_data); + break; + case 3: + get_vector_as_array_u64(vs2, &vs2_data); + for (int i = 0; i < vl; i++) + vd_data.u64[i] = op(imm, vs2_data.u64[i]); + if (masked) + set_masked_vector_from_array_u64(vd, &vd_data); + else + set_vector_from_array_u64(vd, &vd_data); + break; + } +} + +static inline void foreach_velem_v(int vl, int sew, bool masked, int vd, + int vs2, u64 op(u64)) +{ + sbi_vector_data vs2_data; + sbi_vector_data vd_data; + + /* treat as no-op if VL is 0 */ + if (vl == 0) + return; + + switch (sew) { + case 0: + get_vector_as_array_u8(vs2, &vs2_data); + for (int i = 0; i < vl; i++) + vd_data.u8[i] = op(vs2_data.u8[i]); + if (masked) + set_masked_vector_from_array_u8(vd, &vd_data); + else + set_vector_from_array_u8(vd, &vd_data); + break; + case 1: + get_vector_as_array_u16(vs2, &vs2_data); + for (int i = 0; i < vl; i++) + vd_data.u16[i] = op(vs2_data.u16[i]); + if (masked) + set_masked_vector_from_array_u16(vd, &vd_data); + else + set_vector_from_array_u16(vd, &vd_data); + break; + case 2: + get_vector_as_array_u32(vs2, &vs2_data); + for (int i = 0; i < vl; i++) + vd_data.u32[i] = op(vs2_data.u32[i]); + if (masked) + set_masked_vector_from_array_u32(vd, &vd_data); + else + set_vector_from_array_u32(vd, &vd_data); + break; + case 3: + get_vector_as_array_u64(vs2, &vs2_data); + for (int i = 0; i < vl; i++) + vd_data.u64[i] = op(vs2_data.u64[i]); + if (masked) + set_masked_vector_from_array_u64(vd, &vd_data); + else + set_vector_from_array_u64(vd, &vd_data); + break; + } +} + +static inline bool foreach_velem_wvv(int vl, int sew, bool masked, int vd, + int vs1, int vs2, u64 op(u64, u64)) +{ + sbi_vector_data vs1_data; + sbi_vector_data vs2_data; + sbi_vector_data vd_data; + + /* treat as no-op if VL is 0 */ + if (vl == 0) + return true; + /* back out if this VL combined with the widened SEW is too big */ + if (vl * (2 << sew) > VLMAX_BYTES) + return false; + + switch (sew) { + case 0: + get_vector_as_array_u8(vs1, &vs1_data); + get_vector_as_array_u8(vs2, &vs2_data); + for (int i = 0; i < vl; i++) + vd_data.u16[i] = op(vs1_data.u8[i], vs2_data.u8[i]); + if (masked) + set_masked_vector_from_array_u16(vd, &vd_data); + else + set_vector_from_array_u16(vd, &vd_data); + break; + case 1: + get_vector_as_array_u16(vs1, &vs1_data); + get_vector_as_array_u16(vs2, &vs2_data); + for (int i = 0; i < vl; i++) + vd_data.u32[i] = op(vs1_data.u16[i], vs2_data.u16[i]); + if (masked) + set_masked_vector_from_array_u32(vd, &vd_data); + else + set_vector_from_array_u32(vd, &vd_data); + break; + case 2: + get_vector_as_array_u32(vs1, &vs1_data); + get_vector_as_array_u32(vs2, &vs2_data); + for (int i = 0; i < vl; i++) + vd_data.u64[i] = op(vs1_data.u32[i], vs2_data.u32[i]); + if (masked) + set_masked_vector_from_array_u64(vd, &vd_data); + else + set_vector_from_array_u64(vd, &vd_data); + break; + } + return true; +} + +static inline bool foreach_velem_wvi(int vl, int sew, bool masked, int vd, + u64 imm, int vs2, u64 op(u64, u64)) +{ + sbi_vector_data vs2_data; + sbi_vector_data vd_data; + + /* treat as no-op if VL is 0 */ + if (vl == 0) + return true; + /* back out if this VL combined with the widened SEW is too big */ + if (vl * (2 << sew) > VLMAX_BYTES) + return false; + + switch (sew) { + case 0: + get_vector_as_array_u8(vs2, &vs2_data); + for (int i = 0; i < vl; i++) + vd_data.u16[i] = op(imm, vs2_data.u8[i]); + if (masked) + set_masked_vector_from_array_u16(vd, &vd_data); + else + set_vector_from_array_u16(vd, &vd_data); + break; + case 1: + get_vector_as_array_u16(vs2, &vs2_data); + for (int i = 0; i < vl; i++) + vd_data.u32[i] = op(imm, vs2_data.u16[i]); + if (masked) + set_masked_vector_from_array_u32(vd, &vd_data); + else + set_vector_from_array_u32(vd, &vd_data); + break; + case 2: + get_vector_as_array_u32(vs2, &vs2_data); + for (int i = 0; i < vl; i++) + vd_data.u64[i] = op(imm, vs2_data.u32[i]); + if (masked) + set_masked_vector_from_array_u64(vd, &vd_data); + else + set_vector_from_array_u64(vd, &vd_data); + break; + } + return true; +} + +static inline u64 op_andn(u64 op1, u64 op2) +{ + return ~op1 & op2; +} + +static inline u64 op_rol_u8(u64 op1, u64 op2) +{ + op2 &= 0xff; + op1 &= 7; + return ((op2 << op1) | (op2 >> (8 - op1))) & 0xff; +} + +static inline u64 op_rol_u16(u64 op1, u64 op2) +{ + op2 &= 0xffff; + op1 &= 0xf; + return ((op2 << op1) | (op2 >> (16 - op1))) & 0xffff; +} + +static inline u64 op_rol_u32(u64 op1, u64 op2) +{ + op2 &= 0xffffffff; + op1 &= 0x1f; + return ((op2 << op1) | (op2 >> (32 - op1))) & 0xffffffff; +} + +static inline u64 op_rol_u64(u64 op1, u64 op2) +{ + op1 &= 0x3f; + return (op2 << op1) | (op2 >> (64 - op1)); +} + +typeof(u64(u64, u64)) *ops_rol[4] = { op_rol_u8, op_rol_u16, op_rol_u32, + op_rol_u64 }; + +static inline u64 op_ror_u8(u64 op1, u64 op2) +{ + op2 &= 0xff; + op1 &= 7; + return ((op2 >> op1) | (op2 << (8 - op1))) & 0xff; +} + +static inline u64 op_ror_u16(u64 op1, u64 op2) +{ + op2 &= 0xffff; + op1 &= 0xf; + return ((op2 >> op1) | (op2 << (16 - op1))) & 0xffff; +} + +static inline u64 op_ror_u32(u64 op1, u64 op2) +{ + op2 &= 0xffffffff; + op1 &= 0x1f; + return ((op2 >> op1) | (op2 << (32 - op1))) & 0xffffffff; +} + +static inline u64 op_ror_u64(u64 op1, u64 op2) +{ + op1 &= 0x3f; + return (op2 >> op1) | (op2 << (64 - op1)); +} + +typeof(u64(u64, u64)) *ops_ror[4] = { op_ror_u8, op_ror_u16, op_ror_u32, + op_ror_u64 }; + +static inline u64 op_wsll_u8(u64 op1, u64 op2) +{ + op1 &= 0xf; + return (op2 << op1) & 0xffff; +} + +static inline u64 op_wsll_u16(u64 op1, u64 op2) +{ + op1 &= 0x1f; + return (op2 << op1) & 0xffffffff; +} + +static inline u64 op_wsll_u32(u64 op1, u64 op2) +{ + op1 &= 0x3f; + return op2 << op1; +} + +typeof(u64(u64, u64)) *ops_wsll[4] = { op_wsll_u8, op_wsll_u16, op_wsll_u32, + op_wsll_u32 }; + +static inline u64 op_brev8(u64 op) +{ + return ((op & 0x8080808080808080) >> 7) | + ((op & 0x4040404040404040) >> 5) | + ((op & 0x2020202020202020) >> 3) | + ((op & 0x1010101010101010) >> 1) | + ((op & 0x0808080808080808) << 1) | + ((op & 0x0404040404040404) << 3) | + ((op & 0x0202020202020202) << 5) | + ((op & 0x0101010101010101) << 7); +} + +static inline u64 op_rev8(u64 op1, u64 op2) +{ + u64 result; + asm volatile(".option push\n\t" + ".option arch, +zbb\n\t" + "rev8 %0, %2\n\t" + "srl %0, %0, %1\n\t" + ".option pop\n\t" + : "=&r"(result) + : "r"(op1), "r"(op2)); + return result; +} + +static inline u64 op_brev(u64 op1, u64 op2) +{ + return op_rev8(op1, op_brev8(op2)); +} + +static inline u64 op_clz(u64 op1, u64 op2) +{ + u64 result; + asm volatile(".option push\n\t" + ".option arch, +zbb\n\t" + "clz %0, %2\n\t" + "sub %0, %0, %1\n\t" + ".option pop\n\t" + : "=&r"(result) + : "r"(op1), "r"(op2)); + return result; +} + +static inline u64 op_ctz(u64 op1, u64 op2) +{ + u64 result; + asm volatile(".option push\n\t" + ".option arch, +zbb\n\t" + "ctz %0, %2\n\t" + "minu %0, %0, %1\n\t" + ".option pop\n\t" + : "=&r"(result) + : "r"(op1), "r"(op2)); + return result; +} + +static inline u64 op_cpop(u64 op) +{ + u64 result; + asm volatile(".option push\n\t" + ".option arch, +zbb\n\t" + "cpop %0, %1\n\t" + ".option pop\n\t" + : "=r"(result) + : "r"(op)); + return result; +} + +int sbi_insn_emu_op_v(ulong insn, struct sbi_trap_regs *regs) +{ + /* back out if vector unit is not available */ + if ((regs->mstatus & MSTATUS_VS) == 0 || + (sbi_mstatus_prev_mode(regs->mstatus) == PRV_U && + (csr_read(CSR_SSTATUS) & SSTATUS_VS) == 0)) + return truly_illegal_insn(insn, regs); + + int vl = csr_read(CSR_VL); + int vs1 = GET_VS1(insn); + int vs2 = GET_VS2(insn); + int vd = GET_VD(insn); + u32 vtype = csr_read(CSR_VTYPE); + int sew = GET_VSEW(vtype); + bool m = IS_MASKED(insn); + u64 rs1 = GET_RS1(insn, regs); + + /* back out if this VL combined with this SEW is too big */ + if (vl * (1 << sew) > VLMAX_BYTES) + return truly_illegal_insn(insn, regs); + + switch (insn & INSN_MASK_VXUNARY0) { + /* Emulate Zvbb unary operations */ + case INSN_MATCH_VBREVV: + foreach_velem_vi(vl, sew, m, vd, 64 - (8 << sew), vs2, op_brev); + break; + case INSN_MATCH_VBREV8V: + foreach_velem_v(vl, sew, m, vd, vs2, op_brev8); + break; + case INSN_MATCH_VREV8V: + foreach_velem_vi(vl, sew, m, vd, 64 - (8 << sew), vs2, op_rev8); + break; + case INSN_MATCH_VCLZV: + foreach_velem_vi(vl, sew, m, vd, 64 - (8 << sew), vs2, op_clz); + break; + case INSN_MATCH_VCTZV: + foreach_velem_vi(vl, sew, m, vd, 8 << sew, vs2, op_ctz); + break; + case INSN_MATCH_VCPOPV: + foreach_velem_v(vl, sew, m, vd, vs2, op_cpop); + break; + default: + switch (insn & INSN_MASK_VVBINARY0) { + /* Emulate Zvbb binary operations */ + case INSN_MATCH_VANDNVV: + foreach_velem_vv(vl, sew, m, vd, vs1, vs2, op_andn); + break; + case INSN_MATCH_VANDNVX: + foreach_velem_vi(vl, sew, m, vd, rs1, vs2, op_andn); + break; + case INSN_MATCH_VROLVV: + foreach_velem_vv(vl, sew, m, vd, vs1, vs2, + ops_rol[sew]); + break; + case INSN_MATCH_VROLVX: + foreach_velem_vi(vl, sew, m, vd, rs1, vs2, + ops_rol[sew]); + break; + case INSN_MATCH_VRORVV: + foreach_velem_vv(vl, sew, m, vd, vs1, vs2, + ops_ror[sew]); + break; + case INSN_MATCH_VRORVX: + foreach_velem_vi(vl, sew, m, vd, rs1, vs2, + ops_ror[sew]); + break; + case INSN_MATCH_VRORVI: + case INSN_MATCH_VRORVI | 0x04000000: + foreach_velem_vi(vl, sew, m, vd, + GET_RS1_NUM(insn) | + ((insn & 0x04000000) >> 21), + vs2, ops_ror[sew]); + break; + case INSN_MATCH_VWSLLVV: + if (!foreach_velem_wvv(vl, sew, m, vd, vs1, vs2, + ops_wsll[sew])) + return truly_illegal_insn(insn, regs); + break; + case INSN_MATCH_VWSLLVX: + if (!foreach_velem_wvi(vl, sew, m, vd, rs1, vs2, + ops_wsll[sew])) + return truly_illegal_insn(insn, regs); + break; + case INSN_MATCH_VWSLLVI: + if (!foreach_velem_wvi(vl, sew, m, vd, + GET_RS1_NUM(insn), vs2, + ops_wsll[sew])) + return truly_illegal_insn(insn, regs); + break; + default: + return truly_illegal_insn(insn, regs); + } + } + + if (sbi_regs_from_virt(regs)) + csr_set(CSR_VSSTATUS, MSTATUS_VS); + regs->mstatus |= MSTATUS_VS; + + regs->mepc += 4; + + return 0; +} + +#endif diff --git a/lib/sbi/sbi_irqchip.c b/lib/sbi/sbi_irqchip.c index 0594e05a3f0..6d0df02ed38 100644 --- a/lib/sbi/sbi_irqchip.c +++ b/lib/sbi/sbi_irqchip.c @@ -7,53 +7,508 @@ * Anup Patel */ +#include +#include +#include #include #include #include +#include +/** Internal irqchip hardware interrupt data */ +struct sbi_irqchip_hwirq_data { + /** raw hardware interrupt handler */ + int (*raw_handler)(struct sbi_irqchip_device *chip, u32 hwirq); + + /** target hart index */ + u32 hart_index; +}; + +/** Internal irqchip interrupt handler */ +struct sbi_irqchip_handler { + /** Node in the list of irqchip handlers (private) */ + struct sbi_dlist node; + + /** First hardware IRQ handled by this handler */ + u32 first_hwirq; + + /** Number of consecutive hardware IRQs handled by this handler */ + u32 num_hwirq; + + /** Write MSI function of this handler */ + void (*write_msi)(u32 hwirq, const struct sbi_irqchip_msi_msg *msg, void *priv); + + /** Callback function of this handler */ + int (*callback)(u32 hwirq, void *priv); + + /** Callback private data */ + void *priv; +}; + +struct sbi_irqchip_hart_data { + struct sbi_irqchip_device *chip; +}; + +static unsigned long irqchip_hart_data_off; static SBI_LIST_HEAD(irqchip_list); -static int default_irqfn(void) +int sbi_irqchip_process(void) +{ + struct sbi_irqchip_hart_data *hd; + + hd = sbi_scratch_thishart_offset_ptr(irqchip_hart_data_off); + if (!hd || !hd->chip || !hd->chip->process_hwirqs) + return SBI_ENODEV; + + return hd->chip->process_hwirqs(hd->chip); +} + +int sbi_irqchip_process_hwirq(struct sbi_irqchip_device *chip, u32 hwirq) { - return SBI_ENODEV; + struct sbi_irqchip_hwirq_data *data; + + if (!chip || chip->num_hwirq <= hwirq) + return SBI_EINVAL; + + data = &chip->hwirqs[hwirq]; + if (!data->raw_handler) + return SBI_ENOENT; + + return data->raw_handler(chip, hwirq); } -static int (*ext_irqfn)(void) = default_irqfn; +int sbi_irqchip_unmask_hwirq(struct sbi_irqchip_device *chip, u32 hwirq) +{ + if (!chip || chip->num_hwirq <= hwirq) + return SBI_EINVAL; -int sbi_irqchip_process(void) + if (chip->hwirq_unmask) + chip->hwirq_unmask(chip, hwirq); + return 0; +} + +int sbi_irqchip_mask_hwirq(struct sbi_irqchip_device *chip, u32 hwirq) { - return ext_irqfn(); + if (!chip || chip->num_hwirq <= hwirq) + return SBI_EINVAL; + + if (chip->hwirq_mask) + chip->hwirq_mask(chip, hwirq); + return 0; } -void sbi_irqchip_add_device(struct sbi_irqchip_device *dev) +static struct sbi_irqchip_handler *sbi_irqchip_find_handler(struct sbi_irqchip_device *chip, + u32 hwirq) { - sbi_list_add_tail(&dev->node, &irqchip_list); + struct sbi_irqchip_handler *h; - if (dev->irq_handle) - ext_irqfn = dev->irq_handle; + if (!chip || chip->num_hwirq <= hwirq) + return NULL; + + sbi_list_for_each_entry(h, &chip->handler_list, node) { + if (h->first_hwirq <= hwirq && hwirq < (h->first_hwirq + h->num_hwirq)) + return h; + } + + return NULL; } -int sbi_irqchip_init(struct sbi_scratch *scratch, bool cold_boot) +int sbi_irqchip_raw_handler_default(struct sbi_irqchip_device *chip, u32 hwirq) +{ + struct sbi_irqchip_handler *h; + int rc = SBI_OK; + + if (!chip || chip->num_hwirq <= hwirq) + return SBI_EINVAL; + + h = sbi_irqchip_find_handler(chip, hwirq); + if (h->callback) + rc = h->callback(hwirq, h->priv); + + if (chip->hwirq_eoi) + chip->hwirq_eoi(chip, hwirq); + + return rc; +} + +int sbi_irqchip_set_raw_handler(struct sbi_irqchip_device *chip, u32 hwirq, + int (*raw_hndl)(struct sbi_irqchip_device *, u32)) +{ + struct sbi_irqchip_hwirq_data *data; + + if (!chip || chip->num_hwirq <= hwirq) + return SBI_EINVAL; + + data = &chip->hwirqs[hwirq]; + data->raw_handler = raw_hndl; + return 0; +} + +int sbi_irqchip_write_msi(struct sbi_irqchip_device *chip, u32 hwirq, + const struct sbi_irqchip_msi_msg *msg) +{ + struct sbi_irqchip_handler *h; + + if (!chip || chip->num_hwirq <= hwirq || !msg) + return SBI_EINVAL; + + h = sbi_irqchip_find_handler(chip, hwirq); + if (!h) + return SBI_EFAIL; + if (!h->write_msi) + return SBI_ENOTSUPP; + + h->write_msi(hwirq, msg, h->priv); + return 0; +} + +int sbi_irqchip_get_affinity(struct sbi_irqchip_device *chip, u32 hwirq, + u32 *out_hart_index) +{ + if (!chip || chip->num_hwirq <= hwirq) + return SBI_EINVAL; + + /* + * If no handler registered for hwirq then hwirq + * is not being used so return failure + */ + if (!sbi_irqchip_find_handler(chip, hwirq)) + return SBI_ENOTSUPP; + + *out_hart_index = chip->hwirqs[hwirq].hart_index; + return 0; +} + +int sbi_irqchip_set_affinity(struct sbi_irqchip_device *chip, u32 hwirq, + u32 hart_index) +{ + struct sbi_irqchip_hwirq_data *data; + int rc; + + if (!chip || chip->num_hwirq <= hwirq || sbi_hart_count() <= hart_index) + return SBI_EINVAL; + + /* + * If no handler registered for hwirq then hwirq + * is not being used so return failure + */ + if (!sbi_irqchip_find_handler(chip, hwirq)) + return SBI_ENOTSUPP; + + data = &chip->hwirqs[hwirq]; + if (data->hart_index != hart_index) { + if (chip->hwirq_set_affinity) { + rc = chip->hwirq_set_affinity(chip, hwirq, hart_index); + if (rc) + return rc; + } + data->hart_index = hart_index; + } + + return 0; +} + +static int __sbi_irqchip_handler_set_affinity(struct sbi_irqchip_device *chip, + struct sbi_irqchip_handler *h, + u32 compare_hart_index, + u32 hart_index) +{ + u32 i, current_hart_index; + int rc; + + for (i = 0; i < h->num_hwirq; i++) { + rc = sbi_irqchip_get_affinity(chip, h->first_hwirq + i, + ¤t_hart_index); + if (rc) + return rc; + + if (compare_hart_index != -1U && + current_hart_index != compare_hart_index) + continue; + + rc = sbi_irqchip_set_affinity(chip, h->first_hwirq + i, hart_index); + if (rc) + return rc; + } + + return 0; +} + +static int __sbi_irqchip_register_handler(struct sbi_irqchip_device *chip, + u32 first_hwirq, u32 num_hwirq, u32 hwirq_flags, + void (*write_msi)(u32 hwirq, + const struct sbi_irqchip_msi_msg *msg, + void *priv), + int (*callback)(u32 hwirq, void *priv), void *priv) { + struct sbi_irqchip_handler *h, *th, *nh; + u32 i, j; int rc; + + for (i = first_hwirq; i < (first_hwirq + num_hwirq); i++) { + h = sbi_irqchip_find_handler(chip, i); + if (h) + return SBI_EALREADY; + } + + h = sbi_zalloc(sizeof(*h)); + if (!h) + return SBI_ENOMEM; + h->first_hwirq = first_hwirq; + h->num_hwirq = num_hwirq; + h->write_msi = write_msi; + h->callback = callback; + h->priv = priv; + + nh = NULL; + sbi_list_for_each_entry(th, &chip->handler_list, node) { + if (h->first_hwirq < th->first_hwirq) { + nh = th; + break; + } + } + if (nh) + sbi_list_add(&h->node, &nh->node); + else + sbi_list_add_tail(&h->node, &chip->handler_list); + + if (chip->hwirq_setup) { + for (i = 0; i < h->num_hwirq; i++) { + rc = chip->hwirq_setup(chip, h->first_hwirq + i, hwirq_flags); + if (rc) { + if (chip->hwirq_cleanup) { + for (j = 0; j < i; j++) + chip->hwirq_cleanup(chip, h->first_hwirq + j); + } + sbi_list_del(&h->node); + sbi_free(h); + return rc; + } + } + } + + rc = __sbi_irqchip_handler_set_affinity(chip, h, -1U, current_hartindex()); + if (rc) { + if (chip->hwirq_cleanup) { + for (i = 0; i < h->num_hwirq; i++) + chip->hwirq_cleanup(chip, h->first_hwirq + i); + } + sbi_list_del(&h->node); + sbi_free(h); + return rc; + } + + if (chip->hwirq_unmask) { + for (i = 0; i < h->num_hwirq; i++) + chip->hwirq_unmask(chip, h->first_hwirq + i); + } + + return 0; +} + +int sbi_irqchip_register_msi(struct sbi_irqchip_device *chip, u32 num_hwirq, + void (*write_msi)(u32 hwirq, + const struct sbi_irqchip_msi_msg *msg, + void *priv), + int (*callback)(u32 hwirq, void *priv), void *priv, + u32 *out_first_hwirq) +{ + struct sbi_irqchip_handler *h; + bool found; + u32 hwirq; + + if (!chip || !chip->hwirq_set_affinity || !num_hwirq || + !write_msi || !callback || !out_first_hwirq) + return SBI_EINVAL; + if (chip->num_hwirq < num_hwirq) + return SBI_EBAD_RANGE; + + hwirq = 0; + found = false; + sbi_list_for_each_entry(h, &chip->handler_list, node) { + if (h->first_hwirq <= hwirq && hwirq < (h->first_hwirq + h->num_hwirq)) { + hwirq = h->first_hwirq + h->num_hwirq; + } else if (hwirq < h->first_hwirq) { + if (h->first_hwirq - hwirq < num_hwirq) { + found = true; + break; + } else { + hwirq = h->first_hwirq + h->num_hwirq; + } + } + } + if (!found && !hwirq) + found = true; + if (!found) + return SBI_ENOSPC; + *out_first_hwirq = hwirq; + + return __sbi_irqchip_register_handler(chip, *out_first_hwirq, + num_hwirq, SBI_HWIRQ_FLAGS_NONE, + write_msi, callback, priv); +} + +int sbi_irqchip_register_handler(struct sbi_irqchip_device *chip, + u32 first_hwirq, u32 num_hwirq, u32 hwirq_flags, + int (*callback)(u32 hwirq, void *priv), void *priv) +{ + if (!chip || !num_hwirq || !callback) + return SBI_EINVAL; + if (chip->num_hwirq <= first_hwirq || + chip->num_hwirq <= (first_hwirq + num_hwirq - 1)) + return SBI_EBAD_RANGE; + + return __sbi_irqchip_register_handler(chip, first_hwirq, num_hwirq, hwirq_flags, + NULL, callback, priv); +} + +int sbi_irqchip_register_reserved(struct sbi_irqchip_device *chip, + u32 first_hwirq, u32 num_hwirq) +{ + if (!chip || !num_hwirq) + return SBI_EINVAL; + if (chip->num_hwirq <= first_hwirq || + chip->num_hwirq <= (first_hwirq + num_hwirq - 1)) + return SBI_EBAD_RANGE; + + return __sbi_irqchip_register_handler(chip, first_hwirq, num_hwirq, + SBI_HWIRQ_FLAGS_NONE, NULL, NULL, NULL); +} + +int sbi_irqchip_unregister_handler(struct sbi_irqchip_device *chip, + u32 first_hwirq, u32 num_hwirq) +{ + struct sbi_irqchip_handler *fh, *lh; + u32 i; + + if (!chip || !num_hwirq) + return SBI_EINVAL; + if (chip->num_hwirq <= first_hwirq || + chip->num_hwirq <= (first_hwirq + num_hwirq - 1)) + return SBI_EBAD_RANGE; + + fh = sbi_irqchip_find_handler(chip, first_hwirq); + if (!fh || fh->first_hwirq != first_hwirq || fh->num_hwirq != num_hwirq) + return SBI_ENODEV; + lh = sbi_irqchip_find_handler(chip, first_hwirq + num_hwirq - 1); + if (!lh || lh != fh) + return SBI_ENODEV; + + if (chip->hwirq_mask) { + for (i = 0; i < fh->num_hwirq; i++) + chip->hwirq_mask(chip, fh->first_hwirq + i); + } + + if (chip->hwirq_cleanup) { + for (i = 0; i < fh->num_hwirq; i++) + chip->hwirq_cleanup(chip, fh->first_hwirq + i); + } + + sbi_list_del(&fh->node); + return 0; +} + +struct sbi_irqchip_device *sbi_irqchip_find_device_by_caps(unsigned long caps, + struct sbi_irqchip_device *first) +{ + struct sbi_irqchip_device *chip; + bool found = false; + + sbi_list_for_each_entry(chip, &irqchip_list, node) { + if (!found) { + if (first == chip) + found = true; + else + continue; + } + if ((chip->caps & caps) == caps) + return chip; + } + + return NULL; +} + +struct sbi_irqchip_device *sbi_irqchip_find_device(u32 id) +{ + struct sbi_irqchip_device *chip; + + sbi_list_for_each_entry(chip, &irqchip_list, node) { + if (chip->id == id) + return chip; + } + + return NULL; +} + +int sbi_irqchip_add_device(struct sbi_irqchip_device *chip) +{ + struct sbi_irqchip_hart_data *hd; + struct sbi_scratch *scratch; + u32 i, h; + + if (!chip || !chip->num_hwirq || !sbi_hartmask_weight(&chip->target_harts)) + return SBI_EINVAL; + + if (sbi_irqchip_find_device(chip->id)) + return SBI_EALREADY; + + if (chip->process_hwirqs) { + sbi_hartmask_for_each_hartindex(h, &chip->target_harts) { + scratch = sbi_hartindex_to_scratch(h); + if (!scratch) + continue; + + hd = sbi_scratch_offset_ptr(scratch, irqchip_hart_data_off); + if (hd->chip && hd->chip != chip) + return SBI_EINVAL; + + hd->chip = chip; + } + } + + chip->hwirqs = sbi_zalloc(sizeof(*chip->hwirqs) * chip->num_hwirq); + if (!chip->hwirqs) + return SBI_ENOMEM; + for (i = 0; i < chip->num_hwirq; i++) { + sbi_irqchip_set_raw_handler(chip, i, sbi_irqchip_raw_handler_default); + chip->hwirqs[i].hart_index = -1U; + } + + SBI_INIT_LIST_HEAD(&chip->handler_list); + + sbi_list_add_tail(&chip->node, &irqchip_list); + return 0; +} + +int sbi_irqchip_init(struct sbi_scratch *scratch, bool cold_boot) +{ const struct sbi_platform *plat = sbi_platform_ptr(scratch); - struct sbi_irqchip_device *dev; + struct sbi_irqchip_hart_data *hd; + struct sbi_irqchip_device *chip; + int rc; if (cold_boot) { + irqchip_hart_data_off = + sbi_scratch_alloc_offset(sizeof(struct sbi_irqchip_hart_data)); + if (!irqchip_hart_data_off) + return SBI_ENOMEM; rc = sbi_platform_irqchip_init(plat); if (rc) return rc; } - sbi_list_for_each_entry(dev, &irqchip_list, node) { - if (!dev->warm_init) + sbi_list_for_each_entry(chip, &irqchip_list, node) { + if (!chip->warm_init) continue; - rc = dev->warm_init(dev); + if (!sbi_hartmask_test_hartindex(current_hartindex(), &chip->target_harts)) + continue; + rc = chip->warm_init(chip); if (rc) return rc; } - if (ext_irqfn != default_irqfn) + hd = sbi_scratch_thishart_offset_ptr(irqchip_hart_data_off); + if (hd && hd->chip && hd->chip->process_hwirqs) csr_set(CSR_MIE, MIP_MEIP); return 0; @@ -61,6 +516,40 @@ int sbi_irqchip_init(struct sbi_scratch *scratch, bool cold_boot) void sbi_irqchip_exit(struct sbi_scratch *scratch) { - if (ext_irqfn != default_irqfn) + struct sbi_irqchip_hart_data *hd; + struct sbi_irqchip_device *chip; + struct sbi_irqchip_handler *h; + u32 migrate_hidx = -1U; + bool migrate = false; + int rc; + + sbi_for_each_hartindex(i) { + if (i == current_hartindex()) + continue; + if (__sbi_hsm_hart_get_state(i) == SBI_HSM_STATE_STOPPED || + __sbi_hsm_hart_get_state(i) == SBI_HSM_STATE_STOP_PENDING) + continue; + migrate_hidx = i; + migrate = true; + break; + } + + if (!migrate) + goto skip_migrate; + sbi_list_for_each_entry(chip, &irqchip_list, node) { + sbi_list_for_each_entry(h, &chip->handler_list, node) { + rc = __sbi_irqchip_handler_set_affinity(chip, h, + current_hartindex(), + migrate_hidx); + if (rc) { + sbi_printf("%s: chip 0x%x handler 0x%x set affinity (err %d)\n", + __func__, chip->id, h->first_hwirq, rc); + } + } + } +skip_migrate: + + hd = sbi_scratch_thishart_offset_ptr(irqchip_hart_data_off); + if (hd && hd->chip && hd->chip->process_hwirqs) csr_clear(CSR_MIE, MIP_MEIP); } diff --git a/lib/sbi/sbi_mpxy.c b/lib/sbi/sbi_mpxy.c index a83cf16c006..ff5c6ab6b4e 100644 --- a/lib/sbi/sbi_mpxy.c +++ b/lib/sbi/sbi_mpxy.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -28,7 +29,7 @@ static unsigned long mpxy_shmem_size = PAGE_SIZE; static SBI_LIST_HEAD(mpxy_channel_list); /** Invalid Physical Address(all bits 1) */ -#define INVALID_ADDR (-1U) +#define INVALID_ADDR (-1UL) /** MPXY Attribute size in bytes */ #define ATTR_SIZE (4) @@ -255,11 +256,10 @@ static int domain_mpxy_state_data_setup(struct sbi_domain *dom, return SBI_ENOMEM; /* - * TODO: Proper support for checking msi support from - * platform. Currently disable msi and sse and use - * polling + * TODO: Proper support for checking sse support from + * platform. Currently disable sse and use polling */ - ms->msi_avail = false; + ms->msi_avail = !!sbi_irqchip_find_device_by_caps(SBI_IRQCHIP_CAPS_MSI, NULL); ms->sse_avail = false; sbi_mpxy_shmem_disable(ms); @@ -531,8 +531,8 @@ static int mpxy_check_write_std_attr(struct sbi_mpxy_channel *channel, if (attr_val > 1) ret = SBI_ERR_INVALID_PARAM; if (attr_val == 1 && - (attrs->msi_info.msi_addr_lo == INVALID_ADDR) && - (attrs->msi_info.msi_addr_hi == INVALID_ADDR)) + (attrs->msi_info.msi_addr_lo == (u32)INVALID_ADDR) && + (attrs->msi_info.msi_addr_hi == (u32)INVALID_ADDR)) ret = SBI_ERR_DENIED; break; case SBI_MPXY_ATTR_MSI_ADDR_LO: diff --git a/lib/sbi/sbi_pmp.c b/lib/sbi/sbi_pmp.c new file mode 100644 index 00000000000..fd1a0fd0043 --- /dev/null +++ b/lib/sbi/sbi_pmp.c @@ -0,0 +1,105 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2019 Western Digital Corporation or its affiliates. + * + * Authors: + * Anup Patel + */ + +#include +#include +#include +#include +#include + +static unsigned long ctz(unsigned long x) +{ + unsigned long ret = 0; + + if (x == 0) + return 8 * sizeof(x); + + while (!(x & 1UL)) { + ret++; + x = x >> 1; + } + + return ret; +} + +bool sbi_pmp_is_enabled(pmp_t *pmp) +{ + /* If address matching bits are non-zero, the entry is enable */ + if (pmp->cfg & PMP_A) + return true; + + return false; +} + +int sbi_pmp_encode(pmp_t *pmp, unsigned long prot, unsigned long addr, + unsigned long log2len) +{ + /* check parameters */ + if (log2len > __riscv_xlen || log2len < PMP_SHIFT) + return SBI_EINVAL; + + /* encode PMP config */ + prot &= ~PMP_A; + prot |= (log2len == PMP_SHIFT) ? PMP_A_NA4 : PMP_A_NAPOT; + pmp->cfg = prot; + + /* encode PMP address */ + if (log2len == PMP_SHIFT) { + pmp->addr = (addr >> PMP_SHIFT); + } else { + if (log2len == __riscv_xlen) { + pmp->addr = -1UL; + } else { + unsigned long addrmask; + addrmask = (1UL << (log2len - PMP_SHIFT)) - 1; + pmp->addr = ((addr >> PMP_SHIFT) & ~addrmask); + pmp->addr |= (addrmask >> 1); + } + } + + return SBI_OK; +} + +int sbi_pmp_decode(pmp_t *pmp, unsigned long *prot_out, unsigned long *addr_out, + unsigned long *log2len) +{ + unsigned long prot; + unsigned long t1, addr, len; + + /* check parameters */ + if (!prot_out || !addr_out || !log2len) + return SBI_EINVAL; + *prot_out = *addr_out = *log2len = 0; + + /* decode PMP config */ + prot = pmp->cfg; + + /* decode PMP address */ + if ((prot & PMP_A) == PMP_A_NAPOT) { + addr = pmp->addr; + if (addr == -1UL) { + addr = 0; + len = __riscv_xlen; + } else { + t1 = ctz(~addr); + addr = (addr & ~((1UL << t1) - 1)) << PMP_SHIFT; + len = (t1 + PMP_SHIFT + 1); + } + } else { + addr = pmp->addr << PMP_SHIFT; + len = PMP_SHIFT; + } + + /* return details */ + *prot_out = prot; + *addr_out = addr; + *log2len = len; + + return SBI_OK; +} diff --git a/lib/sbi/sbi_pmu.c b/lib/sbi/sbi_pmu.c index e084005d11f..a0f6d2facb0 100644 --- a/lib/sbi/sbi_pmu.c +++ b/lib/sbi/sbi_pmu.c @@ -223,11 +223,19 @@ static int pmu_ctr_validate(struct sbi_pmu_hart_state *phs, static bool pmu_ctr_idx_validate(unsigned long cbase, unsigned long cmask) { - /* Do a basic sanity check of counter base & mask */ - return cmask && cbase + sbi_fls(cmask) < total_ctrs; + unsigned long last; + + if (!cmask) + return false; + + last = sbi_fls(cmask); + if (cbase > -1UL - last) + return false; + + return (cbase + last) < total_ctrs; } -int sbi_pmu_ctr_fw_read(uint32_t cidx, uint64_t *cval) +int sbi_pmu_ctr_fw_read(unsigned long cidx, uint64_t *cval, bool high_bits) { int event_idx_type; uint32_t event_code; @@ -236,6 +244,14 @@ int sbi_pmu_ctr_fw_read(uint32_t cidx, uint64_t *cval) if (unlikely(!phs)) return SBI_EINVAL; + if (cidx < num_hw_ctrs || cidx >= total_ctrs) + return SBI_EINVAL; + +#if __riscv_xlen > 32 + if (high_bits) + return 0; +#endif + event_idx_type = pmu_ctr_validate(phs, cidx, &event_code); if (event_idx_type != SBI_PMU_EVENT_TYPE_FW) return SBI_EINVAL; @@ -448,6 +464,8 @@ static int pmu_ctr_start_fw(struct sbi_pmu_hart_state *phs, uint64_t event_data, uint64_t ival, bool ival_update) { + int ret; + if ((event_code >= SBI_PMU_FW_MAX && event_code <= SBI_PMU_FW_RESERVED_MAX) || event_code > SBI_PMU_FW_PLATFORM) @@ -468,9 +486,11 @@ static int pmu_ctr_start_fw(struct sbi_pmu_hart_state *phs, cidx - num_hw_ctrs, ival); - return pmu_dev->fw_counter_start(phs->hartid, + ret = pmu_dev->fw_counter_start(phs->hartid, cidx - num_hw_ctrs, event_data); + if (ret) + return ret; } else { if (ival_update) phs->fw_counters_data[cidx - num_hw_ctrs] = ival; @@ -818,13 +838,20 @@ static int pmu_ctr_find_hw(struct sbi_pmu_hart_state *phs, if (ctr_idx == SBI_ENOTSUPP) { /** - * We can't find any programmable counters for cycle/instret. - * Return the fixed counter as they are mandatory anyways. + * We can't find a programmable counter, see if we can use a + * fixed counter instead if one was found for this event. + * + * If sscofpmf is present but smcntrpmf is not, we can't + * fallback to a fixed counter, because the fixed counter + * doesn't support filtering whereas a programmable counter + * would. */ - if (fixed_ctr >= 0) - return pmu_fixed_ctr_update_inhibit_bits(fixed_ctr, flags); - else + if (fixed_ctr < 0 || + ((sbi_hart_has_extension(scratch, SBI_HART_EXT_SSCOFPMF) && + !sbi_hart_has_extension(scratch, SBI_HART_EXT_SMCNTRPMF)))) return SBI_EFAIL; + + return pmu_fixed_ctr_update_inhibit_bits(fixed_ctr, flags); } ret = pmu_update_hw_mhpmevent(temp, ctr_idx, flags, event_idx, data); @@ -896,6 +923,9 @@ int sbi_pmu_ctr_cfg_match(unsigned long cidx_base, unsigned long cidx_mask, */ unsigned long cidx_first = cidx_base + sbi_ffs(cidx_mask); + if (cidx_first >= total_ctrs) + return SBI_EINVAL; + if (phs->active_events[cidx_first] == SBI_PMU_EVENT_IDX_INVALID) return SBI_EINVAL; ctr_idx = cidx_first; @@ -927,7 +957,10 @@ int sbi_pmu_ctr_cfg_match(unsigned long cidx_base, unsigned long cidx_mask, phs->active_events[ctr_idx] = event_idx; skip_match: - if (event_type == SBI_PMU_EVENT_TYPE_HW) { + if (event_type == SBI_PMU_EVENT_TYPE_HW || + event_type == SBI_PMU_EVENT_TYPE_HW_CACHE || + event_type == SBI_PMU_EVENT_TYPE_HW_RAW || + event_type == SBI_PMU_EVENT_TYPE_HW_RAW_V2) { if (flags & SBI_PMU_CFG_FLAG_CLEAR_VALUE) pmu_ctr_write_hw(ctr_idx, 0); if (flags & SBI_PMU_CFG_FLAG_AUTO_START) diff --git a/lib/sbi/sbi_sse.c b/lib/sbi/sbi_sse.c index a14754f8bb9..818afb871e0 100644 --- a/lib/sbi/sbi_sse.c +++ b/lib/sbi/sbi_sse.c @@ -548,7 +548,7 @@ static void sse_event_inject(struct sbi_sse_event *e, sse_event_set_state(e, SBI_SSE_STATE_RUNNING); - e->attrs.status = ~BIT(SBI_SSE_ATTR_STATUS_PENDING_OFFSET); + e->attrs.status &= ~BIT(SBI_SSE_ATTR_STATUS_PENDING_OFFSET); i_ctx->a6 = regs->a6; i_ctx->a7 = regs->a7; diff --git a/lib/sbi/sbi_string.c b/lib/sbi/sbi_string.c index f4f13942e80..3a10c254be7 100644 --- a/lib/sbi/sbi_string.c +++ b/lib/sbi/sbi_string.c @@ -88,13 +88,12 @@ char *sbi_strncpy(char *dest, const char *src, size_t count) char *sbi_strchr(const char *s, int c) { - while (*s != '\0' && *s != (char)c) - s++; + do { + if (*s == (char)c) + return (char *)s; + } while (*s++ != '\0'); - if (*s == '\0') - return NULL; - else - return (char *)s; + return NULL; } char *sbi_strrchr(const char *s, int c) diff --git a/lib/sbi/sbi_timer.c b/lib/sbi/sbi_timer.c index 4088a5970ed..4d737d22a18 100644 --- a/lib/sbi/sbi_timer.c +++ b/lib/sbi/sbi_timer.c @@ -10,15 +10,24 @@ #include #include #include +#include #include #include #include +#include #include #include #include #include -static unsigned long time_delta_off; +struct timer_state { + u64 time_delta; + spinlock_t event_list_lock; + struct sbi_dlist event_list; + struct sbi_timer_event smode_ev; +}; + +static unsigned long timer_state_off; static u64 (*get_time_val)(void); static const struct sbi_timer_device *timer_dev = NULL; @@ -49,6 +58,15 @@ static void nop_delay_fn(void *opaque) cpu_relax(); } +u64 sbi_timer_compute_delta(ulong units, u64 unit_freq) +{ + u64 delta; + + delta = ((u64)timer_dev->timer_freq * (u64)units); + delta = delta / unit_freq; + return delta; +} + void sbi_timer_delay_loop(ulong units, u64 unit_freq, void (*delay_fn)(void *), void *opaque) { @@ -63,15 +81,12 @@ void sbi_timer_delay_loop(ulong units, u64 unit_freq, /* Save starting timer value */ start_val = get_time_val(); - /* Compute desired timer value delta */ - delta = ((u64)timer_dev->timer_freq * (u64)units); - delta = delta / unit_freq; - /* Use NOP delay function if delay function not available */ if (!delay_fn) delay_fn = nop_delay_fn; /* Busy loop until desired timer value delta reached */ + delta = sbi_timer_compute_delta(units, unit_freq); while ((get_time_val() - start_val) < delta) delay_fn(opaque); } @@ -98,40 +113,164 @@ u64 sbi_timer_value(void) u64 sbi_timer_virt_value(void) { - u64 *time_delta = sbi_scratch_offset_ptr(sbi_scratch_thishart_ptr(), - time_delta_off); + struct timer_state *tstate = sbi_scratch_thishart_offset_ptr(timer_state_off); - return sbi_timer_value() + *time_delta; + return sbi_timer_value() + tstate->time_delta; } u64 sbi_timer_get_delta(void) { - u64 *time_delta = sbi_scratch_offset_ptr(sbi_scratch_thishart_ptr(), - time_delta_off); + struct timer_state *tstate = sbi_scratch_thishart_offset_ptr(timer_state_off); - return *time_delta; + return tstate->time_delta; } void sbi_timer_set_delta(ulong delta) { - ulong *time_delta = sbi_scratch_offset_ptr(sbi_scratch_thishart_ptr(), - time_delta_off); + struct timer_state *tstate = sbi_scratch_thishart_offset_ptr(timer_state_off); - *time_delta = delta; +#if __riscv_xlen == 32 + tstate->time_delta &= ~0xffffffffUL; + tstate->time_delta |= (u32)delta; +#else + tstate->time_delta = delta; +#endif } #if __riscv_xlen == 32 void sbi_timer_set_delta_upper(ulong delta_upper) { - ulong *time_delta = sbi_scratch_offset_ptr(sbi_scratch_thishart_ptr(), - time_delta_off); + struct timer_state *tstate = sbi_scratch_thishart_offset_ptr(timer_state_off); - *(time_delta + 1) = delta_upper; + tstate->time_delta &= 0xffffffffUL; + tstate->time_delta |= (u64)delta_upper << 32; } #endif -void sbi_timer_event_start(u64 next_event) +static void __sbi_timer_update_device(struct timer_state *tstate) +{ + struct sbi_timer_event *ev; + + if (!timer_dev) + return; + + if (sbi_list_empty(&tstate->event_list)) { + if (timer_dev->timer_event_stop) + timer_dev->timer_event_stop(); + csr_clear(CSR_MIE, MIP_MTIP); + } else { + ev = sbi_list_first_entry(&tstate->event_list, struct sbi_timer_event, head); + if (timer_dev->timer_event_start) + timer_dev->timer_event_start(ev->time_stamp); + csr_set(CSR_MIE, MIP_MTIP); + } +} + +static void __sbi_timer_event_stop(struct sbi_timer_event *ev) +{ + if (ev->hart_index > -1) { + sbi_list_del(&ev->head); + ev->hart_index = -1; + } +} + +static void __sbi_timer_event_start(struct timer_state *tstate, + struct sbi_timer_event *ev, u64 next_event) +{ + struct sbi_timer_event *tev, *next_ev = NULL; + + /* Find where to insert the event in per-HART event list */ + sbi_list_for_each_entry(tev, &tstate->event_list, head) { + if (next_event < tev->time_stamp) { + next_ev = tev; + break; + } + } + + /* Insert the event in per-HART event list */ + ev->hart_index = current_hartindex(); + ev->time_stamp = next_event; + if (next_ev) + sbi_list_add(&ev->head, &next_ev->head); + else + sbi_list_add_tail(&ev->head, &tstate->event_list); +} + +void sbi_timer_event_start(struct sbi_timer_event *ev, u64 next_event) +{ + struct timer_state *tstate; + + if (!ev) + return; + + /* Ensure that event is not on the per-HART event list */ + if (ev->hart_index > -1) { + tstate = sbi_scratch_offset_ptr(sbi_hartindex_to_scratch(ev->hart_index), + timer_state_off); + spin_lock(&tstate->event_list_lock); + __sbi_timer_event_stop(ev); + spin_unlock(&tstate->event_list_lock); + } + + tstate = sbi_scratch_thishart_offset_ptr(timer_state_off); + spin_lock(&tstate->event_list_lock); + + __sbi_timer_event_start(tstate, ev, next_event); + __sbi_timer_update_device(tstate); + + spin_unlock(&tstate->event_list_lock); +} + +void sbi_timer_event_stop(struct sbi_timer_event *ev) +{ + struct timer_state *tstate; + int ev_hart_index; + + if (!ev) + return; + + /* Ensure that event is not on the per-HART event list */ + ev_hart_index = ev->hart_index; + if (ev->hart_index > -1) { + tstate = sbi_scratch_offset_ptr(sbi_hartindex_to_scratch(ev->hart_index), + timer_state_off); + spin_lock(&tstate->event_list_lock); + __sbi_timer_event_stop(ev); + spin_unlock(&tstate->event_list_lock); + } + + /* Re-program timer device on the current HART */ + if (ev_hart_index == current_hartindex()) { + tstate = sbi_scratch_thishart_offset_ptr(timer_state_off); + spin_lock(&tstate->event_list_lock); + __sbi_timer_update_device(tstate); + spin_unlock(&tstate->event_list_lock); + } +} + +static void sbi_timer_smode_event_callback(struct sbi_timer_event *ev, + struct sbi_timer_event_restart *restart) { + /* + * If sstc extension is available, supervisor can receive the timer + * directly without M-mode come in between. This function should + * only invoked if M-mode programs the timer for its own purpose. + */ + if (!sbi_hart_has_extension(sbi_scratch_thishart_ptr(), SBI_HART_EXT_SSTC)) + csr_set(CSR_MIP, MIP_STIP); +} + +static void sbi_timer_smode_event_cleanup(struct sbi_timer_event *ev) +{ + if (!sbi_hart_has_extension(sbi_scratch_thishart_ptr(), SBI_HART_EXT_SSTC)) + csr_clear(CSR_MIP, MIP_STIP); +} + +void sbi_timer_smode_event_start(u64 next_event) +{ + struct timer_state *tstate = sbi_scratch_offset_ptr(sbi_scratch_thishart_ptr(), + timer_state_off); + sbi_pmu_ctr_incr_fw(SBI_PMU_FW_SET_TIMER); /** @@ -140,23 +279,47 @@ void sbi_timer_event_start(u64 next_event) */ if (sbi_hart_has_extension(sbi_scratch_thishart_ptr(), SBI_HART_EXT_SSTC)) { csr_write64(CSR_STIMECMP, next_event); - } else if (timer_dev && timer_dev->timer_event_start) { - timer_dev->timer_event_start(next_event); + } else { csr_clear(CSR_MIP, MIP_STIP); + sbi_timer_event_start(&tstate->smode_ev, next_event); } - csr_set(CSR_MIE, MIP_MTIP); } void sbi_timer_process(void) { - csr_clear(CSR_MIE, MIP_MTIP); - /* - * If sstc extension is available, supervisor can receive the timer - * directly without M-mode come in between. This function should - * only invoked if M-mode programs the timer for its own purpose. - */ - if (!sbi_hart_has_extension(sbi_scratch_thishart_ptr(), SBI_HART_EXT_SSTC)) - csr_set(CSR_MIP, MIP_STIP); + struct timer_state *tstate = sbi_scratch_thishart_offset_ptr(timer_state_off); + struct sbi_timer_event_restart restart; + SBI_LIST_HEAD(restart_list); + struct sbi_timer_event *ev; + + spin_lock(&tstate->event_list_lock); + + while (!sbi_list_empty(&tstate->event_list)) { + ev = sbi_list_first_entry(&tstate->event_list, struct sbi_timer_event, head); + if (ev->time_stamp > sbi_timer_value()) + break; + + __sbi_timer_event_stop(ev); + if (ev->callback) { + restart.required = false; + restart.next_event = 0; + ev->callback(ev, &restart); + if (restart.required) { + ev->time_stamp = restart.next_event; + sbi_list_add_tail(&ev->head, &restart_list); + } + } + } + + while (!sbi_list_empty(&restart_list)) { + ev = sbi_list_first_entry(&tstate->event_list, struct sbi_timer_event, head); + sbi_list_del(&ev->head); + __sbi_timer_event_start(tstate, ev, ev->time_stamp); + } + + __sbi_timer_update_device(tstate); + + spin_unlock(&tstate->event_list_lock); } const struct sbi_timer_device *sbi_timer_get_device(void) @@ -176,13 +339,13 @@ void sbi_timer_set_device(const struct sbi_timer_device *dev) int sbi_timer_init(struct sbi_scratch *scratch, bool cold_boot) { - u64 *time_delta; const struct sbi_platform *plat = sbi_platform_ptr(scratch); + struct timer_state *tstate; int ret; if (cold_boot) { - time_delta_off = sbi_scratch_alloc_offset(sizeof(*time_delta)); - if (!time_delta_off) + timer_state_off = sbi_scratch_alloc_offset(sizeof(*tstate)); + if (!timer_state_off) return SBI_ENOMEM; if (sbi_hart_has_csr(scratch, SBI_HART_CSR_TIME)) @@ -192,12 +355,17 @@ int sbi_timer_init(struct sbi_scratch *scratch, bool cold_boot) if (ret) return ret; } else { - if (!time_delta_off) + if (!timer_state_off) return SBI_ENOMEM; } - time_delta = sbi_scratch_offset_ptr(scratch, time_delta_off); - *time_delta = 0; + tstate = sbi_scratch_offset_ptr(scratch, timer_state_off); + tstate->time_delta = 0; + SPIN_LOCK_INIT(tstate->event_list_lock); + SBI_INIT_LIST_HEAD(&tstate->event_list); + SBI_INIT_TIMER_EVENT(&tstate->smode_ev, + sbi_timer_smode_event_callback, + sbi_timer_smode_event_cleanup, NULL); if (timer_dev && timer_dev->warm_init) { ret = timer_dev->warm_init(); @@ -210,9 +378,19 @@ int sbi_timer_init(struct sbi_scratch *scratch, bool cold_boot) void sbi_timer_exit(struct sbi_scratch *scratch) { - if (timer_dev && timer_dev->timer_event_stop) - timer_dev->timer_event_stop(); + struct timer_state *tstate = sbi_scratch_thishart_offset_ptr(timer_state_off); + struct sbi_timer_event *ev; + + spin_lock(&tstate->event_list_lock); + + while (!sbi_list_empty(&tstate->event_list)) { + ev = sbi_list_first_entry(&tstate->event_list, struct sbi_timer_event, head); + __sbi_timer_event_stop(ev); + if (ev->cleanup) + ev->cleanup(ev); + } + + __sbi_timer_update_device(tstate); - csr_clear(CSR_MIP, MIP_STIP); - csr_clear(CSR_MIE, MIP_MTIP); + spin_unlock(&tstate->event_list_lock); } diff --git a/lib/sbi/sbi_tlb.c b/lib/sbi/sbi_tlb.c index ada60c329d5..8255ff59937 100644 --- a/lib/sbi/sbi_tlb.c +++ b/lib/sbi/sbi_tlb.c @@ -43,6 +43,10 @@ static void sbi_tlb_local_hfence_vvma(struct sbi_tlb_info *tinfo) sbi_pmu_ctr_incr_fw(SBI_PMU_FW_HFENCE_VVMA_RCVD); + if (sbi_platform_local_hfence_gvma( + sbi_platform_thishart_ptr(), tinfo) == SBI_OK) + return; + hgatp = csr_swap(CSR_HGATP, (vmid << HGATP_VMID_SHIFT) & HGATP_VMID_MASK); @@ -67,6 +71,10 @@ static void sbi_tlb_local_hfence_gvma(struct sbi_tlb_info *tinfo) sbi_pmu_ctr_incr_fw(SBI_PMU_FW_HFENCE_GVMA_RCVD); + if (sbi_platform_local_hfence_gvma( + sbi_platform_thishart_ptr(), tinfo) == SBI_OK) + return; + if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) { __sbi_hfence_gvma_all(); return; @@ -85,6 +93,10 @@ static void sbi_tlb_local_sfence_vma(struct sbi_tlb_info *tinfo) sbi_pmu_ctr_incr_fw(SBI_PMU_FW_SFENCE_VMA_RCVD); + if (sbi_platform_local_sfence_vma( + sbi_platform_thishart_ptr(), tinfo) == SBI_OK) + return; + if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) { __sbi_sfence_vma_all(); return; @@ -108,6 +120,10 @@ static void sbi_tlb_local_hfence_vvma_asid(struct sbi_tlb_info *tinfo) sbi_pmu_ctr_incr_fw(SBI_PMU_FW_HFENCE_VVMA_ASID_RCVD); + if (sbi_platform_local_hfence_vvma_asid( + sbi_platform_thishart_ptr(), tinfo) == SBI_OK) + return; + hgatp = csr_swap(CSR_HGATP, (vmid << HGATP_VMID_SHIFT) & HGATP_VMID_MASK); @@ -133,6 +149,10 @@ static void sbi_tlb_local_hfence_gvma_vmid(struct sbi_tlb_info *tinfo) sbi_pmu_ctr_incr_fw(SBI_PMU_FW_HFENCE_GVMA_VMID_RCVD); + if (sbi_platform_local_hfence_gvma_vmid( + sbi_platform_thishart_ptr(), tinfo) == SBI_OK) + return; + if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) { __sbi_hfence_gvma_vmid(vmid); return; @@ -152,6 +172,10 @@ static void sbi_tlb_local_sfence_vma_asid(struct sbi_tlb_info *tinfo) sbi_pmu_ctr_incr_fw(SBI_PMU_FW_SFENCE_VMA_ASID_RCVD); + if (sbi_platform_local_sfence_vma_asid( + sbi_platform_thishart_ptr(), tinfo) == SBI_OK) + return; + /* Flush entire MM context for a given ASID */ if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) { __asm__ __volatile__("sfence.vma x0, %0" @@ -173,6 +197,10 @@ static void sbi_tlb_local_fence_i(struct sbi_tlb_info *tinfo) { sbi_pmu_ctr_incr_fw(SBI_PMU_FW_FENCE_I_RECVD); + if (sbi_platform_local_fence_i( + sbi_platform_thishart_ptr(), tinfo) == SBI_OK) + return; + __asm__ __volatile("fence.i"); } diff --git a/lib/sbi/sbi_trap.c b/lib/sbi/sbi_trap.c index f41db4d1d5f..d336013423e 100644 --- a/lib/sbi/sbi_trap.c +++ b/lib/sbi/sbi_trap.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -285,6 +286,19 @@ static int sbi_trap_aia_irq(void) return 0; } +#if __riscv_xlen > 32 && defined(CONFIG_EMU_SUPM) +static inline bool sbi_pm_changes_ptr(ulong ptr, struct sbi_scratch *scratch) +{ + return scratch->sw_pm && + ptr != (ulong)((long)(ptr << scratch->sw_pm) >> scratch->sw_pm); +} + +static inline void sbi_mask_ptr(ulong *pptr, struct sbi_scratch *scratch) +{ + *pptr = (long)(*pptr << scratch->sw_pm) >> scratch->sw_pm; +} +#endif + /** * Handle trap/interrupt * @@ -345,11 +359,29 @@ struct sbi_trap_context *sbi_trap_handler(struct sbi_trap_context *tcntx) msg = "ecall handler failed"; break; case CAUSE_LOAD_ACCESS: +#if __riscv_xlen > 32 && defined(CONFIG_EMU_SUPM) + if (sbi_pm_changes_ptr(trap->tval, scratch)) { + sbi_mask_ptr(&tcntx->trap.tval, scratch); + /* redirect to misaligned load handler */ + rc = sbi_misaligned_load_handler(tcntx); + msg = "pointer masking load handler failed"; + break; + } +#endif sbi_pmu_ctr_incr_fw(SBI_PMU_FW_ACCESS_LOAD); rc = sbi_load_access_handler(tcntx); msg = "load fault handler failed"; break; case CAUSE_STORE_ACCESS: +#if __riscv_xlen > 32 && defined(CONFIG_EMU_SUPM) + if (sbi_pm_changes_ptr(trap->tval, scratch)) { + sbi_mask_ptr(&tcntx->trap.tval, scratch); + /* redirect to misaligned store handler */ + rc = sbi_misaligned_store_handler(tcntx); + msg = "pointer masking store handler failed"; + break; + } +#endif sbi_pmu_ctr_incr_fw(SBI_PMU_FW_ACCESS_STORE); rc = sbi_store_access_handler(tcntx); msg = "store fault handler failed"; @@ -358,6 +390,47 @@ struct sbi_trap_context *sbi_trap_handler(struct sbi_trap_context *tcntx) rc = sbi_double_trap_handler(tcntx); msg = "double trap handler failed"; break; +#if __riscv_xlen > 32 && defined(CONFIG_EMU_SUPM) + case CAUSE_FETCH_PAGE_FAULT: + if (sbi_pm_changes_ptr(regs->mepc, scratch)) { + /* mask the program counter and try to continue */ + sbi_mask_ptr(®s->mepc, scratch); + rc = 0; + msg = "pointer masking fetch handler failed"; + } + else { + /* If the trap came from S or U mode, redirect it there */ + msg = "trap redirect failed (fetch page fault)"; + rc = sbi_trap_redirect(regs, trap); + } + break; + case CAUSE_LOAD_PAGE_FAULT: + if (sbi_pm_changes_ptr(trap->tval, scratch)) { + sbi_mask_ptr(&tcntx->trap.tval, scratch); + /* redirect to misaligned load handler */ + rc = sbi_misaligned_load_handler(tcntx); + msg = "pointer masking load handler failed"; + } + else { + /* If the trap came from S or U mode, redirect it there */ + msg = "trap redirect failed (load page fault)"; + rc = sbi_trap_redirect(regs, trap); + } + break; + case CAUSE_STORE_PAGE_FAULT: + if (sbi_pm_changes_ptr(trap->tval, scratch)) { + sbi_mask_ptr(&tcntx->trap.tval, scratch); + /* redirect to misaligned store handler */ + rc = sbi_misaligned_store_handler(tcntx); + msg = "pointer masking store handler failed"; + } + else { + /* If the trap came from S or U mode, redirect it there */ + msg = "trap redirect failed (store page fault)"; + rc = sbi_trap_redirect(regs, trap); + } + break; +#endif default: /* If the trap came from S or U mode, redirect it there */ msg = "trap redirect failed"; @@ -375,3 +448,41 @@ struct sbi_trap_context *sbi_trap_handler(struct sbi_trap_context *tcntx) sbi_trap_set_context(scratch, tcntx->prev_context); return tcntx; } + +/** + * Default Resumable NMI (RNMI) handler + * + * This function is called from the _trap_rnmi_handler assembly code. + * It provides a simple wrapper that calls the platform-specific + * NMI handler if registered. If no handler is registered, it prints + * diagnostic information and hangs, similar to unhandled traps. + * + * Note: The trap context stores NMI CSR values (MNCAUSE, MNEPC, MNSTATUS) + * in the generic trap context fields (cause, mepc, mstatus). + * + * @param tcntx Pointer to trap context (saved on stack) + * @return Same trap context pointer (needed for restore macros) + */ +struct sbi_trap_context *sbi_trap_rnmi_handler(struct sbi_trap_context *tcntx) +{ + int rc; + const struct sbi_platform *plat = sbi_platform_thishart_ptr(); + const struct sbi_platform_operations *ops = sbi_platform_ops(plat); + + /* Call platform-specific NMI handler if registered */ + if (ops && ops->rnmi_handler) { + rc = ops->rnmi_handler(tcntx); + if (rc) { + /* Platform handler failed to handle NMI */ + sbi_trap_error("platform NMI handler failed", rc, tcntx); + } + return tcntx; + } + + /* No platform handler - treat as unhandled NMI */ + sbi_trap_error("unhandled NMI (no platform rnmi_handler)", + SBI_ENOTSUPP, tcntx); + + /* Never returns */ + return tcntx; +} diff --git a/lib/sbi/sbi_trap_ldst.c b/lib/sbi/sbi_trap_ldst.c index 448406b150c..4ba53973f70 100644 --- a/lib/sbi/sbi_trap_ldst.c +++ b/lib/sbi/sbi_trap_ldst.c @@ -18,30 +18,65 @@ /** * Load emulator callback: - * - * @return rlen=success, 0=success w/o regs modification, or negative error + * Refer to comments of `sbi_platform_emulate_load`. */ -typedef int (*sbi_trap_ld_emulator)(int rlen, union sbi_ldst_data *out_val, +typedef int (*sbi_trap_ld_emulator)(ulong insn, int rlen, ulong raddr, + union sbi_ldst_data *out_val, struct sbi_trap_context *tcntx); /** * Store emulator callback: - * - * @return wlen=success, 0=success w/o regs modification, or negative error + * Refer to comments of `sbi_platform_emulate_store`. */ -typedef int (*sbi_trap_st_emulator)(int wlen, union sbi_ldst_data in_val, +typedef int (*sbi_trap_st_emulator)(ulong insn, int wlen, ulong waddr, + union sbi_ldst_data in_val, struct sbi_trap_context *tcntx); -ulong sbi_misaligned_tinst_fixup(ulong orig_tinst, ulong new_tinst, - ulong addr_offset) +/** + * Handling of misaligned fault is done by a collection of smaller, but + * aligned load/store(s). Another fault (load/store, page fault...) can + * arise from any of them, then the handling gets aborted. We must fixup + * the tinst to pretend the fault was rised from the original insn. + * Specifically, fixup the offset field using the tval diff between the + * new trap and the original one (if required). + */ +static inline void sbi_misaligned_tinst_fixup( + const struct sbi_trap_info *orig_trap, + struct sbi_trap_info *uptrap) { - if (new_tinst == INSN_PSEUDO_VS_LOAD || - new_tinst == INSN_PSEUDO_VS_STORE) - return new_tinst; - else if (orig_tinst == 0) - return 0UL; + ulong offset = uptrap->tval - orig_trap->tval; + + /* + * The function is called in code path for handling a scalar + * load/store misaligned fault, thus the new uptrap can't have + * custom value of tinst + */ + if (uptrap->tinst == INSN_PSEUDO_VS_LOAD || + uptrap->tinst == INSN_PSEUDO_VS_STORE) + /* Use uptrap as-is for guest-page faults */ + return; + /* + * Only fixup if orig tinst is valid. Otherwise, discard the + * new tinst to be on the safe side. Never use new tinst as-is! + * It's load/store width surely mismatches the original width. + * For vector, discard it regardless. It doesn't make sense to + * have a transformed tinst + */ + else if (orig_trap->tinst == 0) + uptrap->tinst = 0; else - return orig_tinst | (addr_offset << SH_RS1); + uptrap->tinst = orig_trap->tinst | (offset << SH_RS1); +} + +static inline bool sbi_trap_tinst_valid(ulong tinst) +{ + /* + * Bit[0] == 1 implies trapped instruction value is + * transformed instruction or custom instruction. + * Also do proper checking per Privileged ISA 19.6.3, + * and make sure high 32 bits of tinst is 0 + */ + return tinst == (uint32_t)tinst && (tinst & 0x1); } static int sbi_trap_emulate_load(struct sbi_trap_context *tcntx, @@ -49,23 +84,18 @@ static int sbi_trap_emulate_load(struct sbi_trap_context *tcntx, { const struct sbi_trap_info *orig_trap = &tcntx->trap; struct sbi_trap_regs *regs = &tcntx->regs; - ulong insn, insn_len; + ulong insn, insn_len, imm = 0, shift = 0, off = 0; union sbi_ldst_data val = { 0 }; struct sbi_trap_info uptrap; - int rc, fp = 0, shift = 0, len = 0, vector = 0; + bool xform = false, fp = false, c_load = false, c_ldsp = false; + int rc, len = 0, prev_xlen = 0; - if (orig_trap->tinst & 0x1) { - /* - * Bit[0] == 1 implies trapped instruction value is - * transformed instruction or custom instruction. - */ + if (sbi_trap_tinst_valid(orig_trap->tinst)) { + xform = true; insn = orig_trap->tinst | INSN_16BIT_MASK; insn_len = (orig_trap->tinst & 0x2) ? INSN_LEN(insn) : 2; } else { - /* - * Bit[0] == 0 implies trapped instruction value is - * zero or special value. - */ + /* trapped instruction value is zero or special value */ insn = sbi_get_insn(regs->mepc, &uptrap); if (uptrap.cause) { return sbi_trap_redirect(regs, &uptrap); @@ -73,11 +103,20 @@ static int sbi_trap_emulate_load(struct sbi_trap_context *tcntx, insn_len = INSN_LEN(insn); } + /** + * Common for RV32/RV64: + * lb, lbu, lh, lhu, lw, flw, flw + * c.lbu, c.lh, c.lhu, c.lw, c.lwsp, c.fld, c.fldsp + */ if ((insn & INSN_MASK_LB) == INSN_MATCH_LB) { - len = 1; - shift = 8 * (sizeof(ulong) - len); + len = -1; } else if ((insn & INSN_MASK_LBU) == INSN_MATCH_LBU) { len = 1; + } else if ((insn & INSN_MASK_C_LBU) == INSN_MATCH_C_LBU) { + /* Zcb */ + len = 1; + imm = RVC_LB_IMM(insn); + c_load = true; } else if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) { len = 4; shift = 8 * (sizeof(ulong) - len); @@ -95,77 +134,173 @@ static int sbi_trap_emulate_load(struct sbi_trap_context *tcntx, } else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) { fp = 1; len = 4; + } else if ((insn & INSN_MASK_FLH) == INSN_MATCH_FLH) { + fp = 1; + len = 2; #endif } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) { - len = 2; - shift = 8 * (sizeof(ulong) - len); + len = -2; + } else if ((insn & INSN_MASK_C_LH) == INSN_MATCH_C_LH) { + /* Zcb */ + len = -2; + imm = RVC_LH_IMM(insn); + c_load = true; } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) { len = 2; -#if __riscv_xlen >= 64 - } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) { - len = 8; - shift = 8 * (sizeof(ulong) - len); - insn = RVC_RS2S(insn) << SH_RD; - } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP && - ((insn >> SH_RD) & 0x1f)) { - len = 8; - shift = 8 * (sizeof(ulong) - len); -#endif + } else if ((insn & INSN_MASK_C_LHU) == INSN_MATCH_C_LHU) { + /* Zcb */ + len = 2; + imm = RVC_LH_IMM(insn); + c_load = true; + } else if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) { + len = -4; } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) { - len = 4; - shift = 8 * (sizeof(ulong) - len); - insn = RVC_RS2S(insn) << SH_RD; + /* Zca */ + len = -4; + imm = RVC_LW_IMM(insn); + c_load = true; } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP && - ((insn >> SH_RD) & 0x1f)) { - len = 4; - shift = 8 * (sizeof(ulong) - len); + GET_RD_NUM(insn)) { + /* Zca */ + len = -4; + imm = RVC_LWSP_IMM(insn); + c_ldsp = true; #ifdef __riscv_flen + } else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) { + len = 4; + fp = true; + } else if ((insn & INSN_MASK_FLD) == INSN_MATCH_FLD) { + len = 8; + fp = true; } else if ((insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD) { - fp = 1; - len = 8; - insn = RVC_RS2S(insn) << SH_RD; + /* Zcd */ + len = 8; + imm = RVC_LD_IMM(insn); + c_load = true; + fp = true; } else if ((insn & INSN_MASK_C_FLDSP) == INSN_MATCH_C_FLDSP) { - fp = 1; + /* Zcd */ len = 8; -#if __riscv_xlen == 32 - } else if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) { - fp = 1; - len = 4; - insn = RVC_RS2S(insn) << SH_RD; - } else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) { - fp = 1; - len = 4; + imm = RVC_LDSP_IMM(insn); + c_ldsp = true; + fp = true; #endif + } else { + prev_xlen = sbi_regs_prev_xlen(regs); + } + + /** + * Must distinguish between rv64 and rv32, RVC instructions have + * overlapping encoding: + * c.ld in rv64 == c.flw in rv32 + * c.ldsp in rv64 == c.flwsp in rv32 + */ + if (prev_xlen == 64) { + /* RV64 Only: lwu, ld, c.ld, c.ldsp */ + if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) { + len = 4; + } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) { + len = 8; + } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) { + /* Zca */ + len = 8; + imm = RVC_LD_IMM(insn); + c_load = true; + } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP && + GET_RD_NUM(insn)) { + /* Zca */ + len = 8; + imm = RVC_LDSP_IMM(insn); + c_ldsp = true; + } +#ifdef __riscv_flen + } else if (prev_xlen == 32) { + /* RV32 Only: c.flw, c.flwsp */ + if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) { + /* Zcf */ + len = 4; + imm = RVC_LW_IMM(insn); + c_load = true; + fp = true; + } else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) { + /* Zcf */ + len = 4; + imm = RVC_LWSP_IMM(insn); + c_ldsp = true; + fp = true; + } #endif - } else if ((insn & INSN_MASK_C_LHU) == INSN_MATCH_C_LHU) { - len = 2; - insn = RVC_RS2S(insn) << SH_RD; - } else if ((insn & INSN_MASK_C_LH) == INSN_MATCH_C_LH) { - len = 2; + } + + if (len < 0) { + len = -len; shift = 8 * (sizeof(ulong) - len); - insn = RVC_RS2S(insn) << SH_RD; - } else if (IS_VECTOR_LOAD_STORE(insn)) { - vector = 1; - emu = sbi_misaligned_v_ld_emulator; - } else { - return sbi_trap_redirect(regs, orig_trap); } - rc = emu(len, &val, tcntx); + if (!len) // Unknown instruction + goto do_emu; + +#if !defined(OPENSBI_DEBUG) + /** + * For misaligned faults. Skip offset calculation unless DEBUG + * builds. It helps validating OpenSBI and HW. + */ + if (orig_trap->cause == CAUSE_MISALIGNED_LOAD) + goto do_emu; +#endif + + if (xform) + /* Transformed insn */ + off = GET_RS1_NUM(insn); + else if (c_load) + /* non SP-based compressed load */ + off = orig_trap->tval - GET_RS1S(insn, regs) - imm; + else if (c_ldsp) + /* SP-based compressed load */ + off = orig_trap->tval - REG_VAL(2, regs) - imm; + else + /* I-type non-compressed load */ + off = orig_trap->tval - GET_RS1(insn, regs) - (ulong)IMM_I(insn); + /** + * Normalize offset, in case the XLEN of unpriv mode is smaller, + * and/or pointer masking is in effect + */ + off &= (len - 1); + +do_emu: + rc = emu(insn, len, orig_trap->tval - off, &val, tcntx); if (rc <= 0) return rc; + if (!len) + goto epc_fixup; + + if (!fp) { + ulong v = ((long)(val.data_ulong << shift)) >> shift; - if (!vector) { - if (!fp) - SET_RD(insn, regs, ((long)(val.data_ulong << shift)) >> shift); + if (c_load) + SET_RDS(insn, regs, v); + else + SET_RD(insn, regs, v); #ifdef __riscv_flen - else if (len == 8) + } else if (len == 8) { + if (c_load) + SET_F64_RDS(insn, regs, val.data_u64); + else SET_F64_RD(insn, regs, val.data_u64); + } else if (len == 4) { + if (c_load) + SET_F32_RDS(insn, regs, val.data_ulong); else SET_F32_RD(insn, regs, val.data_ulong); + } else { + if (c_load) + SET_F16_RDS(insn, regs, val.data_ulong); + else + SET_F16_RD(insn, regs, val.data_ulong); #endif } +epc_fixup: regs->mepc += insn_len; return 0; @@ -176,23 +311,18 @@ static int sbi_trap_emulate_store(struct sbi_trap_context *tcntx, { const struct sbi_trap_info *orig_trap = &tcntx->trap; struct sbi_trap_regs *regs = &tcntx->regs; - ulong insn, insn_len; + ulong insn, insn_len, imm = 0, off = 0; union sbi_ldst_data val; struct sbi_trap_info uptrap; - int rc, len = 0; + bool xform = false, fp = false, c_store = false, c_stsp = false; + int rc, len = 0, prev_xlen = 0; - if (orig_trap->tinst & 0x1) { - /* - * Bit[0] == 1 implies trapped instruction value is - * transformed instruction or custom instruction. - */ + if (sbi_trap_tinst_valid(orig_trap->tinst)) { + xform = true; insn = orig_trap->tinst | INSN_16BIT_MASK; insn_len = (orig_trap->tinst & 0x2) ? INSN_LEN(insn) : 2; } else { - /* - * Bit[0] == 0 implies trapped instruction value is - * zero or special value. - */ + /* trapped instruction value is zero or special value */ insn = sbi_get_insn(regs->mepc, &uptrap); if (uptrap.cause) { return sbi_trap_redirect(regs, &uptrap); @@ -200,10 +330,18 @@ static int sbi_trap_emulate_store(struct sbi_trap_context *tcntx, insn_len = INSN_LEN(insn); } - val.data_ulong = GET_RS2(insn, regs); - + /** + * Common for RV32/RV64: + * sb, sh, sw, fsw, fsd + * c.sb, c.sh, c.sw, c.swsp, c.fsd, c.fsdsp + */ if ((insn & INSN_MASK_SB) == INSN_MATCH_SB) { len = 1; + } else if ((insn & INSN_MASK_C_SB) == INSN_MATCH_C_SB) { + /* Zcb */ + len = 1; + imm = RVC_SB_IMM(insn); + c_store = true; } else if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) { len = 4; #if __riscv_xlen == 64 @@ -217,49 +355,150 @@ static int sbi_trap_emulate_store(struct sbi_trap_context *tcntx, } else if ((insn & INSN_MASK_FSW) == INSN_MATCH_FSW) { len = 4; val.data_ulong = GET_F32_RS2(insn, regs); + } else if ((insn & INSN_MASK_FSH) == INSN_MATCH_FSH) { + len = 2; + val.data_ulong = GET_F16_RS2(insn, regs); #endif } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) { len = 2; -#if __riscv_xlen >= 64 - } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) { - len = 8; - val.data_ulong = GET_RS2S(insn, regs); - } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP) { - len = 8; - val.data_ulong = GET_RS2C(insn, regs); -#endif + } else if ((insn & INSN_MASK_C_SH) == INSN_MATCH_C_SH) { + /* Zcb */ + len = 2; + imm = RVC_SH_IMM(insn); + c_store = true; + } else if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) { + len = 4; } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) { - len = 4; - val.data_ulong = GET_RS2S(insn, regs); + /* Zca */ + len = 4; + imm = RVC_SW_IMM(insn); + c_store = true; } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP) { - len = 4; - val.data_ulong = GET_RS2C(insn, regs); + /* Zca */ + len = 4; + imm = RVC_SWSP_IMM(insn); + c_stsp = true; #ifdef __riscv_flen + } else if ((insn & INSN_MASK_FSW) == INSN_MATCH_FSW) { + len = 4; + fp = true; + } else if ((insn & INSN_MASK_FSD) == INSN_MATCH_FSD) { + len = 8; + fp = true; } else if ((insn & INSN_MASK_C_FSD) == INSN_MATCH_C_FSD) { - len = 8; - val.data_u64 = GET_F64_RS2S(insn, regs); + /* Zcd */ + len = 8; + imm = RVC_SD_IMM(insn); + c_store = true; + fp = true; } else if ((insn & INSN_MASK_C_FSDSP) == INSN_MATCH_C_FSDSP) { - len = 8; - val.data_u64 = GET_F64_RS2C(insn, regs); -#if __riscv_xlen == 32 - } else if ((insn & INSN_MASK_C_FSW) == INSN_MATCH_C_FSW) { - len = 4; - val.data_ulong = GET_F32_RS2S(insn, regs); - } else if ((insn & INSN_MASK_C_FSWSP) == INSN_MATCH_C_FSWSP) { - len = 4; - val.data_ulong = GET_F32_RS2C(insn, regs); + /* Zcd */ + len = 8; + imm = RVC_SDSP_IMM(insn); + c_stsp = true; + fp = true; #endif + } else { + prev_xlen = sbi_regs_prev_xlen(regs); + } + + /** + * Must distinguish between rv64 and rv32, RVC instructions have + * overlapping encoding: + * c.sd in rv64 == c.fsw in rv32 + * c.sdsp in rv64 == c.fswsp in rv32 + */ + if (prev_xlen == 64) { + /* RV64 Only: sd, c.sd, c.sdsp */ + if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) { + len = 8; + } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) { + /* Zca */ + len = 8; + imm = RVC_SD_IMM(insn); + c_store = true; + } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP) { + /* Zca */ + len = 8; + imm = RVC_SDSP_IMM(insn); + c_stsp = true; + } +#ifdef __riscv_flen + } else if (prev_xlen == 32) { + /* RV32 Only: c.fsw, c.fswsp */ + if ((insn & INSN_MASK_C_FSW) == INSN_MATCH_C_FSW) { + /* Zcf */ + len = 4; + imm = RVC_SW_IMM(insn); + c_store = true; + fp = true; + } else if ((insn & INSN_MASK_C_FSWSP) == INSN_MATCH_C_FSWSP) { + /* Zcf */ + len = 4; + imm = RVC_SWSP_IMM(insn); + c_stsp = true; + fp = true; + } #endif - } else if ((insn & INSN_MASK_C_SH) == INSN_MATCH_C_SH) { - len = 2; - val.data_ulong = GET_RS2S(insn, regs); - } else if (IS_VECTOR_LOAD_STORE(insn)) { - emu = sbi_misaligned_v_st_emulator; + } + + if (!fp) { + if (c_store) + val.data_ulong = GET_RS2S(insn, regs); + else if (c_stsp) + val.data_ulong = GET_RS2C(insn, regs); + else + val.data_ulong = GET_RS2(insn, regs); +#ifdef __riscv_flen + } else if (len == 8) { + if (c_store) + val.data_u64 = GET_F64_RS2S(insn, regs); + else if (c_stsp) + val.data_u64 = GET_F64_RS2C(insn, regs); + else + val.data_u64 = GET_F64_RS2(insn, regs); } else { - return sbi_trap_redirect(regs, orig_trap); + if (c_store) + val.data_ulong = GET_F32_RS2S(insn, regs); + else if (c_stsp) + val.data_ulong = GET_F32_RS2C(insn, regs); + else + val.data_ulong = GET_F32_RS2(insn, regs); +#endif } - rc = emu(len, val, tcntx); + if (!len) // Unknown instruction + goto do_emu; + +#if !defined(OPENSBI_DEBUG) + /** + * For misaligned faults. Skip offset calculation unless DEBUG + * builds. It helps validating OpenSBI and HW. + */ + if (orig_trap->cause == CAUSE_MISALIGNED_STORE) + goto do_emu; +#endif + + if (xform) + /* Transformed insn */ + off = GET_RS1_NUM(insn); + else if (c_store) + /* non SP-based compressed store */ + off = orig_trap->tval - GET_RS1S(insn, regs) - imm; + else if (c_stsp) + /* SP-based compressed store */ + off = orig_trap->tval - REG_VAL(2, regs) - imm; + else + /* S-type non-compressed store */ + off = orig_trap->tval - GET_RS1(insn, regs) - (ulong)IMM_S(insn); + /** + * Normalize offset, in case the XLEN of unpriv mode is smaller, + * and/or pointer masking is in effect + */ + off &= (len - 1); + +do_emu: + rc = emu(insn, len, orig_trap->tval - off, val, tcntx); if (rc <= 0) return rc; @@ -268,22 +507,29 @@ static int sbi_trap_emulate_store(struct sbi_trap_context *tcntx, return 0; } -static int sbi_misaligned_ld_emulator(int rlen, union sbi_ldst_data *out_val, +static int sbi_misaligned_ld_emulator(ulong insn, int rlen, ulong addr, + union sbi_ldst_data *out_val, struct sbi_trap_context *tcntx) { const struct sbi_trap_info *orig_trap = &tcntx->trap; struct sbi_trap_regs *regs = &tcntx->regs; struct sbi_trap_info uptrap; - int i; - for (i = 0; i < rlen; i++) { - out_val->data_bytes[i] = - sbi_load_u8((void *)(orig_trap->tval + i), &uptrap); - if (uptrap.cause) { - uptrap.tinst = sbi_misaligned_tinst_fixup( - orig_trap->tinst, uptrap.tinst, i); - return sbi_trap_redirect(regs, &uptrap); - } + if (!rlen) { + if (IS_VECTOR_LOAD_STORE(insn)) + return sbi_misaligned_v_ld_emulator(insn, tcntx); + else + /* Unrecognized instruction. Can't emulate it. */ + return sbi_trap_redirect(regs, orig_trap); + } + /* For misaligned fault, addr must be the same as orig_trap->tval */ + if (addr != orig_trap->tval) + return SBI_EFAIL; + + sbi_load_loop(out_val->data_bytes, addr, rlen, &uptrap); + if (uptrap.cause) { + sbi_misaligned_tinst_fixup(orig_trap, &uptrap); + return sbi_trap_redirect(regs, &uptrap); } return rlen; } @@ -293,22 +539,29 @@ int sbi_misaligned_load_handler(struct sbi_trap_context *tcntx) return sbi_trap_emulate_load(tcntx, sbi_misaligned_ld_emulator); } -static int sbi_misaligned_st_emulator(int wlen, union sbi_ldst_data in_val, +static int sbi_misaligned_st_emulator(ulong insn, int wlen, ulong addr, + union sbi_ldst_data in_val, struct sbi_trap_context *tcntx) { const struct sbi_trap_info *orig_trap = &tcntx->trap; struct sbi_trap_regs *regs = &tcntx->regs; struct sbi_trap_info uptrap; - int i; - for (i = 0; i < wlen; i++) { - sbi_store_u8((void *)(orig_trap->tval + i), - in_val.data_bytes[i], &uptrap); - if (uptrap.cause) { - uptrap.tinst = sbi_misaligned_tinst_fixup( - orig_trap->tinst, uptrap.tinst, i); - return sbi_trap_redirect(regs, &uptrap); - } + if (!wlen) { + if (IS_VECTOR_LOAD_STORE(insn)) + return sbi_misaligned_v_st_emulator(insn, tcntx); + else + /* Unrecognized instruction. Can't emulate it. */ + return sbi_trap_redirect(regs, orig_trap); + } + /* For misaligned fault, addr must be the same as orig_trap->tval */ + if (addr != orig_trap->tval) + return SBI_EFAIL; + + sbi_store_loop(in_val.data_bytes, addr, wlen, &uptrap); + if (uptrap.cause) { + sbi_misaligned_tinst_fixup(orig_trap, &uptrap); + return sbi_trap_redirect(regs, &uptrap); } return wlen; } @@ -318,22 +571,26 @@ int sbi_misaligned_store_handler(struct sbi_trap_context *tcntx) return sbi_trap_emulate_store(tcntx, sbi_misaligned_st_emulator); } -static int sbi_ld_access_emulator(int rlen, union sbi_ldst_data *out_val, +static int sbi_ld_access_emulator(ulong insn, int rlen, ulong addr, + union sbi_ldst_data *out_val, struct sbi_trap_context *tcntx) { const struct sbi_trap_info *orig_trap = &tcntx->trap; struct sbi_trap_regs *regs = &tcntx->regs; + int rc; /* If fault came from M mode, just fail */ if (sbi_mstatus_prev_mode(regs->mstatus) == PRV_M) return SBI_EINVAL; + rc = sbi_platform_emulate_load(sbi_platform_thishart_ptr(), + insn, rlen, addr, out_val, tcntx); + /* If platform emulator failed, we redirect instead of fail */ - if (sbi_platform_emulate_load(sbi_platform_thishart_ptr(), rlen, - orig_trap->tval, out_val)) + if (rc < 0) return sbi_trap_redirect(regs, orig_trap); - return rlen; + return rc; } int sbi_load_access_handler(struct sbi_trap_context *tcntx) @@ -341,22 +598,26 @@ int sbi_load_access_handler(struct sbi_trap_context *tcntx) return sbi_trap_emulate_load(tcntx, sbi_ld_access_emulator); } -static int sbi_st_access_emulator(int wlen, union sbi_ldst_data in_val, +static int sbi_st_access_emulator(ulong insn, int wlen, ulong addr, + union sbi_ldst_data in_val, struct sbi_trap_context *tcntx) { const struct sbi_trap_info *orig_trap = &tcntx->trap; struct sbi_trap_regs *regs = &tcntx->regs; + int rc; /* If fault came from M mode, just fail */ if (sbi_mstatus_prev_mode(regs->mstatus) == PRV_M) return SBI_EINVAL; + rc = sbi_platform_emulate_store(sbi_platform_thishart_ptr(), + insn, wlen, addr, in_val, tcntx); + /* If platform emulator failed, we redirect instead of fail */ - if (sbi_platform_emulate_store(sbi_platform_thishart_ptr(), wlen, - orig_trap->tval, in_val)) + if (rc < 0) return sbi_trap_redirect(regs, orig_trap); - return wlen; + return rc; } int sbi_store_access_handler(struct sbi_trap_context *tcntx) diff --git a/lib/sbi/sbi_trap_v_ldst.c b/lib/sbi/sbi_trap_v_ldst.c index f4d469dcf28..540f655ff2e 100644 --- a/lib/sbi/sbi_trap_v_ldst.c +++ b/lib/sbi/sbi_trap_v_ldst.c @@ -11,15 +11,16 @@ #include #include +#include #include #include #include #include -#include +#include #ifdef OPENSBI_CC_SUPPORT_VECTOR -#define VLEN_MAX 65536 +#define MASK_BUFFLEN 1024 static inline void set_vreg(ulong vlenb, ulong which, ulong pos, ulong size, const uint8_t *bytes) @@ -137,17 +138,37 @@ static inline void vsetvl(ulong vl, ulong vtype) :: "r" (vl), "r" (vtype)); } -int sbi_misaligned_v_ld_emulator(int rlen, union sbi_ldst_data *out_val, - struct sbi_trap_context *tcntx) +/** + * Handling of misaligned fault is done by a collection of smaller, but + * aligned load/store(s). Another fault (load/store, page fault...) can + * arise from any of them, then the handling gets aborted. We must fixup + * the tinst to pretend the fault was rised from the original insn. For + * vector insn, simply null out tinst if it's not a guest-page fault, as + * there's no transformed insn for vector load/store + */ +static inline void sbi_misaligned_v_tinst_fixup(struct sbi_trap_info *uptrap) +{ + /* + * The function is called in code path for handling a vector + * load/store misaligned fault, thus the new uptrap can't have + * custom value of tinst + */ + if (uptrap->tinst == INSN_PSEUDO_VS_LOAD || + uptrap->tinst == INSN_PSEUDO_VS_STORE) + /* Use uptrap as-is for guest-page faults */ + return; + + uptrap->tinst = 0; +} + +int sbi_misaligned_v_ld_emulator(ulong insn, struct sbi_trap_context *tcntx) { - const struct sbi_trap_info *orig_trap = &tcntx->trap; struct sbi_trap_regs *regs = &tcntx->regs; struct sbi_trap_info uptrap; - ulong insn = sbi_get_insn(regs->mepc, &uptrap); ulong vl = csr_read(CSR_VL); ulong vtype = csr_read(CSR_VTYPE); ulong vlenb = csr_read(CSR_VLENB); - ulong vstart = csr_read(CSR_VSTART); + ulong vstart = csr_read(CSR_VSTART), orig_vstart = vstart; ulong base = GET_RS1(insn, regs); ulong stride = GET_RS2(insn, regs); ulong vd = GET_VD(insn); @@ -157,8 +178,9 @@ int sbi_misaligned_v_ld_emulator(int rlen, union sbi_ldst_data *out_val, ulong vlmul = GET_VLMUL(vtype); bool illegal = GET_MEW(insn); bool masked = IS_MASKED(insn); - uint8_t mask[VLEN_MAX / 8]; + uint8_t mask[MASK_BUFFLEN / 8]; uint8_t bytes[8 * sizeof(uint64_t)]; + ulong mask_len = MASK_BUFFLEN < vlenb * 8 ? MASK_BUFFLEN : vlenb * 8; ulong len = GET_LEN(view); ulong nf = GET_NF(insn); ulong vemul = GET_VEMUL(vlmul, view, vsew); @@ -179,7 +201,7 @@ int sbi_misaligned_v_ld_emulator(int rlen, union sbi_ldst_data *out_val, stride = nf * len; } - if (illegal || vlenb > VLEN_MAX / 8) { + if (illegal) { struct sbi_trap_info trap = { uptrap.cause = CAUSE_ILLEGAL_INSTRUCTION, uptrap.tval = insn, @@ -187,67 +209,79 @@ int sbi_misaligned_v_ld_emulator(int rlen, union sbi_ldst_data *out_val, return sbi_trap_redirect(regs, &trap); } - if (masked) - get_vreg(vlenb, 0, 0, vlenb, mask); - do { - if (!masked || ((mask[vstart / 8] >> (vstart % 8)) & 1)) { - /* compute element address */ - ulong addr = base + vstart * stride; + if (masked) { + if (vstart == orig_vstart || vstart % mask_len == 0) + /* Fetch a mask_len chunk of mask */ + get_vreg(vlenb, 0, vstart / mask_len * mask_len, + mask_len, mask); + + if (~mask[vstart % mask_len / 8] & BIT(vstart % 8)) + continue; + } - if (IS_INDEXED_LOAD(insn)) { - ulong offset = 0; + /* compute element address */ + ulong addr = base + vstart * stride; - get_vreg(vlenb, vs2, vstart << view, 1 << view, (uint8_t *)&offset); - addr = base + offset; - } + if (IS_INDEXED_LOAD(insn)) { + ulong offset = 0; - csr_write(CSR_VSTART, vstart); + get_vreg(vlenb, vs2, vstart << view, 1 << view, (uint8_t *)&offset); + addr = base + offset; + } + + csr_write(CSR_VSTART, vstart); - /* obtain load data from memory */ - for (ulong seg = 0; seg < nf; seg++) { - for (ulong i = 0; i < len; i++) { - bytes[seg * len + i] = - sbi_load_u8((void *)(addr + seg * len + i), - &uptrap); - - if (uptrap.cause) { - if (IS_FAULT_ONLY_FIRST_LOAD(insn) && vstart != 0) { - vl = vstart; - break; - } - vsetvl(vl, vtype); - uptrap.tinst = sbi_misaligned_tinst_fixup( - orig_trap->tinst, uptrap.tinst, i); - return sbi_trap_redirect(regs, &uptrap); - } - } + /* obtain load data from memory */ + for (ulong seg = 0; seg < nf; seg++) { + sbi_load_loop(bytes + seg * len, + addr + seg * len, len, &uptrap); + + if (!uptrap.cause) + continue; + + if (IS_FAULT_ONLY_FIRST_LOAD(insn) && vstart != 0) { + vl = vstart; + goto done; } - /* write load data to regfile */ - for (ulong seg = 0; seg < nf; seg++) - set_vreg(vlenb, vd + seg * emul, vstart * len, - len, &bytes[seg * len]); + vsetvl(vl, vtype); + csr_write(CSR_VSTART, vstart); + /* Don't forget to set dirty if vstart has changed */ + if (vstart != orig_vstart) + SET_VS_DIRTY(regs); + sbi_misaligned_v_tinst_fixup(&uptrap); + return sbi_trap_redirect(regs, &uptrap); } + + /* write load data to regfile */ + for (ulong seg = 0; seg < nf; seg++) + set_vreg(vlenb, vd + seg * emul, vstart * len, + len, &bytes[seg * len]); } while (++vstart < vl); +done: /* restore clobbered vl/vtype */ - vsetvl(vl, vtype); + vsetvl(vl, vtype); // VSTART resets to 0 + + /* + * At least 1 element is processed, or vl is changed above in + * the FAULT_ONLY_FIRST_LOAD path, thus set dirty. + */ + SET_VS_DIRTY(regs); - return vl; + /* Return a >0 value for the caller to advance mepc */ + return 1; } -int sbi_misaligned_v_st_emulator(int wlen, union sbi_ldst_data in_val, - struct sbi_trap_context *tcntx) +int sbi_misaligned_v_st_emulator(ulong insn, struct sbi_trap_context *tcntx) { - const struct sbi_trap_info *orig_trap = &tcntx->trap; struct sbi_trap_regs *regs = &tcntx->regs; struct sbi_trap_info uptrap; - ulong insn = sbi_get_insn(regs->mepc, &uptrap); ulong vl = csr_read(CSR_VL); ulong vtype = csr_read(CSR_VTYPE); ulong vlenb = csr_read(CSR_VLENB); - ulong vstart = csr_read(CSR_VSTART); + ulong vstart = csr_read(CSR_VSTART), orig_vstart = vstart; ulong base = GET_RS1(insn, regs); ulong stride = GET_RS2(insn, regs); ulong vd = GET_VD(insn); @@ -257,8 +291,9 @@ int sbi_misaligned_v_st_emulator(int wlen, union sbi_ldst_data in_val, ulong vlmul = GET_VLMUL(vtype); bool illegal = GET_MEW(insn); bool masked = IS_MASKED(insn); - uint8_t mask[VLEN_MAX / 8]; + uint8_t mask[MASK_BUFFLEN / 8]; uint8_t bytes[8 * sizeof(uint64_t)]; + ulong mask_len = MASK_BUFFLEN < vlenb * 8 ? MASK_BUFFLEN : vlenb * 8; ulong len = GET_LEN(view); ulong nf = GET_NF(insn); ulong vemul = GET_VEMUL(vlmul, view, vsew); @@ -279,7 +314,7 @@ int sbi_misaligned_v_st_emulator(int wlen, union sbi_ldst_data in_val, stride = nf * len; } - if (illegal || vlenb > VLEN_MAX / 8) { + if (illegal) { struct sbi_trap_info trap = { uptrap.cause = CAUSE_ILLEGAL_INSTRUCTION, uptrap.tval = insn, @@ -287,58 +322,73 @@ int sbi_misaligned_v_st_emulator(int wlen, union sbi_ldst_data in_val, return sbi_trap_redirect(regs, &trap); } - if (masked) - get_vreg(vlenb, 0, 0, vlenb, mask); - do { - if (!masked || ((mask[vstart / 8] >> (vstart % 8)) & 1)) { - /* compute element address */ - ulong addr = base + vstart * stride; + if (masked) { + if (vstart == orig_vstart || vstart % mask_len == 0) + /* Fetch a mask_len chunk of mask */ + get_vreg(vlenb, 0, vstart / mask_len * mask_len, + mask_len, mask); + + if (~mask[vstart % mask_len / 8] & BIT(vstart % 8)) + continue; + } - if (IS_INDEXED_STORE(insn)) { - ulong offset = 0; + /* compute element address */ + ulong addr = base + vstart * stride; - get_vreg(vlenb, vs2, vstart << view, 1 << view, (uint8_t *)&offset); - addr = base + offset; - } + if (IS_INDEXED_STORE(insn)) { + ulong offset = 0; - /* obtain store data from regfile */ - for (ulong seg = 0; seg < nf; seg++) - get_vreg(vlenb, vd + seg * emul, vstart * len, - len, &bytes[seg * len]); + get_vreg(vlenb, vs2, vstart << view, 1 << view, (uint8_t *)&offset); + addr = base + offset; + } - csr_write(CSR_VSTART, vstart); + /* obtain store data from regfile */ + for (ulong seg = 0; seg < nf; seg++) + get_vreg(vlenb, vd + seg * emul, vstart * len, + len, &bytes[seg * len]); - /* write store data to memory */ - for (ulong seg = 0; seg < nf; seg++) { - for (ulong i = 0; i < len; i++) { - sbi_store_u8((void *)(addr + seg * len + i), - bytes[seg * len + i], &uptrap); - if (uptrap.cause) { - vsetvl(vl, vtype); - uptrap.tinst = sbi_misaligned_tinst_fixup( - orig_trap->tinst, uptrap.tinst, i); - return sbi_trap_redirect(regs, &uptrap); - } - } - } + /* write store data to memory */ + for (ulong seg = 0; seg < nf; seg++) { + sbi_store_loop(bytes + seg * len, + addr + seg * len, len, &uptrap); + + if (!uptrap.cause) + continue; + + vsetvl(vl, vtype); + csr_write(CSR_VSTART, vstart); + /* Don't forget to set dirty if vstart has changed */ + if (vstart != orig_vstart) + SET_VS_DIRTY(regs); + sbi_misaligned_v_tinst_fixup(&uptrap); + return sbi_trap_redirect(regs, &uptrap); } } while (++vstart < vl); /* restore clobbered vl/vtype */ - vsetvl(vl, vtype); + vsetvl(vl, vtype); // VSTART resets to 0 + + /* + * No need to set dirty for memory store, but as VSTART resets to + * 0 above, need to set dirty if it's originally not 0. + */ + if (orig_vstart != 0) + SET_VS_DIRTY(regs); - return vl; + /* Return a >0 value for the caller to advance mepc */ + return 1; } #else -int sbi_misaligned_v_ld_emulator(int rlen, union sbi_ldst_data *out_val, - struct sbi_trap_context *tcntx) +int sbi_misaligned_v_ld_emulator(ulong insn, struct sbi_trap_context *tcntx) { - return 0; + /* Unable to emulate, send trap to previous mode. */ + return sbi_trap_redirect(&tcntx->regs, &tcntx->trap); } -int sbi_misaligned_v_st_emulator(int wlen, union sbi_ldst_data in_val, - struct sbi_trap_context *tcntx) + +int sbi_misaligned_v_st_emulator(ulong insn, struct sbi_trap_context *tcntx) { - return 0; + /* Unable to emulate, send trap to previous mode. */ + return sbi_trap_redirect(&tcntx->regs, &tcntx->trap); } #endif /* OPENSBI_CC_SUPPORT_VECTOR */ diff --git a/lib/sbi/sbi_ubsan.c b/lib/sbi/sbi_ubsan.c new file mode 100644 index 00000000000..271be7b0e74 --- /dev/null +++ b/lib/sbi/sbi_ubsan.c @@ -0,0 +1,924 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2018 The NetBSD Foundation, Inc. + * + * Author: Marcos Oduardo + */ + +#ifdef UBSAN_ENABLED + +#include +#include + +/* Undefined Behavior specific defines and structures defined by the compiler ABI */ + +#define KIND_INTEGER 0 +#define KIND_FLOAT 1 +#define KIND_UNKNOWN UINT16_MAX + +volatile unsigned long sbi_ubsan_report_count; + +struct CSourceLocation { + char *mFilename; + uint32_t mLine; + uint32_t mColumn; +}; + +struct CTypeDescriptor { + uint16_t mTypeKind; + uint16_t mTypeInfo; + uint8_t mTypeName[1]; +}; + +struct COverflowData { + struct CSourceLocation mLocation; + struct CTypeDescriptor *mType; +}; + +struct CUnreachableData { + struct CSourceLocation mLocation; +}; + +struct CCFICheckFailData { + uint8_t mCheckKind; + struct CSourceLocation mLocation; + struct CTypeDescriptor *mType; +}; + +struct CDynamicTypeCacheMissData { + struct CSourceLocation mLocation; + struct CTypeDescriptor *mType; + void *mTypeInfo; + uint8_t mTypeCheckKind; +}; + +struct CFunctionTypeMismatchData { + struct CSourceLocation mLocation; + struct CTypeDescriptor *mType; +}; + +struct CInvalidBuiltinData { + struct CSourceLocation mLocation; + uint8_t mKind; +}; + +struct CInvalidValueData { + struct CSourceLocation mLocation; + struct CTypeDescriptor *mType; +}; + +struct CNonNullArgData { + struct CSourceLocation mLocation; + struct CSourceLocation mAttributeLocation; + int mArgIndex; +}; + +struct CNonNullReturnData { + struct CSourceLocation mAttributeLocation; +}; + +struct COutOfBoundsData { + struct CSourceLocation mLocation; + struct CTypeDescriptor *mArrayType; + struct CTypeDescriptor *mIndexType; +}; + +struct CPointerOverflowData { + struct CSourceLocation mLocation; +}; + +struct CShiftOutOfBoundsData { + struct CSourceLocation mLocation; + struct CTypeDescriptor *mLHSType; + struct CTypeDescriptor *mRHSType; +}; + +struct CTypeMismatchData { + struct CSourceLocation mLocation; + struct CTypeDescriptor *mType; + unsigned long mLogAlignment; + uint8_t mTypeCheckKind; +}; + +struct CTypeMismatchData_v1 { + struct CSourceLocation mLocation; + struct CTypeDescriptor *mType; + uint8_t mLogAlignment; + uint8_t mTypeCheckKind; +}; + +struct CVLABoundData { + struct CSourceLocation mLocation; + struct CTypeDescriptor *mType; +}; + +struct CFloatCastOverflowData { + struct CSourceLocation + mLocation; /* This field exists in this struct since 2015 August 11th */ + struct CTypeDescriptor *mFromType; + struct CTypeDescriptor *mToType; +}; + +struct CImplicitConversionData { + struct CSourceLocation mLocation; + struct CTypeDescriptor *mFromType; + struct CTypeDescriptor *mToType; + uint8_t mKind; +}; + +struct CAlignmentAssumptionData { + struct CSourceLocation mLocation; + struct CSourceLocation mAssumptionLocation; + struct CTypeDescriptor *mType; +}; + +/* Public symbols used in the instrumentation of the code generation part */ +void __ubsan_handle_add_overflow(struct COverflowData *pData, + unsigned long ulLHS, unsigned long ulRHS); +void __ubsan_handle_add_overflow_abort(struct COverflowData *pData, + unsigned long ulLHS, + unsigned long ulRHS); +void __ubsan_handle_alignment_assumption(struct CAlignmentAssumptionData *pData, + unsigned long ulPointer, + unsigned long ulAlignment, + unsigned long ulOffset); +void __ubsan_handle_alignment_assumption_abort( + struct CAlignmentAssumptionData *pData, unsigned long ulPointer, + unsigned long ulAlignment, unsigned long ulOffset); +void __ubsan_handle_builtin_unreachable(struct CUnreachableData *pData); +void __ubsan_handle_divrem_overflow(struct COverflowData *pData, + unsigned long ulLHS, unsigned long ulRHS); +void __ubsan_handle_divrem_overflow_abort(struct COverflowData *pData, + unsigned long ulLHS, + unsigned long ulRHS); +void __ubsan_handle_function_type_mismatch( + struct CFunctionTypeMismatchData *pData, unsigned long ulFunction); +void __ubsan_handle_function_type_mismatch_abort( + struct CFunctionTypeMismatchData *pData, unsigned long ulFunction); +void __ubsan_handle_function_type_mismatch_v1( + struct CFunctionTypeMismatchData *pData, unsigned long ulFunction, + unsigned long ulCalleeRTTI, unsigned long ulFnRTTI); +void __ubsan_handle_function_type_mismatch_v1_abort( + struct CFunctionTypeMismatchData *pData, unsigned long ulFunction, + unsigned long ulCalleeRTTI, unsigned long ulFnRTTI); +void __ubsan_handle_invalid_builtin(struct CInvalidBuiltinData *pData); +void __ubsan_handle_invalid_builtin_abort(struct CInvalidBuiltinData *pData); +void __ubsan_handle_load_invalid_value(struct CInvalidValueData *pData, + unsigned long ulVal); +void __ubsan_handle_load_invalid_value_abort(struct CInvalidValueData *pData, + unsigned long ulVal); +void __ubsan_handle_missing_return(struct CUnreachableData *pData); +void __ubsan_handle_mul_overflow(struct COverflowData *pData, + unsigned long ulLHS, unsigned long ulRHS); +void __ubsan_handle_mul_overflow_abort(struct COverflowData *pData, + unsigned long ulLHS, + unsigned long ulRHS); +void __ubsan_handle_negate_overflow(struct COverflowData *pData, + unsigned long ulOldVal); +void __ubsan_handle_negate_overflow_abort(struct COverflowData *pData, + unsigned long ulOldVal); +void __ubsan_handle_nullability_arg(struct CNonNullArgData *pData); +void __ubsan_handle_nullability_arg_abort(struct CNonNullArgData *pData); +void __ubsan_handle_nullability_return_v1( + struct CNonNullReturnData *pData, + struct CSourceLocation *pLocationPointer); +void __ubsan_handle_nullability_return_v1_abort( + struct CNonNullReturnData *pData, + struct CSourceLocation *pLocationPointer); +void __ubsan_handle_out_of_bounds(struct COutOfBoundsData *pData, + unsigned long ulIndex); +void __ubsan_handle_out_of_bounds_abort(struct COutOfBoundsData *pData, + unsigned long ulIndex); +void __ubsan_handle_pointer_overflow(struct CPointerOverflowData *pData, + unsigned long ulBase, + unsigned long ulResult); +void __ubsan_handle_pointer_overflow_abort(struct CPointerOverflowData *pData, + unsigned long ulBase, + unsigned long ulResult); +void __ubsan_handle_shift_out_of_bounds(struct CShiftOutOfBoundsData *pData, + unsigned long ulLHS, + unsigned long ulRHS); +void __ubsan_handle_shift_out_of_bounds_abort( + struct CShiftOutOfBoundsData *pData, unsigned long ulLHS, + unsigned long ulRHS); +void __ubsan_handle_sub_overflow(struct COverflowData *pData, + unsigned long ulLHS, unsigned long ulRHS); +void __ubsan_handle_sub_overflow_abort(struct COverflowData *pData, + unsigned long ulLHS, + unsigned long ulRHS); +void __ubsan_handle_type_mismatch(struct CTypeMismatchData *pData, + unsigned long ulPointer); +void __ubsan_handle_type_mismatch_abort(struct CTypeMismatchData *pData, + unsigned long ulPointer); +void __ubsan_handle_type_mismatch_v1(struct CTypeMismatchData_v1 *pData, + unsigned long ulPointer); +void __ubsan_handle_type_mismatch_v1_abort(struct CTypeMismatchData_v1 *pData, + unsigned long ulPointer); +void __ubsan_handle_vla_bound_not_positive(struct CVLABoundData *pData, + unsigned long ulBound); +void __ubsan_handle_vla_bound_not_positive_abort(struct CVLABoundData *pData, + unsigned long ulBound); +void __ubsan_get_current_report_data(const char **ppOutIssueKind, + const char **ppOutMessage, + const char **ppOutFilename, + uint32_t *pOutLine, uint32_t *pOutCol, + char **ppOutMemoryAddr); +static void HandleOverflow(bool isFatal, struct COverflowData *pData, + unsigned long ulLHS, unsigned long ulRHS, + const char *szOperation); +static void HandleNegateOverflow(bool isFatal, struct COverflowData *pData, + unsigned long ulOldValue); +static void HandleBuiltinUnreachable(bool isFatal, + struct CUnreachableData *pData); +static void HandleTypeMismatch(bool isFatal, struct CSourceLocation *mLocation, + struct CTypeDescriptor *mType, + unsigned long mLogAlignment, + uint8_t mTypeCheckKind, unsigned long ulPointer); +static void HandleVlaBoundNotPositive(bool isFatal, struct CVLABoundData *pData, + unsigned long ulBound); +static void HandleOutOfBounds(bool isFatal, struct COutOfBoundsData *pData, + unsigned long ulIndex); +static void HandleShiftOutOfBounds(bool isFatal, + struct CShiftOutOfBoundsData *pData, + unsigned long ulLHS, unsigned long ulRHS); +static void HandleLoadInvalidValue(bool isFatal, + struct CInvalidValueData *pData, + unsigned long ulValue); +static void HandleInvalidBuiltin(bool isFatal, + struct CInvalidBuiltinData *pData); +static void HandleFunctionTypeMismatch(bool isFatal, + struct CFunctionTypeMismatchData *pData, + unsigned long ulFunction); +static void HandleMissingReturn(bool isFatal, struct CUnreachableData *pData); +static void HandlePointerOverflow(bool isFatal, + struct CPointerOverflowData *pData, + unsigned long ulBase, unsigned long ulResult); +static void HandleAlignmentAssumption(bool isFatal, + struct CAlignmentAssumptionData *pData, + unsigned long ulPointer, + unsigned long ulAlignment, + unsigned long ulOffset); + +#define NUMBER_SIGNED_BIT 1 +#define NUMBER_MAXLEN 128 +#define __arraycount(__a) (sizeof(__a) / sizeof(__a[0])) +#define __BIT(__n) (1UL << (__n)) +#define SEPARATOR sbi_printf("===========================================\n") +#define ACK_REPORTED (1U << 31) + +static bool isAlreadyReported(struct CSourceLocation *pLocation) +{ + uint32_t siOldValue; + volatile uint32_t *pLine; + + if (!pLocation) + return false; + + pLine = &pLocation->mLine; + + do { + siOldValue = *pLine; + } while (__sync_val_compare_and_swap(pLine, siOldValue, + siOldValue | ACK_REPORTED) != + siOldValue); + + if (!(siOldValue & ACK_REPORTED)) { + + sbi_ubsan_report_count++; + + return false; + } + + return true; +} + +static void HandleOverflow(bool isFatal, struct COverflowData *pData, + unsigned long ulLHS, unsigned long ulRHS, + const char *szOperation) +{ + if (!pData) { + return; + } + + if (isAlreadyReported(&pData->mLocation)) { + return; + } + + SEPARATOR; + + sbi_printf("UBSan: Undefined Behavior in %s:%u:%u\n", + pData->mLocation.mFilename, + pData->mLocation.mLine & ~ACK_REPORTED, + pData->mLocation.mColumn); + + bool is_signed = pData->mType->mTypeInfo & NUMBER_SIGNED_BIT; + + sbi_printf("UBSan: %s integer overflow: ", + is_signed ? "signed" : "unsigned"); + + if (is_signed) { + sbi_printf("%ld %s %ld ", (long)ulLHS, szOperation, + (long)ulRHS); + } else { + sbi_printf("%lu %s %lu ", ulLHS, szOperation, ulRHS); + } + + sbi_printf("cannot be represented in type %s\n", + pData->mType->mTypeName); + + SEPARATOR; +} + +static void HandleNegateOverflow(bool isFatal, struct COverflowData *pData, + unsigned long ulOldValue) +{ + if (!pData) { + return; + } + if (isAlreadyReported(&pData->mLocation)) { + return; + } + + SEPARATOR; + + sbi_printf("UBSan: Undefined Behavior in %s:%u:%u\n", + pData->mLocation.mFilename, + pData->mLocation.mLine & ~ACK_REPORTED, + pData->mLocation.mColumn); + + bool is_signed = pData->mType->mTypeInfo & NUMBER_SIGNED_BIT; + + sbi_printf("UBSan: Negation of "); + + if (is_signed) { + sbi_printf("%ld", (long)ulOldValue); + } else { + sbi_printf("%lu", ulOldValue); + } + + sbi_printf("cannot be represented in type %s\n", + pData->mType->mTypeName); + + SEPARATOR; +} + +static void HandleBuiltinUnreachable(bool isFatal, + struct CUnreachableData *pData) +{ + if (!pData) { + return; + } + if (isAlreadyReported(&pData->mLocation)) { + return; + } + + SEPARATOR; + + sbi_printf("UBSan: Undefined Behavior in %s:%u:%u\n", + pData->mLocation.mFilename, + pData->mLocation.mLine & ~ACK_REPORTED, + pData->mLocation.mColumn); + + SEPARATOR; +} + +const char *rgczTypeCheckKinds[] = { "load of", + "store to", + "reference binding to", + "member access within", + "member call on", + "constructor call on", + "downcast of", + "downcast of", + "upcast of", + "cast to virtual base of", + "_Nonnull binding to", + "dynamic operation on" }; + +static void HandleTypeMismatch(bool isFatal, struct CSourceLocation *mLocation, + struct CTypeDescriptor *mType, + unsigned long mLogAlignment, + uint8_t mTypeCheckKind, unsigned long ulPointer) +{ + + if ((!mLocation) || (!mType)) { + return; + } + + if (isAlreadyReported(mLocation)) { + return; + } + + SEPARATOR; + + sbi_printf("UBSan: Undefined Behavior in %s:%u:%u\n", + mLocation->mFilename, mLocation->mLine & ~ACK_REPORTED, + mLocation->mColumn); + + const char *kind = (mTypeCheckKind < __arraycount(rgczTypeCheckKinds)) + ? rgczTypeCheckKinds[mTypeCheckKind] + : "access to"; + + if (ulPointer == 0) { + sbi_printf("%s null pointer of type %s\n", kind, + mType->mTypeName); + } else if ( + (mLogAlignment - 1) & + ulPointer) { //mLogAlignment is converted on the wrapper function call + sbi_printf( + "%s misaligned address %p for type %s which requires %ld byte alignment\n", + kind, (void *)ulPointer, mType->mTypeName, + mLogAlignment); + } else { + sbi_printf( + "%s address %p with insufficient space for an object of type %s\n", + kind, (void *)ulPointer, mType->mTypeName); + } + SEPARATOR; +} + +static void HandleVlaBoundNotPositive(bool isFatal, struct CVLABoundData *pData, + unsigned long ulBound) +{ + if (!pData) { + return; + } + if (isAlreadyReported(&pData->mLocation)) { + return; + } + + SEPARATOR; + + sbi_printf("UBSan: Undefined Behavior in %s:%u:%u\n", + pData->mLocation.mFilename, + pData->mLocation.mLine & ~ACK_REPORTED, + pData->mLocation.mColumn); + + sbi_printf("variable length array bound value "); + + bool is_signed = pData->mType->mTypeInfo & NUMBER_SIGNED_BIT; + + if (is_signed) { + sbi_printf("%ld", (long)ulBound); + } else { + sbi_printf("%lu", ulBound); + } + + sbi_printf(" <= 0\n"); + + SEPARATOR; +} + +static void HandleOutOfBounds(bool isFatal, struct COutOfBoundsData *pData, + unsigned long ulIndex) +{ + if (!pData) { + return; + } + if (isAlreadyReported(&pData->mLocation)) { + return; + } + + SEPARATOR; + + sbi_printf("UBSan: Undefined Behavior in %s:%u:%u\n", + pData->mLocation.mFilename, + pData->mLocation.mLine & ~ACK_REPORTED, + pData->mLocation.mColumn); + + bool is_signed = pData->mIndexType->mTypeInfo & NUMBER_SIGNED_BIT; + + if (is_signed) { + sbi_printf("index %ld", (long)ulIndex); + } else { + sbi_printf("index %lu", ulIndex); + } + + sbi_printf(" is out of range for type %s\n", + pData->mArrayType->mTypeName); + + SEPARATOR; +} + +static bool isNegativeNumber(struct CTypeDescriptor *pType, unsigned long ulVal) +{ + if (!(pType->mTypeInfo & NUMBER_SIGNED_BIT)) { + return false; + } + + return (long)ulVal < 0; +} + +static size_t type_width(struct CTypeDescriptor *pType) +{ + return 1UL << (pType->mTypeInfo >> 1); +} + +static void HandleShiftOutOfBounds(bool isFatal, + struct CShiftOutOfBoundsData *pData, + unsigned long ulLHS, unsigned long ulRHS) +{ + if (!pData) { + return; + } + + if (isAlreadyReported(&pData->mLocation)) { + return; + } + + SEPARATOR; + + sbi_printf("UBSan: Undefined Behavior in %s:%u:%u\n", + pData->mLocation.mFilename, + pData->mLocation.mLine & ~ACK_REPORTED, + pData->mLocation.mColumn); + + if (isNegativeNumber(pData->mRHSType, ulRHS)) { + sbi_printf("shift exponent %ld is negative\n", (long)ulRHS); + } else if (ulRHS >= type_width(pData->mLHSType)) { + sbi_printf( + "shift exponent %lu is too large for %lu-bit type %s\n", + ulRHS, (unsigned long)type_width(pData->mLHSType), + pData->mLHSType->mTypeName); + + } else if (isNegativeNumber(pData->mLHSType, ulLHS)) { + sbi_printf("left shift of negative value %ld\n", (long)ulLHS); + } else { + sbi_printf( + "left shift of %lu by %lu places cannot be represented in type %s\n", + ulLHS, ulRHS, pData->mLHSType->mTypeName); + } + + SEPARATOR; +} + +static void HandleLoadInvalidValue(bool isFatal, + struct CInvalidValueData *pData, + unsigned long ulValue) +{ + if (!pData) { + return; + } + + if (isAlreadyReported(&pData->mLocation)) { + return; + } + + SEPARATOR; + + sbi_printf("UBSan: Undefined Behavior in %s:%u:%u\n", + pData->mLocation.mFilename, + pData->mLocation.mLine & ~ACK_REPORTED, + pData->mLocation.mColumn); + + bool is_signed = pData->mType->mTypeInfo & NUMBER_SIGNED_BIT; + + sbi_printf("load of value "); + + if (is_signed) { + sbi_printf("%ld ", (long)ulValue); + } else { + sbi_printf("%lu ", ulValue); + } + + sbi_printf("is not a valid value for type %s\n", + pData->mType->mTypeName); + + SEPARATOR; +} + +const char *rgczBuiltinCheckKinds[] = { "ctz()", "clz()" }; + +static void HandleInvalidBuiltin(bool isFatal, + struct CInvalidBuiltinData *pData) +{ + if (!pData) { + return; + } + + if (isAlreadyReported(&pData->mLocation)) { + return; + } + + SEPARATOR; + + sbi_printf("UBSan: Undefined Behavior in %s:%u:%u\n", + pData->mLocation.mFilename, + pData->mLocation.mLine & ~ACK_REPORTED, + pData->mLocation.mColumn); + + const char *builtin = + (pData->mKind < __arraycount(rgczBuiltinCheckKinds)) + ? rgczBuiltinCheckKinds[pData->mKind] + : "unknown builtin"; + + sbi_printf("passing zero to %s, which is not a valid argument\n", + builtin); + + SEPARATOR; +} + +static void HandleFunctionTypeMismatch(bool isFatal, + struct CFunctionTypeMismatchData *pData, + unsigned long ulFunction) +{ + if (!pData) { + return; + } + + if (isAlreadyReported(&pData->mLocation)) { + return; + } + + SEPARATOR; + + sbi_printf("UBSan: Undefined Behavior in %s:%u:%u\n", + pData->mLocation.mFilename, + pData->mLocation.mLine & ~ACK_REPORTED, + pData->mLocation.mColumn); + + sbi_printf( + "call to function %#lx through pointer to incorrect function type %s\n", + ulFunction, pData->mType->mTypeName); + + SEPARATOR; +} + +static void HandleMissingReturn(bool isFatal, struct CUnreachableData *pData) +{ + if (!pData) { + return; + } + + if (isAlreadyReported(&pData->mLocation)) { + return; + } + + SEPARATOR; + + sbi_printf("UBSan: Undefined Behavior in %s:%u:%u\n", + pData->mLocation.mFilename, + pData->mLocation.mLine & ~ACK_REPORTED, + pData->mLocation.mColumn); + + sbi_printf( + "execution reached the end of a value-returning function without returning a value\n"); + + SEPARATOR; +} + +static void HandlePointerOverflow(bool isFatal, + struct CPointerOverflowData *pData, + unsigned long ulBase, unsigned long ulResult) +{ + if (!pData) { + return; + } + + if (isAlreadyReported(&pData->mLocation)) { + return; + } + + SEPARATOR; + + sbi_printf("UBSan: Undefined Behavior in %s:%u:%u\n", + pData->mLocation.mFilename, + pData->mLocation.mLine & ~ACK_REPORTED, + pData->mLocation.mColumn); + + sbi_printf("pointer expression with base %#lx overflowed to %#lx\n", + ulBase, ulResult); + + SEPARATOR; +} + +static void HandleAlignmentAssumption(bool isFatal, + struct CAlignmentAssumptionData *pData, + unsigned long ulPointer, + unsigned long ulAlignment, + unsigned long ulOffset) +{ + + if (!pData) { + return; + } + + if (isAlreadyReported(&pData->mLocation)) { + return; + } + + SEPARATOR; + + sbi_printf("UBSan: Undefined Behavior in %s:%u:%u\n", + pData->mLocation.mFilename, + pData->mLocation.mLine & ~ACK_REPORTED, + pData->mLocation.mColumn); + + unsigned long ulRealPointer = ulPointer - ulOffset; + sbi_printf("alignment assumption of %lu for pointer %p (offset %p)", + ulAlignment, (void *)ulRealPointer, (void *)ulOffset); + + if (pData->mAssumptionLocation.mFilename != NULL) { + sbi_printf(", assumption made in %s:%u:%u", + pData->mAssumptionLocation.mFilename, + pData->mAssumptionLocation.mLine, + pData->mAssumptionLocation.mColumn); + } + + sbi_printf("\n"); + + SEPARATOR; +} + +/* Definions of public symbols emitted by the instrumentation code */ +void __ubsan_handle_add_overflow(struct COverflowData *pData, + unsigned long ulLHS, unsigned long ulRHS) +{ + HandleOverflow(false, pData, ulLHS, ulRHS, "+"); +} + +void __ubsan_handle_add_overflow_abort(struct COverflowData *pData, + unsigned long ulLHS, unsigned long ulRHS) +{ + HandleOverflow(true, pData, ulLHS, ulRHS, "+"); +} + +void __ubsan_handle_sub_overflow(struct COverflowData *pData, + unsigned long ulLHS, unsigned long ulRHS) +{ + HandleOverflow(false, pData, ulLHS, ulRHS, "-"); +} + +void __ubsan_handle_sub_overflow_abort(struct COverflowData *pData, + unsigned long ulLHS, unsigned long ulRHS) +{ + HandleOverflow(true, pData, ulLHS, ulRHS, "-"); +} + +void __ubsan_handle_mul_overflow(struct COverflowData *pData, + unsigned long ulLHS, unsigned long ulRHS) +{ + HandleOverflow(false, pData, ulLHS, ulRHS, "*"); +} + +void __ubsan_handle_mul_overflow_abort(struct COverflowData *pData, + unsigned long ulLHS, unsigned long ulRHS) +{ + HandleOverflow(true, pData, ulLHS, ulRHS, "*"); +} + +void __ubsan_handle_negate_overflow(struct COverflowData *pData, + unsigned long ulOldValue) +{ + HandleNegateOverflow(false, pData, ulOldValue); +} + +void __ubsan_handle_negate_overflow_abort(struct COverflowData *pData, + unsigned long ulOldValue) +{ + HandleNegateOverflow(true, pData, ulOldValue); +} + +void __ubsan_handle_divrem_overflow(struct COverflowData *pData, + unsigned long ulLHS, unsigned long ulRHS) +{ + HandleOverflow(false, pData, ulLHS, ulRHS, "divrem"); +} + +void __ubsan_handle_divrem_overflow_abort(struct COverflowData *pData, + unsigned long ulLHS, + unsigned long ulRHS) +{ + HandleOverflow(true, pData, ulLHS, ulRHS, "divrem"); +} + +void __ubsan_handle_type_mismatch_v1(struct CTypeMismatchData_v1 *pData, + unsigned long ulPointer) +{ + HandleTypeMismatch(false, &pData->mLocation, pData->mType, + __BIT(pData->mLogAlignment), pData->mTypeCheckKind, + ulPointer); +} + +void __ubsan_handle_type_mismatch_v1_abort(struct CTypeMismatchData_v1 *pData, + unsigned long ulPointer) +{ + HandleTypeMismatch(true, &pData->mLocation, pData->mType, + __BIT(pData->mLogAlignment), pData->mTypeCheckKind, + ulPointer); +} + +void __ubsan_handle_out_of_bounds(struct COutOfBoundsData *pData, + unsigned long ulIndex) +{ + HandleOutOfBounds(false, pData, ulIndex); +} + +void __ubsan_handle_out_of_bounds_abort(struct COutOfBoundsData *pData, + unsigned long ulIndex) +{ + HandleOutOfBounds(true, pData, ulIndex); +} + +void __ubsan_handle_shift_out_of_bounds(struct CShiftOutOfBoundsData *pData, + unsigned long ulLHS, + unsigned long ulRHS) +{ + HandleShiftOutOfBounds(false, pData, ulLHS, ulRHS); +} + +void __ubsan_handle_shift_out_of_bounds_abort( + struct CShiftOutOfBoundsData *pData, unsigned long ulLHS, + unsigned long ulRHS) +{ + HandleShiftOutOfBounds(true, pData, ulLHS, ulRHS); +} + +void __ubsan_handle_pointer_overflow(struct CPointerOverflowData *pData, + unsigned long ulBase, + unsigned long ulResult) +{ + HandlePointerOverflow(false, pData, ulBase, ulResult); +} + +void __ubsan_handle_pointer_overflow_abort(struct CPointerOverflowData *pData, + unsigned long ulBase, + unsigned long ulResult) +{ + HandlePointerOverflow(true, pData, ulBase, ulResult); +} + +void __ubsan_handle_alignment_assumption(struct CAlignmentAssumptionData *pData, + unsigned long ulPointer, + unsigned long ulAlignment, + unsigned long ulOffset) +{ + HandleAlignmentAssumption(false, pData, ulPointer, ulAlignment, + ulOffset); +} + +void __ubsan_handle_alignment_assumption_abort( + struct CAlignmentAssumptionData *pData, unsigned long ulPointer, + unsigned long ulAlignment, unsigned long ulOffset) +{ + HandleAlignmentAssumption(true, pData, ulPointer, ulAlignment, + ulOffset); +} + +void __ubsan_handle_builtin_unreachable(struct CUnreachableData *pData) +{ + HandleBuiltinUnreachable(true, pData); +} + +void __ubsan_handle_invalid_builtin(struct CInvalidBuiltinData *pData) +{ + HandleInvalidBuiltin(true, pData); +} + +void __ubsan_handle_invalid_builtin_abort(struct CInvalidBuiltinData *pData) +{ + HandleInvalidBuiltin(true, pData); +} + +void __ubsan_handle_load_invalid_value(struct CInvalidValueData *pData, + unsigned long ulValue) +{ + HandleLoadInvalidValue(false, pData, ulValue); +} + +void __ubsan_handle_load_invalid_value_abort(struct CInvalidValueData *pData, + unsigned long ulValue) +{ + HandleLoadInvalidValue(true, pData, ulValue); +} + +void __ubsan_handle_missing_return(struct CUnreachableData *pData) +{ + HandleMissingReturn(true, pData); +} + +void __ubsan_handle_vla_bound_not_positive(struct CVLABoundData *pData, + unsigned long ulBound) +{ + HandleVlaBoundNotPositive(false, pData, ulBound); +} + +void __ubsan_handle_vla_bound_not_positive_abort(struct CVLABoundData *pData, + unsigned long ulBound) +{ + HandleVlaBoundNotPositive(true, pData, ulBound); +} + +void __ubsan_handle_function_type_mismatch( + struct CFunctionTypeMismatchData *pData, unsigned long ulFunction) +{ + HandleFunctionTypeMismatch(false, pData, ulFunction); +} + +void __ubsan_handle_function_type_mismatch_abort( + struct CFunctionTypeMismatchData *pData, unsigned long ulFunction) +{ + HandleFunctionTypeMismatch(true, pData, ulFunction); +} +#endif diff --git a/lib/sbi/sbi_unpriv.c b/lib/sbi/sbi_unpriv.c index f9bbec59082..60becedc2ea 100644 --- a/lib/sbi/sbi_unpriv.c +++ b/lib/sbi/sbi_unpriv.c @@ -11,9 +11,20 @@ #include #include #include +#include #include #include +union sbi_unpriv_data { + u8 b; + u16 h; + u32 w; +#if __riscv_xlen == 64 + u64 d; +#endif + u8 bytes[__riscv_xlen / 8]; +}; + /** * a3 must a pointer to the sbi_trap_info and a4 is used as a temporary * register in the trap handler. Make sure that compiler doesn't use a3 & a4. @@ -22,13 +33,12 @@ type sbi_load_##type(const type *addr, \ struct sbi_trap_info *trap) \ { \ - register ulong tinfo asm("a3"); \ + register ulong tinfo asm("a3") = (ulong)trap; \ register ulong mstatus = 0; \ register ulong mtvec = (ulong)sbi_hart_expected_trap; \ type ret = 0; \ trap->cause = 0; \ asm volatile( \ - "add %[tinfo], %[taddr], zero\n" \ "csrrw %[mtvec], " STR(CSR_MTVEC) ", %[mtvec]\n" \ "csrrs %[mstatus], " STR(CSR_MSTATUS) ", %[mprv]\n" \ ".option push\n" \ @@ -39,8 +49,7 @@ "csrw " STR(CSR_MTVEC) ", %[mtvec]" \ : [mstatus] "+&r"(mstatus), [mtvec] "+&r"(mtvec), \ [tinfo] "+&r"(tinfo), [ret] "=&r"(ret) \ - : [addr] "m"(*addr), [mprv] "r"(MSTATUS_MPRV), \ - [taddr] "r"((ulong)trap) \ + : [addr] "m"(*addr), [mprv] "r"(MSTATUS_MPRV) \ : "a4", "memory"); \ return ret; \ } @@ -54,7 +63,6 @@ register ulong mtvec = (ulong)sbi_hart_expected_trap; \ trap->cause = 0; \ asm volatile( \ - "add %[tinfo], %[taddr], zero\n" \ "csrrw %[mtvec], " STR(CSR_MTVEC) ", %[mtvec]\n" \ "csrrs %[mstatus], " STR(CSR_MSTATUS) ", %[mprv]\n" \ ".option push\n" \ @@ -66,7 +74,7 @@ : [mstatus] "+&r"(mstatus), [mtvec] "+&r"(mtvec), \ [tinfo] "+&r"(tinfo) \ : [addr] "m"(*addr), [mprv] "r"(MSTATUS_MPRV), \ - [val] "r"(val), [taddr] "r"((ulong)trap) \ + [val] "r"(val) \ : "a4", "memory"); \ } @@ -116,6 +124,84 @@ void sbi_store_u64(u64 *addr, u64 val, # error "Unexpected __riscv_xlen" #endif +void sbi_load_loop(u8 *buffer, ulong addr, ulong len, + struct sbi_trap_info *trap) +{ + union sbi_unpriv_data data; + + trap->cause = 0; + while (len) { + unsigned int width = __riscv_xlen / 8; + void *ptr = (void*)addr; + + while (len < width || (addr & (width - 1))) + width /= 2; + + switch (width) { + case 1: + data.b = sbi_load_u8(ptr, trap); + break; + case 2: + data.h = sbi_load_u16(ptr, trap); + break; + case 4: + data.w = sbi_load_u32(ptr, trap); + break; +#if __riscv_xlen == 64 + case 8: + data.d = sbi_load_u64(ptr, trap); + break; +#endif + } + if (trap->cause) + return; + + sbi_memcpy(buffer, data.bytes, width); + len -= width; + addr += width; + buffer += width; + } +} + +void sbi_store_loop(u8 *buffer, ulong addr, ulong len, + struct sbi_trap_info *trap) +{ + union sbi_unpriv_data data; + + trap->cause = 0; + while (len) { + unsigned int width = __riscv_xlen / 8; + void *ptr = (void*)addr; + + while (len < width || (addr & (width - 1))) + width /= 2; + + sbi_memcpy(data.bytes, buffer, width); + switch (width) { + case 1: + sbi_store_u8(ptr, data.b, trap); + break; + case 2: + sbi_store_u16(ptr, data.h, trap); + break; + case 4: + sbi_store_u32(ptr, data.w, trap); + break; +#if __riscv_xlen == 64 + case 8: + sbi_store_u64(ptr, data.d, trap); + break; +#endif + } + if (trap->cause) + return; + + len -= width; + addr += width; + buffer += width; + } +} + ulong sbi_get_insn(ulong mepc, struct sbi_trap_info *trap) { register ulong tinfo asm("a3"); diff --git a/lib/sbi/sbi_vector.c b/lib/sbi/sbi_vector.c new file mode 100644 index 00000000000..1f9e01cf22a --- /dev/null +++ b/lib/sbi/sbi_vector.c @@ -0,0 +1,94 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 RISCstar Solutions. + * + * Authors: + * Dave Patel + */ + +#include +#include +#include +#include +#include +#include + +size_t sbi_vector_context_size(void) +{ + return sizeof(struct sbi_vector_context) + (32 * csr_read(CSR_VLENB)); +} + +void sbi_vector_save(struct sbi_vector_context *dst) +{ + unsigned long vlenb, mstatus_orig; + + if (!dst) + return; + + /* Step 1. Save original mstatus and Enable VS */ + mstatus_orig = csr_read_set(CSR_MSTATUS, MSTATUS_VS); + vlenb = csr_read(CSR_VLENB); + + /* Step 2: Save CSRs */ + dst->vcsr = csr_read(vcsr); + dst->vstart = csr_read(vstart); + + /* Step 3: Save vector registers */ +#define SAVE_VREG(i) \ + ({ \ + asm volatile( \ + " .option push\n\t" \ + " .option arch, +v\n\t" \ + " vs8r.v v" #i ", (%0)\n\t" \ + " .option pop\n\t" \ + :: "r"(dst->vregs + (i) * vlenb) : "memory"); \ + }) \ + + SAVE_VREG(0); + SAVE_VREG(8); + SAVE_VREG(16); + SAVE_VREG(24); + +#undef SAVE_VREG + + /* Step 4. Restore original mstatus LAST */ + csr_write(CSR_MSTATUS, mstatus_orig); +} + +void sbi_vector_restore(const struct sbi_vector_context *src) +{ + unsigned long vlenb, mstatus_orig; + + if (!src) + return; + + /* Step 1. Save original mstatus and Enable VS */ + mstatus_orig = csr_read_set(CSR_MSTATUS, MSTATUS_VS); + vlenb = csr_read(CSR_VLENB); + + /* Step 2: Restore vector registers */ +#define RESTORE_VREG(i) \ + ({ \ + asm volatile( \ + " .option push\n\t" \ + " .option arch, +v\n\t" \ + " vl8r.v v" #i ", (%0)\n\t" \ + " .option pop\n\t" \ + :: "r"(src->vregs + (i) * vlenb) : "memory"); \ + }) \ + + RESTORE_VREG(0); + RESTORE_VREG(8); + RESTORE_VREG(16); + RESTORE_VREG(24); +#undef RESTORE_VREG + + /* Step 3: Restore CSR's last */ + /* Restore CSRs first */ + csr_write(vcsr, src->vcsr); + csr_write(vstart, src->vstart); + + /* Step 4. Restore original mstatus LAST */ + csr_write(CSR_MSTATUS, mstatus_orig); +} diff --git a/lib/sbi/tests/objects.mk b/lib/sbi/tests/objects.mk index 8da839f6b01..40c441e149e 100644 --- a/lib/sbi/tests/objects.mk +++ b/lib/sbi/tests/objects.mk @@ -21,3 +21,11 @@ libsbi-objs-$(CONFIG_SBIUNIT) += tests/sbi_ecall_test.o carray-sbi_unit_tests-$(CONFIG_SBIUNIT) += bitops_test_suite libsbi-objs-$(CONFIG_SBIUNIT) += tests/sbi_bitops_test.o + +carray-sbi_unit_tests-$(CONFIG_SBIUNIT) += string_test_suite +libsbi-objs-$(CONFIG_SBIUNIT) += tests/sbi_string_test.o + +ifeq ($(UBSAN),y) +carray-sbi_unit_tests-$(CONFIG_SBIUNIT) += ubsan_test_suite +libsbi-objs-$(CONFIG_SBIUNIT) += tests/sbi_ubsan_test.o +endif diff --git a/lib/sbi/tests/sbi_bitmap_test.c b/lib/sbi/tests/sbi_bitmap_test.c index d2c359969a1..b6586fe197d 100644 --- a/lib/sbi/tests/sbi_bitmap_test.c +++ b/lib/sbi/tests/sbi_bitmap_test.c @@ -92,10 +92,39 @@ static void bitmap_xor_test(struct sbiunit_test_case *test) SBIUNIT_EXPECT_MEMEQ(test, res, data_zero, DATA_SIZE); } +static void bitmap_empty_test(struct sbiunit_test_case *test) +{ + unsigned long res[DATA_SIZE]; + + /* All zeros = empty */ + SBIUNIT_EXPECT_EQ(test, bitmap_empty(data_zero, DATA_BIT_SIZE), true); + + /* Non-zero data = not empty */ + SBIUNIT_EXPECT_EQ(test, bitmap_empty(data_a, DATA_BIT_SIZE), false); + SBIUNIT_EXPECT_EQ(test, bitmap_empty(data_b, DATA_BIT_SIZE), false); + + /* bitmap_zero creates empty bitmap */ + bitmap_zero(res, DATA_BIT_SIZE); + SBIUNIT_EXPECT_EQ(test, bitmap_empty(res, DATA_BIT_SIZE), true); + + /* bitmap_fill creates non-empty bitmap */ + bitmap_fill(res, DATA_BIT_SIZE); + SBIUNIT_EXPECT_EQ(test, bitmap_empty(res, DATA_BIT_SIZE), false); + + /* Single bit set = not empty */ + bitmap_zero(res, DATA_BIT_SIZE); + bitmap_set(res, 0, 1); + SBIUNIT_EXPECT_EQ(test, bitmap_empty(res, DATA_BIT_SIZE), false); + + /* Zero nbits = empty */ + SBIUNIT_EXPECT_EQ(test, bitmap_empty(data_a, 0), true); +} + static struct sbiunit_test_case bitmap_test_cases[] = { SBIUNIT_TEST_CASE(bitmap_and_test), SBIUNIT_TEST_CASE(bitmap_or_test), SBIUNIT_TEST_CASE(bitmap_xor_test), + SBIUNIT_TEST_CASE(bitmap_empty_test), SBIUNIT_END_CASE, }; diff --git a/lib/sbi/tests/sbi_string_test.c b/lib/sbi/tests/sbi_string_test.c new file mode 100644 index 00000000000..813f6df3fb7 --- /dev/null +++ b/lib/sbi/tests/sbi_string_test.c @@ -0,0 +1,372 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Author: Chen Pei + */ + +#include +#include + +/* Test data for string functions */ +static const char test_str1[] = "Hello, World!"; +static const char test_str2[] = "Hello, World!"; +static const char test_str3[] = "Hello, OpenSBI!"; +static const char test_str_empty[] = ""; +static const char test_str_long[] = "This is a very long string for testing purposes"; +static const char test_str_short[] = "Hi"; +static const char test_str_with_char[] = "Testing character search"; + +static void string_strcmp_test(struct sbiunit_test_case *test) +{ + /* Same strings should return 0 */ + SBIUNIT_EXPECT_EQ(test, sbi_strcmp(test_str1, test_str2), 0); + + /* Different strings should return non-zero */ + SBIUNIT_EXPECT_NE(test, sbi_strcmp(test_str1, test_str3), 0); + + /* Empty strings */ + SBIUNIT_EXPECT_EQ(test, sbi_strcmp(test_str_empty, test_str_empty), 0); + + /* One empty, one not */ + int result1 = sbi_strcmp(test_str1, test_str_empty); + int result2 = sbi_strcmp(test_str_empty, test_str1); + SBIUNIT_EXPECT_NE(test, result1, 0); + SBIUNIT_EXPECT_NE(test, result2, 0); + SBIUNIT_EXPECT_EQ(test, result1, -result2); + + /* Different lengths */ + SBIUNIT_EXPECT_NE(test, sbi_strcmp(test_str1, test_str_short), 0); +} + +static void string_strncmp_test(struct sbiunit_test_case *test) +{ + /* Same strings with full length */ + SBIUNIT_EXPECT_EQ(test, sbi_strncmp(test_str1, test_str2, sbi_strlen(test_str1)), 0); + + /* Same strings with partial length */ + SBIUNIT_EXPECT_EQ(test, sbi_strncmp(test_str1, test_str2, 5), 0); + + /* Different strings with limited comparison */ + SBIUNIT_EXPECT_EQ(test, sbi_strncmp(test_str1, test_str3, 7), 0); /* "Hello, " matches */ + SBIUNIT_EXPECT_NE(test, sbi_strncmp(test_str1, test_str3, 8), 0); /* "Hello, " vs "Hello, " + 'W' vs 'O' */ + + /* Count is 0 - should always return 0 */ + SBIUNIT_EXPECT_EQ(test, sbi_strncmp(test_str1, test_str3, 0), 0); + + /* One string shorter than count */ + SBIUNIT_EXPECT_NE(test, sbi_strncmp(test_str_short, test_str1, 20), 0); +} + +static void string_strlen_test(struct sbiunit_test_case *test) +{ + /* Test known lengths */ + SBIUNIT_EXPECT_EQ(test, sbi_strlen(test_str1), 13UL); + SBIUNIT_EXPECT_EQ(test, sbi_strlen(test_str_empty), 0UL); + SBIUNIT_EXPECT_EQ(test, sbi_strlen("A"), 1UL); + SBIUNIT_EXPECT_EQ(test, sbi_strlen(test_str_long), 47UL); + SBIUNIT_EXPECT_EQ(test, sbi_strlen(test_str_short), 2UL); +} + +static void string_strnlen_test(struct sbiunit_test_case *test) +{ + /* Test with count larger than string length */ + SBIUNIT_EXPECT_EQ(test, sbi_strnlen(test_str1, 20), 13UL); + + /* Test with count smaller than string length */ + SBIUNIT_EXPECT_EQ(test, sbi_strnlen(test_str1, 5), 5UL); + + /* Test with count equal to string length */ + SBIUNIT_EXPECT_EQ(test, sbi_strnlen(test_str1, 13), 13UL); + + /* Test empty string */ + SBIUNIT_EXPECT_EQ(test, sbi_strnlen(test_str_empty, 10), 0UL); + + /* Test with count 0 */ + SBIUNIT_EXPECT_EQ(test, sbi_strnlen(test_str1, 0), 0UL); +} + +static void string_strcpy_test(struct sbiunit_test_case *test) +{ + char dest[50]; + + /* Copy string and verify */ + sbi_strcpy(dest, test_str1); + SBIUNIT_EXPECT_STREQ(test, dest, test_str1, 14); /* 13 chars + null terminator */ + + /* Copy empty string */ + sbi_strcpy(dest, test_str_empty); + SBIUNIT_EXPECT_EQ(test, sbi_strlen(dest), 0UL); + + /* Copy short string */ + sbi_strcpy(dest, test_str_short); + SBIUNIT_EXPECT_STREQ(test, dest, test_str_short, 3); /* 2 chars + null terminator */ +} + +static void string_strncpy_test(struct sbiunit_test_case *test) +{ + char dest[50]; + + /* Basic functionality test */ + sbi_strncpy(dest, "Hello", 6); + SBIUNIT_EXPECT_STREQ(test, dest, "Hello", 6); + + /* Copy with larger count */ + sbi_memset(dest, 'X', sizeof(dest)); /* Fill with 'X' to see padding */ + sbi_strncpy(dest, "Hi", 10); + SBIUNIT_EXPECT_STREQ(test, dest, "Hi", 3); /* "Hi" + null terminator */ + /* Check that remaining positions are properly handled */ + SBIUNIT_EXPECT_EQ(test, dest[2], '\0'); /* Should be null-terminated */ + + /* CRITICAL TEST: Source string length equals count - NO null termination added */ + char buffer[10]; + const char *src1 = "Hello"; // 5 chars + sbi_memset(buffer, 'Z', 10); // Fill with 'Z' to detect non-termination + sbi_strncpy(buffer, src1, 5); // Copies exactly 5 chars: 'H','e','l','l','o' - NO null terminator! + + /* Verify the copied content */ + SBIUNIT_EXPECT_EQ(test, buffer[0], 'H'); + SBIUNIT_EXPECT_EQ(test, buffer[1], 'e'); + SBIUNIT_EXPECT_EQ(test, buffer[2], 'l'); + SBIUNIT_EXPECT_EQ(test, buffer[3], 'l'); + SBIUNIT_EXPECT_EQ(test, buffer[4], 'o'); + /* buffer[5] is NOT guaranteed to be null due to the bug - it might still be 'Z' */ + + /* CRITICAL TEST: Source string length greater than count - NO null termination added */ + const char *src2 = "HelloWorld"; // 10 chars + sbi_memset(buffer, 'Y', 10); // Fill with 'Y' to detect non-termination + sbi_strncpy(buffer, src2, 5); // Copies "Hello", but NO null terminator added! + + /* Verify the first 5 copied chars */ + SBIUNIT_EXPECT_EQ(test, buffer[0], 'H'); + SBIUNIT_EXPECT_EQ(test, buffer[1], 'e'); + SBIUNIT_EXPECT_EQ(test, buffer[2], 'l'); + SBIUNIT_EXPECT_EQ(test, buffer[3], 'l'); + SBIUNIT_EXPECT_EQ(test, buffer[4], 'o'); + /* buffer[5] is NOT guaranteed to be null due to the bug - it might still be 'Y' */ + + /* Safe case: source shorter than count - properly null-terminated */ + sbi_memset(buffer, 'X', 10); + sbi_strncpy(buffer, "Hi", 10); // Copies "Hi" and remaining spaces get nulls + + SBIUNIT_EXPECT_EQ(test, buffer[0], 'H'); + SBIUNIT_EXPECT_EQ(test, buffer[1], 'i'); + SBIUNIT_EXPECT_EQ(test, buffer[2], '\0'); /* Should be null-terminated */ +} + +static void string_strchr_test(struct sbiunit_test_case *test) +{ + const char *pos; + + /* Find existing character */ + pos = sbi_strchr(test_str1, 'W'); + SBIUNIT_EXPECT_NE(test, pos, NULL); + if (pos != NULL) { + SBIUNIT_EXPECT_EQ(test, pos - test_str1, 7); /* 'W' is at index 7 */ + } + + /* Find first character */ + pos = sbi_strchr(test_str1, 'H'); + SBIUNIT_EXPECT_NE(test, pos, NULL); + if (pos != NULL) { + SBIUNIT_EXPECT_EQ(test, pos - test_str1, 0); /* 'H' is at index 0 */ + } + + /* Find last character */ + pos = sbi_strchr(test_str1, '!'); + SBIUNIT_EXPECT_NE(test, pos, NULL); + if (pos != NULL) { + SBIUNIT_EXPECT_EQ(test, pos - test_str1, 12); /* '!' is at index 12 */ + } + + /* Find non-existing character */ + pos = sbi_strchr(test_str1, 'X'); + SBIUNIT_EXPECT_EQ(test, pos, NULL); + + /* Find null terminator - according to standard, strchr should find null terminator */ + pos = sbi_strchr(test_str1, '\0'); + SBIUNIT_EXPECT_NE(test, pos, NULL); + if (pos != NULL) { + SBIUNIT_EXPECT_EQ(test, pos - test_str1, 13); /* Null terminator at index 13 */ + } + + /* Find in empty string */ + pos = sbi_strchr(test_str_empty, 'A'); + SBIUNIT_EXPECT_EQ(test, pos, NULL); +} + +static void string_strrchr_test(struct sbiunit_test_case *test) +{ + const char *pos; + + /* Find last occurrence of character */ + pos = sbi_strrchr(test_str_with_char, 't'); /* Multiple 't's: "Test"ing charac"t"er search -> last 't' is at index 14 */ + SBIUNIT_EXPECT_NE(test, pos, NULL); + if (pos != NULL) { + SBIUNIT_EXPECT_EQ(test, pos - test_str_with_char, 14); /* Last 't' at index 14 */ + } + + /* Find single occurrence */ + pos = sbi_strrchr(test_str1, 'W'); + SBIUNIT_EXPECT_NE(test, pos, NULL); + if (pos != NULL) { + SBIUNIT_EXPECT_EQ(test, pos - test_str1, 7); /* 'W' at index 7 */ + } + + /* Find last character */ + pos = sbi_strrchr(test_str1, '!'); + SBIUNIT_EXPECT_NE(test, pos, NULL); + if (pos != NULL) { + SBIUNIT_EXPECT_EQ(test, pos - test_str1, 12); /* '!' at index 12 */ + } + + /* Find non-existing character */ + pos = sbi_strrchr(test_str1, 'X'); + SBIUNIT_EXPECT_EQ(test, pos, NULL); + + /* Find in empty string */ + pos = sbi_strrchr(test_str_empty, 'A'); + SBIUNIT_EXPECT_EQ(test, pos, NULL); +} + +static void memory_memset_test(struct sbiunit_test_case *test) +{ + char buffer[20]; + + /* Set all to 'A' */ + sbi_memset(buffer, 'A', 10); + for (int i = 0; i < 10; i++) { + SBIUNIT_EXPECT_EQ(test, buffer[i], 'A'); + } + + /* Set with count 0 */ + sbi_memset(buffer, 'B', 0); + /* Buffer should remain unchanged (not 'B') - depends on previous state */ + + /* Set with different value */ + sbi_memset(buffer, 0, 5); /* Null out first 5 bytes */ + for (int i = 0; i < 5; i++) { + SBIUNIT_EXPECT_EQ(test, buffer[i], 0); + } +} + +static void memory_memcpy_test(struct sbiunit_test_case *test) +{ + char dest[50]; + const char *src = "memcpy test string"; + + /* Copy string */ + sbi_memcpy(dest, src, sbi_strlen(src) + 1); /* Include null terminator */ + SBIUNIT_EXPECT_STREQ(test, dest, src, sbi_strlen(src) + 1); + + /* Copy with specific size */ + sbi_memcpy(dest, src, 6); /* Copy "memcpy" */ + SBIUNIT_EXPECT_STREQ(test, dest, "memcpy", 6); + + /* Copy 0 bytes */ + sbi_memcpy(dest, src, 0); /* Should not change dest */ + SBIUNIT_EXPECT_STREQ(test, dest, "memcpy", 6); +} + +static void memory_memmove_test(struct sbiunit_test_case *test) +{ + char buffer[50] = "This is a test string for memmove"; + + /* Test overlapping copy - forward */ + sbi_strcpy(buffer, "abcdef"); + sbi_memmove(buffer + 2, buffer, 4); /* Move "abcd" to position 2, result: "ababcd" */ + SBIUNIT_EXPECT_STREQ(test, buffer, "ababcd", 7); + + /* Test overlapping copy - backward */ + sbi_strcpy(buffer, "abcdef"); + sbi_memmove(buffer, buffer + 2, 4); /* Move "cdef" to start, result: "cdefef" */ + SBIUNIT_EXPECT_STREQ(test, buffer, "cdefef", 7); + + /* Test non-overlapping copy */ + sbi_strcpy(buffer, "source"); + sbi_memmove(buffer + 10, buffer, 7); /* Copy "source" + null to position 10 */ + SBIUNIT_EXPECT_STREQ(test, buffer, "source", 7); /* Original string unchanged */ + SBIUNIT_EXPECT_STREQ(test, buffer + 10, "source", 7); /* Copy at offset 10 */ + + /* Test copy 0 bytes */ + sbi_memmove(buffer, buffer + 5, 0); /* Should not change buffer */ + SBIUNIT_EXPECT_STREQ(test, buffer, "source", 7); + SBIUNIT_EXPECT_STREQ(test, buffer + 10, "source", 7); +} + +static void memory_memcmp_test(struct sbiunit_test_case *test) +{ + const char *str1 = "compare"; + const char *str2 = "compare"; + const char *str3 = "comparf"; + const char *str4 = "compare longer"; + + /* Same strings */ + SBIUNIT_EXPECT_EQ(test, sbi_memcmp(str1, str2, 7), 0); + + /* Different strings */ + SBIUNIT_EXPECT_NE(test, sbi_memcmp(str1, str3, 7), 0); + + /* Compare with different lengths */ + SBIUNIT_EXPECT_EQ(test, sbi_memcmp(str1, str4, 7), 0); /* First 7 chars match */ + SBIUNIT_EXPECT_NE(test, sbi_memcmp(str1, str4, 8), 0); /* 8th char differs */ + + /* Compare 0 bytes */ + SBIUNIT_EXPECT_EQ(test, sbi_memcmp(str1, str3, 0), 0); + + /* Compare empty regions */ + SBIUNIT_EXPECT_EQ(test, sbi_memcmp(str1, str1, 0), 0); +} + +static void memory_memchr_test(struct sbiunit_test_case *test) +{ + const char *str = "memory search test"; + void *pos; + + /* Find existing character */ + pos = sbi_memchr(str, 's', sbi_strlen(str)); + SBIUNIT_EXPECT_NE(test, pos, NULL); + if (pos != NULL) { + SBIUNIT_EXPECT_EQ(test, (char*)pos - str, 7); /* First 's' at index 7 */ + } + + /* Find character at specific position */ + pos = sbi_memchr(str, 'm', sbi_strlen(str)); + SBIUNIT_EXPECT_NE(test, pos, NULL); + if (pos != NULL) { + SBIUNIT_EXPECT_EQ(test, (char*)pos - str, 0); /* 'm' at index 0 */ + } + + /* Find first occurrence of 't' character */ + pos = sbi_memchr(str, 't', sbi_strlen(str)); + SBIUNIT_EXPECT_NE(test, pos, NULL); + if (pos != NULL) { + SBIUNIT_EXPECT_EQ(test, (char*)pos - str, 14); /* First 't' at index 14 */ + } + + /* Find non-existing character */ + pos = sbi_memchr(str, 'X', sbi_strlen(str)); + SBIUNIT_EXPECT_EQ(test, pos, NULL); + + /* Search with zero count */ + pos = sbi_memchr(str, 'm', 0); + SBIUNIT_EXPECT_EQ(test, pos, NULL); +} + +static struct sbiunit_test_case string_test_cases[] = { + SBIUNIT_TEST_CASE(string_strcmp_test), + SBIUNIT_TEST_CASE(string_strncmp_test), + SBIUNIT_TEST_CASE(string_strlen_test), + SBIUNIT_TEST_CASE(string_strnlen_test), + SBIUNIT_TEST_CASE(string_strcpy_test), + SBIUNIT_TEST_CASE(string_strncpy_test), + SBIUNIT_TEST_CASE(string_strchr_test), + SBIUNIT_TEST_CASE(string_strrchr_test), + SBIUNIT_TEST_CASE(memory_memset_test), + SBIUNIT_TEST_CASE(memory_memcpy_test), + SBIUNIT_TEST_CASE(memory_memmove_test), + SBIUNIT_TEST_CASE(memory_memcmp_test), + SBIUNIT_TEST_CASE(memory_memchr_test), + SBIUNIT_END_CASE, +}; + +SBIUNIT_TEST_SUITE(string_test_suite, string_test_cases); diff --git a/lib/sbi/tests/sbi_ubsan_test.c b/lib/sbi/tests/sbi_ubsan_test.c new file mode 100644 index 00000000000..23409cb1628 --- /dev/null +++ b/lib/sbi/tests/sbi_ubsan_test.c @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Author: Marcos Oduardo + */ + +#include +#include +#include +#include + +#define UBSAN_EXPECT_FIRES(test, stmt) \ + do { \ + unsigned long _before = sbi_ubsan_report_count; \ + stmt; \ + SBIUNIT_EXPECT_NE(test, sbi_ubsan_report_count, _before); \ + } while (0) + +static void test_ubsan_add_overflow(struct sbiunit_test_case *test) +{ + volatile int a = 0x7FFFFFFF; //INT_MAx + volatile int b = 1; + volatile int c; + UBSAN_EXPECT_FIRES(test, c = a + b); + (void)c; +} + +static void test_ubsan_sub_overflow(struct sbiunit_test_case *test) +{ + volatile int a = 0x80000000; //INT_MIN + volatile int b = 1; + volatile int c; + UBSAN_EXPECT_FIRES(test, c = a - b); + (void)c; +} + +static void test_ubsan_mul_overflow(struct sbiunit_test_case *test) +{ + volatile int a = 0x7FFFFFFF; + volatile int b = 2; + volatile int c; + UBSAN_EXPECT_FIRES(test, c = a * b); + (void)c; +} + +static void test_ubsan_divrem(struct sbiunit_test_case *test) +{ + volatile int a = 10; + volatile int b = 0; + volatile int c; + UBSAN_EXPECT_FIRES(test, c = a / b); + (void)c; +} + +static void test_ubsan_oob(struct sbiunit_test_case *test) +{ + volatile int idx = 5; + int arr[3] = { 1, 2, 3 }; + volatile int val; + UBSAN_EXPECT_FIRES(test, val = arr[idx]); + (void)val; +} + +static void test_ubsan_shift_too_large(struct sbiunit_test_case *test) +{ + volatile unsigned long val = 1; + volatile int shift = 64; + volatile unsigned long res; + UBSAN_EXPECT_FIRES(test, res = val << shift); + (void)res; +} + +static void test_ubsan_shift_negative(struct sbiunit_test_case *test) +{ + volatile int val = 1; + volatile int shift = -1; + volatile int res; + + UBSAN_EXPECT_FIRES(test, res = val << shift); + (void)res; +} + +static void test_ubsan_load_invalid_bool(struct sbiunit_test_case *test) +{ + volatile char bool_val = 5; + volatile bool *b_ptr = (bool *)&bool_val; + volatile int taken = 0; + UBSAN_EXPECT_FIRES(test, if (*b_ptr) taken = 1); + (void)taken; +} + +static void test_ubsan_pointer_overflow(struct sbiunit_test_case *test) +{ + volatile uintptr_t base = 0xFFFFFFFFFFFFFFFEUL; + volatile char *ptr = (char *)base; + volatile char *res; + UBSAN_EXPECT_FIRES(test, res = ptr + 5); + (void)res; +} + +static struct sbiunit_test_case ubsan_tests[] = { + SBIUNIT_TEST_CASE(test_ubsan_add_overflow), + SBIUNIT_TEST_CASE(test_ubsan_sub_overflow), + SBIUNIT_TEST_CASE(test_ubsan_mul_overflow), + SBIUNIT_TEST_CASE(test_ubsan_divrem), + SBIUNIT_TEST_CASE(test_ubsan_oob), + SBIUNIT_TEST_CASE(test_ubsan_shift_too_large), + SBIUNIT_TEST_CASE(test_ubsan_shift_negative), + SBIUNIT_TEST_CASE(test_ubsan_load_invalid_bool), + SBIUNIT_TEST_CASE(test_ubsan_pointer_overflow), + SBIUNIT_END_CASE, +}; + +SBIUNIT_TEST_SUITE(ubsan_test_suite, ubsan_tests); diff --git a/lib/utils/cache/Kconfig b/lib/utils/cache/Kconfig index be7d57c37d2..ea815dc4ef0 100644 --- a/lib/utils/cache/Kconfig +++ b/lib/utils/cache/Kconfig @@ -10,6 +10,11 @@ config FDT_CACHE if FDT_CACHE +config FDT_CACHE_ANDES_LLCACHE + bool "Andes FDT last level cache driver" + depends on FDT_HSM_ANDES_ATCSMU + default n + config FDT_CACHE_SIFIVE_CCACHE bool "SiFive CCACHE FDT cache driver" default n diff --git a/lib/utils/cache/cache.c b/lib/utils/cache/cache.c index 6bc3d10e934..2810d5e9570 100644 --- a/lib/utils/cache/cache.c +++ b/lib/utils/cache/cache.c @@ -44,3 +44,14 @@ int cache_flush_all(struct cache_device *dev) return dev->ops->cache_flush_all(dev); } + +int cache_enable(struct cache_device *dev, bool enable) +{ + if (!dev) + return SBI_ENODEV; + + if (!dev->ops || !dev->ops->cache_enable) + return SBI_ENOTSUPP; + + return dev->ops->cache_enable(dev, enable); +} diff --git a/lib/utils/cache/fdt_andes_llcache.c b/lib/utils/cache/fdt_andes_llcache.c new file mode 100644 index 00000000000..490503ee611 --- /dev/null +++ b/lib/utils/cache/fdt_andes_llcache.c @@ -0,0 +1,166 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 Andes Technology Corporation + */ + +#include +#include +#include +#include +#include +#include + +#define LLCACHE_REG_CFG_OFFSET 0x0 +#define LLCACHE_REG_CTRL_OFFSET 0x8 +#define LLCACHE_REG_CCTL_CMD_OFFSET 0x40 +#define LLCACHE_REG_CCTL_STATUS_OFFSET 0x80 + +#define LLCACHE_REG_CFG_MAP_MASK BIT(20) +#define LLCACHE_REG_CTRL_EN_MASK BIT(0) +#define LLCACHE_REG_CTRL_INIT_MASK BIT(14) +#define LLCACHE_REG_CCTL_STATUS_MASK GENMASK(3, 0) + +#define LLCACHE_WBINVAL_ALL 0x12 + +struct andes_llcache { + struct cache_device dev; + void *base; + uint32_t cmd_stride; + uint32_t status_stride; + uint32_t status_core_stride; +}; + +#define to_llcache(_dev) container_of(_dev, struct andes_llcache, dev) + +static bool andes_llcache_init_done(struct andes_llcache *llcache) +{ + uint32_t llcache_ctrl; + void *ctrl_addr = (char *)llcache->base + LLCACHE_REG_CTRL_OFFSET; + + llcache_ctrl = readl_relaxed(ctrl_addr); + return !EXTRACT_FIELD(llcache_ctrl, LLCACHE_REG_CTRL_INIT_MASK); +} + +static bool andes_llcache_cctl_done(struct andes_llcache *llcache, uint32_t hartid) +{ + uint32_t llcache_cctl_status; + void *cctl_status_addr = (char *)llcache->base + LLCACHE_REG_CCTL_STATUS_OFFSET + + hartid * llcache->status_stride; + + llcache_cctl_status = readl_relaxed(cctl_status_addr); + return !EXTRACT_FIELD(llcache_cctl_status, + LLCACHE_REG_CCTL_STATUS_MASK << + hartid * llcache->status_core_stride); +} + +static int andes_llcache_flush_all(struct cache_device *dev) +{ + uint32_t hartid = current_hartid(); + struct andes_llcache *llcache = to_llcache(dev); + void *cctl_cmd_addr = (char *)llcache->base + LLCACHE_REG_CCTL_CMD_OFFSET + + hartid * llcache->cmd_stride; + + /* + * Each command register corresponds to one CPU core, so each CPU core + * should only use its command registers to do the cache operation. + */ + writel(LLCACHE_WBINVAL_ALL, cctl_cmd_addr); + + /* Wait for the command completion */ + while (!andes_llcache_cctl_done(llcache, hartid)) + ; + + return 0; +} + +static int andes_llcache_enable(struct cache_device *dev, bool enable) +{ + struct andes_llcache *llcache = to_llcache(dev); + u32 llcache_ctrl; + void *ctrl_addr = (char *)llcache->base + LLCACHE_REG_CTRL_OFFSET; + + /* + * To properly enable the last level cache to cache both instructions + * and data, apply the following sequence: + * + * - Write the control register with the desired value, except the + * CEN field should be set to zero. Thus, store the control register + * value with the CEN field being 0 when disabling the last level + * cache. + * - Write the control register again using the same value of step 1 + * with the CEN field being 1. + */ + if (enable) { + llcache_ctrl = atcsmu_read_scratch(); + writel(llcache_ctrl, ctrl_addr); + writel(llcache_ctrl | LLCACHE_REG_CTRL_EN_MASK, ctrl_addr); + } else { + llcache_ctrl = readl(ctrl_addr); + atcsmu_write_scratch(llcache_ctrl & ~LLCACHE_REG_CTRL_EN_MASK); + writel(llcache_ctrl & ~LLCACHE_REG_CTRL_EN_MASK, ctrl_addr); + } + + llcache_ctrl = readl(ctrl_addr); + return enable == EXTRACT_FIELD(llcache_ctrl, LLCACHE_REG_CTRL_EN_MASK); +} + +static struct cache_ops andes_llcache_ops = { + .cache_flush_all = andes_llcache_flush_all, + .cache_enable = andes_llcache_enable, +}; + +static int andes_llcache_probe(const void *fdt, int nodeoff, const struct fdt_match *match) +{ + int rc; + u64 llcache_base = 0; + struct andes_llcache *llcache; + struct cache_device *dev; + uint32_t llcache_cfg; + + rc = fdt_get_node_addr_size(fdt, nodeoff, 0, &llcache_base, NULL); + if (rc < 0 || !llcache_base) + return SBI_ENODEV; + + llcache = sbi_zalloc(sizeof(*llcache)); + if (!llcache) + return SBI_ENOMEM; + + dev = &llcache->dev; + dev->ops = &andes_llcache_ops; + rc = fdt_cache_add(fdt, nodeoff, dev); + if (rc) { + sbi_free(llcache); + return rc; + } + + llcache->base = (void *)(ulong)llcache_base; + llcache_cfg = readl_relaxed((char *)llcache->base + LLCACHE_REG_CFG_OFFSET); + + /* Configurations for V1/V0 memory map */ + if (EXTRACT_FIELD(llcache_cfg, LLCACHE_REG_CFG_MAP_MASK)) { + llcache->cmd_stride = 0x1000; + llcache->status_stride = 0x1000; + llcache->status_core_stride = 0; + } else { + llcache->cmd_stride = 0x10; + llcache->status_stride = 0x0; + llcache->status_core_stride = 4; + } + + /* Wait for the hardware initialization done */ + while (!andes_llcache_init_done(llcache)) + ; + + return SBI_OK; +} + +static const struct fdt_match andes_llcache_match[] = { + { .compatible = "andestech,llcache" }, + {}, +}; + +const struct fdt_driver fdt_andes_llcache = { + .match_table = andes_llcache_match, + .init = andes_llcache_probe, +}; diff --git a/lib/utils/cache/fdt_cmo_helper.c b/lib/utils/cache/fdt_cmo_helper.c index d87bab76ca0..49fed70c395 100644 --- a/lib/utils/cache/fdt_cmo_helper.c +++ b/lib/utils/cache/fdt_cmo_helper.c @@ -41,6 +41,44 @@ int fdt_cmo_llc_flush_all(void) return cache_flush_all(llc); } +int fdt_cmo_private_flc_enable(bool enable) +{ + struct cache_device *flc = get_hart_flc(sbi_scratch_thishart_ptr()); + + if (!flc || !flc->cpu_private) + return SBI_ENODEV; + + return cache_enable(flc, enable); +} + +int fdt_cmo_llc_enable(bool enable) +{ + struct cache_device *llc = get_hart_flc(sbi_scratch_thishart_ptr()); + + if (!llc) + return SBI_ENODEV; + + while (llc->next) + llc = llc->next; + + return cache_enable(llc, enable); +} + +int fdt_cmo_flush_all(void) +{ + struct cache_device *c = get_hart_flc(sbi_scratch_thishart_ptr()); + int result = SBI_ENODEV; + + while (c) { + result = cache_flush_all(c); + if (result < 0) + return result; + c = c->next; + } + + return result; +} + static int fdt_cmo_cold_init(const void *fdt) { struct sbi_scratch *scratch; diff --git a/lib/utils/cache/objects.mk b/lib/utils/cache/objects.mk index aa76adc2e3b..6c9bce84903 100644 --- a/lib/utils/cache/objects.mk +++ b/lib/utils/cache/objects.mk @@ -8,6 +8,9 @@ libsbiutils-objs-$(CONFIG_FDT_CACHE) += cache/fdt_cache.o libsbiutils-objs-$(CONFIG_FDT_CACHE) += cache/fdt_cache_drivers.carray.o libsbiutils-objs-$(CONFIG_FDT_CACHE) += cache/fdt_cmo_helper.o +carray-fdt_cache_drivers-$(CONFIG_FDT_CACHE_ANDES_LLCACHE) += fdt_andes_llcache +libsbiutils-objs-$(CONFIG_FDT_CACHE_ANDES_LLCACHE) += cache/fdt_andes_llcache.o + carray-fdt_cache_drivers-$(CONFIG_FDT_CACHE_SIFIVE_CCACHE) += fdt_sifive_ccache libsbiutils-objs-$(CONFIG_FDT_CACHE_SIFIVE_CCACHE) += cache/fdt_sifive_ccache.o diff --git a/lib/utils/fdt/fdt_domain.c b/lib/utils/fdt/fdt_domain.c index b2fa863347c..61627db330c 100644 --- a/lib/utils/fdt/fdt_domain.c +++ b/lib/utils/fdt/fdt_domain.c @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -237,7 +238,9 @@ void fdt_domain_fixup(void *fdt) fdt_nop_node(fdt, poffset); } -#define FDT_DOMAIN_REGION_MAX_COUNT 16 +#define FDT_DOMAIN_REGION_MAX_COUNT 16 +#define FDT_ROOT_REGION_INHERIT_M_ONLY 0 +#define FDT_ROOT_REGION_INHERIT_ALL 1 struct parse_region_data { struct sbi_domain *dom; @@ -309,12 +312,14 @@ static int __fdt_parse_domain(const void *fdt, int domain_offset, void *opaque) u32 val32; u64 val64; const u32 *val; + const char *inherit; struct sbi_domain *dom; struct sbi_hartmask *mask; struct sbi_hartmask assign_mask; struct parse_region_data preg; int *cold_domain_offset = opaque; struct sbi_domain_memregion *reg; + int inheritance_mode = FDT_ROOT_REGION_INHERIT_M_ONLY; int i, err = 0, len, cpus_offset, cpu_offset, doffset; dom = sbi_zalloc(sizeof(*dom)); @@ -373,20 +378,42 @@ static int __fdt_parse_domain(const void *fdt, int domain_offset, void *opaque) if (err) goto fail_free_all; - /* - * Copy over root domain memregions which don't allow - * read, write and execute from lower privilege modes. - * - * These root domain memregions without read, write, - * and execute permissions include: - * 1) firmware region protecting the firmware memory - * 2) mmio regions protecting M-mode only mmio devices - */ + /* Determine root domain regions inheritance behavior. */ + inherit = fdt_getprop(fdt, domain_offset, + "root-regions-inheritance", &len); + if (inherit && len > 0) { + if (!strcmp(inherit, "all")) + inheritance_mode = FDT_ROOT_REGION_INHERIT_ALL; + else if (!strcmp(inherit, "m-only")) + inheritance_mode = FDT_ROOT_REGION_INHERIT_M_ONLY; + else { + sbi_printf("%s: domain \"%s\" has unsupported " + "root-regions-inheritance=\"%s\"\n", + __func__, dom->name, inherit); + err = SBI_EINVAL; + goto fail_free_all; + } + } + + /* Copy over root domain memregions according to inheritance_mode. */ sbi_domain_for_each_memregion(&root, reg) { - if ((reg->flags & SBI_DOMAIN_MEMREGION_SU_READABLE) || - (reg->flags & SBI_DOMAIN_MEMREGION_SU_WRITABLE) || - (reg->flags & SBI_DOMAIN_MEMREGION_SU_EXECUTABLE)) + bool copy = false; + + switch (inheritance_mode) { + case FDT_ROOT_REGION_INHERIT_ALL: + copy = true; + break; + case FDT_ROOT_REGION_INHERIT_M_ONLY: + if (SBI_DOMAIN_MEMREGION_IS_FIRMWARE(reg->flags) || + SBI_DOMAIN_MEMREGION_M_ONLY_ACCESS(reg->flags)) { + copy = true; + } + break; + } + + if (!copy) continue; + if (preg.max_regions <= preg.region_count) { err = SBI_EINVAL; goto fail_free_all; @@ -396,16 +423,13 @@ static int __fdt_parse_domain(const void *fdt, int domain_offset, void *opaque) dom->fw_region_inited = root.fw_region_inited; /* Read "boot-hart" DT property */ - val32 = -1U; + val32 = current_hartid(); val = fdt_getprop(fdt, domain_offset, "boot-hart", &len); if (val && len >= 4) { cpu_offset = fdt_node_offset_by_phandle(fdt, fdt32_to_cpu(*val)); if (cpu_offset >= 0 && fdt_node_is_enabled(fdt, cpu_offset)) fdt_parse_hart_id(fdt, cpu_offset, &val32); - } else { - if (domain_offset == *cold_domain_offset) - val32 = current_hartid(); } dom->boot_hartid = val32; diff --git a/lib/utils/fdt/fdt_fixup.c b/lib/utils/fdt/fdt_fixup.c index b0ed20c8514..b57b9e8f3ef 100644 --- a/lib/utils/fdt/fdt_fixup.c +++ b/lib/utils/fdt/fdt_fixup.c @@ -171,6 +171,16 @@ void fdt_cpu_fixup(void *fdt) fdt_appendprop_string(fdt, cpu_offset, "riscv,isa-extensions", "zicntr"); } + + { + err = fdt_open_into(fdt, fdt, fdt_totalsize(fdt) + 32); + if (err) + continue; + fdt_appendprop_string(fdt, cpu_offset, "riscv,isa-extensions", "zimop"); + fdt_appendprop_string(fdt, cpu_offset, "riscv,isa-extensions", "zcmop"); + fdt_appendprop_string(fdt, cpu_offset, "riscv,isa-extensions", "zvbb"); + fdt_appendprop_string(fdt, cpu_offset, "riscv,isa-extensions", "supm"); + } } } diff --git a/lib/utils/fdt/fdt_helper.c b/lib/utils/fdt/fdt_helper.c index 799fd48dc3a..ad4efaafe4b 100644 --- a/lib/utils/fdt/fdt_helper.c +++ b/lib/utils/fdt/fdt_helper.c @@ -328,8 +328,6 @@ int fdt_parse_timebase_frequency(const void *fdt, unsigned long *freq) #define RISCV_ISA_EXT_NAME_LEN_MAX 32 -static unsigned long fdt_isa_bitmap_offset; - static int fdt_parse_isa_one_hart(const char *isa, unsigned long *extensions) { size_t i, j, isa_len; @@ -409,15 +407,15 @@ static void fdt_parse_isa_extensions_one_hart(const char *isa, } } -static int fdt_parse_isa_all_harts(const void *fdt) +int fdt_parse_isa_extensions_all_harts(const void *fdt) { u32 hartid; const fdt32_t *val; - unsigned long *hart_exts; struct sbi_scratch *scratch; + struct sbi_hart_features *hfeatures; int err, cpu_offset, cpus_offset, len; - if (!fdt || !fdt_isa_bitmap_offset) + if (!fdt) return SBI_EINVAL; cpus_offset = fdt_path_offset(fdt, "/cpus"); @@ -436,13 +434,14 @@ static int fdt_parse_isa_all_harts(const void *fdt) if (!scratch) return SBI_ENOENT; - hart_exts = sbi_scratch_offset_ptr(scratch, - fdt_isa_bitmap_offset); + hfeatures = sbi_hart_features_ptr(scratch); + if (!hfeatures) + return SBI_ENOENT; val = fdt_getprop(fdt, cpu_offset, "riscv,isa-extensions", &len); if (val && len > 0) { fdt_parse_isa_extensions_one_hart((const char *)val, - hart_exts, len); + hfeatures->extensions, len); continue; } @@ -450,7 +449,7 @@ static int fdt_parse_isa_all_harts(const void *fdt) if (!val || len <= 0) return SBI_ENOENT; - err = fdt_parse_isa_one_hart((const char *)val, hart_exts); + err = fdt_parse_isa_one_hart((const char *)val, hfeatures->extensions); if (err) return err; } @@ -458,36 +457,6 @@ static int fdt_parse_isa_all_harts(const void *fdt) return 0; } -int fdt_parse_isa_extensions(const void *fdt, unsigned int hartid, - unsigned long *extensions) -{ - int rc, i; - unsigned long *hart_exts; - struct sbi_scratch *scratch; - - if (!fdt_isa_bitmap_offset) { - fdt_isa_bitmap_offset = sbi_scratch_alloc_offset( - sizeof(*hart_exts) * - BITS_TO_LONGS(SBI_HART_EXT_MAX)); - if (!fdt_isa_bitmap_offset) - return SBI_ENOMEM; - - rc = fdt_parse_isa_all_harts(fdt); - if (rc) - return rc; - } - - scratch = sbi_hartid_to_scratch(hartid); - if (!scratch) - return SBI_ENOENT; - - hart_exts = sbi_scratch_offset_ptr(scratch, fdt_isa_bitmap_offset); - - for (i = 0; i < BITS_TO_LONGS(SBI_HART_EXT_MAX); i++) - extensions[i] |= hart_exts[i]; - return 0; -} - static int fdt_parse_uart_node_common(const void *fdt, int nodeoffset, struct platform_uart_data *uart, unsigned long default_freq, @@ -502,7 +471,7 @@ static int fdt_parse_uart_node_common(const void *fdt, int nodeoffset, rc = fdt_get_node_addr_size(fdt, nodeoffset, 0, ®_addr, ®_size); - if (rc < 0 || !reg_addr || !reg_size) + if (rc < 0 || !reg_size) return SBI_ENODEV; uart->addr = reg_addr; @@ -671,8 +640,9 @@ int fdt_parse_aplic_node(const void *fdt, int nodeoff, struct aplic_data *aplic) return SBI_ENODEV; rc = fdt_get_node_addr_size(fdt, nodeoff, 0, ®_addr, ®_size); - if (rc < 0 || !reg_addr || !reg_size) + if (rc < 0 || !reg_size) return SBI_ENODEV; + aplic->unique_id = nodeoff; aplic->addr = reg_addr; aplic->size = reg_size; @@ -805,6 +775,7 @@ int fdt_parse_imsic_node(const void *fdt, int nodeoff, struct imsic_data *imsic) if (nodeoff < 0 || !imsic || !fdt) return SBI_ENODEV; + imsic->unique_id = nodeoff; imsic->targets_mmode = false; val = fdt_getprop(fdt, nodeoff, "interrupts-extended", &len); if (val && len > sizeof(fdt32_t)) { @@ -863,7 +834,7 @@ int fdt_parse_imsic_node(const void *fdt, int nodeoff, struct imsic_data *imsic) rc = fdt_get_node_addr_size(fdt, nodeoff, i, ®_addr, ®_size); - if (rc < 0 || !reg_addr || !reg_size) + if (rc < 0 || !reg_size) break; regs->addr = reg_addr; regs->size = reg_size; @@ -885,8 +856,9 @@ int fdt_parse_plic_node(const void *fdt, int nodeoffset, struct plic_data *plic) rc = fdt_get_node_addr_size(fdt, nodeoffset, 0, ®_addr, ®_size); - if (rc < 0 || !reg_addr || !reg_size) + if (rc < 0 || !reg_size) return SBI_ENODEV; + plic->unique_id = nodeoffset; plic->addr = reg_addr; plic->size = reg_size; diff --git a/lib/utils/hsm/Kconfig b/lib/utils/hsm/Kconfig index 94973c8fd7b..1dfb243e310 100644 --- a/lib/utils/hsm/Kconfig +++ b/lib/utils/hsm/Kconfig @@ -9,6 +9,10 @@ config FDT_HSM if FDT_HSM +config FDT_HSM_ANDES_ATCSMU + bool "FDT Andes ATCSMU driver" + default n + config FDT_HSM_RPMI bool "FDT RPMI HSM driver" depends on FDT_MAILBOX && RPMI_MAILBOX diff --git a/lib/utils/hsm/fdt_hsm_andes_atcsmu.c b/lib/utils/hsm/fdt_hsm_andes_atcsmu.c new file mode 100644 index 00000000000..115916d5b99 --- /dev/null +++ b/lib/utils/hsm/fdt_hsm_andes_atcsmu.c @@ -0,0 +1,211 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 Andes Technology Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned long atcsmu_base; + +void atcsmu_set_wakeup_events(u32 events, u32 hartid) +{ + writel_relaxed(events, (char *)atcsmu_base + PCSm_WE_OFFSET(hartid)); +} + +bool atcsmu_support_sleep_mode(u32 sleep_type, u32 hartid) +{ + u32 pcs_cfg; + u32 mask; + const char *sleep_mode; + + pcs_cfg = readl_relaxed((char *)atcsmu_base + PCSm_CFG_OFFSET(hartid)); + switch (sleep_type) { + case SBI_SUSP_AE350_LIGHT_SLEEP: + mask = PCS_CFG_LIGHT_SLEEP; + sleep_mode = "light sleep"; + break; + case SBI_SUSP_SLEEP_TYPE_SUSPEND: + mask = PCS_CFG_DEEP_SLEEP; + sleep_mode = "deep sleep"; + break; + default: + return false; + } + + if (!EXTRACT_FIELD(pcs_cfg, mask)) { + sbi_printf("ATCSMU: hart%d (PCS%d) does not support %s mode\n", + hartid, hartid + 3, sleep_mode); + return false; + } + + return true; +} + +void atcsmu_set_command(u32 pcs_ctl, u32 hartid) +{ + writel_relaxed(pcs_ctl, (char *)atcsmu_base + PCSm_CTL_OFFSET(hartid)); +} + +int atcsmu_set_reset_vector(u64 wakeup_addr, u32 hartid) +{ + u32 vec_lo; + u32 vec_hi; + u64 reset_vector; + + writel((u32)wakeup_addr, (char *)atcsmu_base + HARTn_RESET_VEC_LO(hartid)); + writel((u32)(wakeup_addr >> 32), (char *)atcsmu_base + HARTn_RESET_VEC_HI(hartid)); + vec_lo = readl((char *)atcsmu_base + HARTn_RESET_VEC_LO(hartid)); + vec_hi = readl((char *)atcsmu_base + HARTn_RESET_VEC_HI(hartid)); + reset_vector = (u64)vec_hi << 32 | vec_lo; + if (reset_vector != wakeup_addr) { + sbi_printf("ATCSMU: hart%d (PCS%d): failed to program the reset vector\n", + hartid, hartid + 3); + return SBI_EFAIL; + } + + return SBI_OK; +} + +u32 atcsmu_get_sleep_type(u32 hartid) +{ + return readl_relaxed((char *)atcsmu_base + PCSm_SCRATCH_OFFSET(hartid)); +} + +void atcsmu_write_scratch(u32 value) +{ + writel_relaxed(value, (char *)atcsmu_base + SCRATCH_PAD_OFFSET); +} + +u32 atcsmu_read_scratch(void) +{ + return readl_relaxed((char *)atcsmu_base + SCRATCH_PAD_OFFSET); +} + +bool atcsmu_pcs_is_sleep(u32 hartid, bool deep_sleep) +{ + u32 pcs_status = readl_relaxed((char *)atcsmu_base + PCSm_STATUS_OFFSET(hartid)); + u32 pd_status = deep_sleep ? PD_STATUS_DEEP_SLEEP : PD_STATUS_LIGHT_SLEEP; + + if (EXTRACT_FIELD(pcs_status, PD_TYPE_MASK) != PD_TYPE_SLEEP) { + sbi_printf("ATCSMU: hart%d (PCS%d): failed to sleep\n", hartid, hartid + 3); + return false; + } + + if (EXTRACT_FIELD(pcs_status, PD_STATUS_MASK) != pd_status) { + sbi_printf("ATCSMU: hart%d (PCS%d): failed to enter %s sleep\n", + hartid, hartid + 3, deep_sleep ? "deep" : "light"); + return false; + } + + return true; +} + +static int ae350_hart_start(u32 hartid, ulong saddr) +{ + u32 hartindex = sbi_hartid_to_hartindex(hartid); + u32 sleep_type = atcsmu_get_sleep_type(hartid); + + /* + * Don't send wakeup command when: + * 1) boot time + * 2) the target hart is non-sleepable 25-series hart0 + * 3) light sleep + */ + if (!sbi_init_count(hartindex) || (is_andes(25) && hartid == 0) || + sleep_type == SBI_SUSP_AE350_LIGHT_SLEEP) + return sbi_ipi_raw_send(hartindex, false); + + atcsmu_set_command(WAKEUP_CMD, hartid); + return 0; +} + +static int ae350_hart_stop(void) +{ + u32 hartid = current_hartid(); + u32 sleep_type = atcsmu_get_sleep_type(hartid); + int rc; + + /* + * For Andes AX25MP, the hart0 shares power domain with the last level + * cache. Instead of turning it off, it should fall through and jump to + * warmboot_addr. + */ + if (is_andes(25) && hartid == 0) + return SBI_ENOTSUPP; + + if (!atcsmu_support_sleep_mode(sleep_type, hartid)) + return SBI_ENOTSUPP; + + /* Prevent the core leaving the WFI mode unexpectedly */ + csr_write(CSR_MIE, 0); + + if (sleep_type == SBI_SUSP_AE350_LIGHT_SLEEP) { + csr_write(CSR_MIE, MIP_MSIP); + atcsmu_set_wakeup_events(PCS_WAKEUP_MSIP_MASK, hartid); + atcsmu_set_command(LIGHT_SLEEP_CMD, hartid); + } else if (sleep_type == SBI_SUSP_SLEEP_TYPE_SUSPEND) { + atcsmu_set_wakeup_events(0x0, hartid); + atcsmu_set_command(DEEP_SLEEP_CMD, hartid); + rc = atcsmu_set_reset_vector((ulong)ae350_enable_coherency_warmboot, hartid); + if (rc) + return SBI_EFAIL; + + ae350_non_ret_save(sbi_scratch_thishart_ptr()); + } + + ae350_disable_coherency(); + wfi(); + + /* Light sleep resumes here */ + ae350_enable_coherency(); + + return SBI_ENOTSUPP; +} + +static const struct sbi_hsm_device hsm_andes_atcsmu = { + .name = "andes_atcsmu", + .hart_start = ae350_hart_start, + .hart_stop = ae350_hart_stop, +}; + +static int hsm_andes_atcsmu_probe(const void *fdt, int nodeoff, const struct fdt_match *match) +{ + int poff, rc; + u64 addr; + + /* Need to find the parent for the address property */ + poff = fdt_parent_offset(fdt, nodeoff); + if (poff < 0) + return SBI_EINVAL; + + rc = fdt_get_node_addr_size(fdt, poff, 0, &addr, NULL); + if (rc < 0 || !addr) + return SBI_ENODEV; + atcsmu_base = addr; + + sbi_hsm_set_device(&hsm_andes_atcsmu); + return 0; +} + +static const struct fdt_match hsm_andes_atcsmu_match[] = { + { .compatible = "andestech,atcsmu-hsm" }, + { }, +}; + +const struct fdt_driver fdt_hsm_andes_atcsmu = { + .match_table = hsm_andes_atcsmu_match, + .init = hsm_andes_atcsmu_probe, +}; diff --git a/lib/utils/hsm/objects.mk b/lib/utils/hsm/objects.mk index 0d0054494f1..f76af03c5c0 100644 --- a/lib/utils/hsm/objects.mk +++ b/lib/utils/hsm/objects.mk @@ -7,6 +7,9 @@ # Anup Patel # +carray-fdt_early_drivers-$(CONFIG_FDT_HSM_ANDES_ATCSMU) += fdt_hsm_andes_atcsmu +libsbiutils-objs-$(CONFIG_FDT_HSM_ANDES_ATCSMU) += hsm/fdt_hsm_andes_atcsmu.o + carray-fdt_early_drivers-$(CONFIG_FDT_HSM_RPMI) += fdt_hsm_rpmi libsbiutils-objs-$(CONFIG_FDT_HSM_RPMI) += hsm/fdt_hsm_rpmi.o @@ -14,4 +17,4 @@ carray-fdt_early_drivers-$(CONFIG_FDT_HSM_SPACEMIT) += fdt_hsm_spacemit libsbiutils-objs-$(CONFIG_FDT_HSM_SPACEMIT) += hsm/fdt_hsm_spacemit.o carray-fdt_early_drivers-$(CONFIG_FDT_HSM_SIFIVE_TMC0) += fdt_hsm_sifive_tmc0 -libsbiutils-objs-$(CONFIG_FDT_HSM_SIFIVE_TMC0) += hsm/fdt_hsm_sifive_tmc0.o \ No newline at end of file +libsbiutils-objs-$(CONFIG_FDT_HSM_SIFIVE_TMC0) += hsm/fdt_hsm_sifive_tmc0.o diff --git a/lib/utils/i2c/Kconfig b/lib/utils/i2c/Kconfig index 7fa32fcf967..bdaaff629aa 100644 --- a/lib/utils/i2c/Kconfig +++ b/lib/utils/i2c/Kconfig @@ -14,6 +14,10 @@ config FDT_I2C_SIFIVE bool "SiFive I2C FDT driver" default n +config FDT_I2C_SPACEMIT + bool "SpacemiT I2C FDT driver" + default n + config FDT_I2C_DW bool "Synopsys Designware I2C FDT driver" select I2C_DW diff --git a/lib/utils/i2c/fdt_i2c_spacemit.c b/lib/utils/i2c/fdt_i2c_spacemit.c new file mode 100644 index 00000000000..9009b6b6246 --- /dev/null +++ b/lib/utils/i2c/fdt_i2c_spacemit.c @@ -0,0 +1,220 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 Aurelien Jarno + * + * Authors: + * Aurelien Jarno + */ + +#include +#include +#include +#include +#include +#include + +/* Controller registers */ +#define ICR_OFFSET 0x00 /* I2C control register */ +#define IDBR_OFFSET 0x0c /* I2C data buffer register */ + +/* Control register bits */ +#define ICR_START BIT(0) /* start */ +#define ICR_STOP BIT(1) /* stop */ +#define ICR_ACKNAK BIT(2) /* ACK(0) or NAK(1) */ +#define ICR_TB BIT(3) /* transfer byte */ +#define ICR_UR BIT(10) /* unit reset */ +#define ICR_SCLE BIT(13) /* SCL enable */ +#define ICR_IUE BIT(14) /* unit enable */ + +/* Timing */ +#define I2C_RESET_US 10 +#define I2C_TIMEOUT_US 1000 + +struct spacemit_i2c_adapter { + unsigned long base; + struct i2c_adapter adapter; +}; + +static inline void spacemit_i2c_set_reg(struct spacemit_i2c_adapter *adap, + uint8_t reg, uint32_t val) +{ + writel(val, (void *)adap->base + reg); +} + +static inline uint32_t spacemit_i2c_get_reg(struct spacemit_i2c_adapter *adap, + uint32_t reg) +{ + return readl((void *)adap->base + reg); +} + +static void spacemit_i2c_reset(struct spacemit_i2c_adapter *adap) +{ + /* disable unit */ + spacemit_i2c_set_reg(adap, ICR_OFFSET, 0); + sbi_timer_udelay(I2C_RESET_US); + + /* reset unit */ + spacemit_i2c_set_reg(adap, ICR_OFFSET, ICR_UR); + sbi_timer_udelay(I2C_RESET_US); + + /* clear reset and enable unit and SCL */ + spacemit_i2c_set_reg(adap, ICR_OFFSET, ICR_IUE | ICR_SCLE); +} + +static int spacemit_i2c_wait_xfer_done(struct spacemit_i2c_adapter *adap) +{ + for (int i = 0; i < I2C_TIMEOUT_US; i++) { + uint32_t val = spacemit_i2c_get_reg(adap, ICR_OFFSET); + + if (!(val & ICR_TB)) + return 0; + + sbi_timer_udelay(1); + }; + + return SBI_ETIMEDOUT; +} + +static void spacemit_i2c_start_xfer(struct spacemit_i2c_adapter *adap, + uint32_t ctrl) +{ + const uint32_t ctrl_mask = ICR_START | ICR_STOP | ICR_ACKNAK; + uint32_t val; + + val = spacemit_i2c_get_reg(adap, ICR_OFFSET); + val &= ~ctrl_mask; + val |= (ctrl & ctrl_mask); + val |= ICR_TB; + + spacemit_i2c_set_reg(adap, ICR_OFFSET, val); +} + +static int spacemit_i2c_xfer_write(struct spacemit_i2c_adapter *adap, + uint8_t byte, uint32_t ctrl) +{ + spacemit_i2c_set_reg(adap, IDBR_OFFSET, byte); + spacemit_i2c_start_xfer(adap, ctrl); + + return spacemit_i2c_wait_xfer_done(adap); +} + +static int spacemit_i2c_xfer_read(struct spacemit_i2c_adapter *adap, + uint8_t *byte, uint32_t ctrl) +{ + int rc; + + spacemit_i2c_start_xfer(adap, ctrl); + + rc = spacemit_i2c_wait_xfer_done(adap); + if (rc) + return rc; + + *byte = spacemit_i2c_get_reg(adap, IDBR_OFFSET); + return 0; +} + +static int spacemit_i2c_adapter_write(struct i2c_adapter *ia, uint8_t addr, + uint8_t reg, uint8_t *buffer, int len) +{ + struct spacemit_i2c_adapter *adap = + container_of(ia, struct spacemit_i2c_adapter, adapter); + int rc; + + /* reset controller to a known state */ + spacemit_i2c_reset(adap); + + /* send device address (in write mode) */ + rc = spacemit_i2c_xfer_write(adap, addr << 1, ICR_START); + if (rc) + return rc; + + /* send register + data bytes */ + for (int i = 0; i < len + 1; i++) { + uint32_t ctrl = (i == len) ? ICR_STOP : 0; + uint8_t byte = (i == 0) ? reg : buffer[i - 1]; + + rc = spacemit_i2c_xfer_write(adap, byte, ctrl); + if (rc) + return rc; + } + + return 0; +} + +static int spacemit_i2c_adapter_read(struct i2c_adapter *ia, uint8_t addr, + uint8_t reg, uint8_t *buffer, int len) +{ + struct spacemit_i2c_adapter *adap = + container_of(ia, struct spacemit_i2c_adapter, adapter); + int rc; + + /* reset controller to a known state */ + spacemit_i2c_reset(adap); + + /* send device address (in write mode) */ + rc = spacemit_i2c_xfer_write(adap, addr << 1, ICR_START); + if (rc) + return rc; + + /* send register */ + rc = spacemit_i2c_xfer_write(adap, reg, 0); + if (rc) + return rc; + + /* repeated start and send device address (in read mode) */ + rc = spacemit_i2c_xfer_write(adap, (addr << 1) | 1, ICR_START); + if (rc) + return rc; + + /* read data bytes */ + for (int i = 0; i < len; i++) { + uint32_t ctrl = (i == len - 1) ? (ICR_ACKNAK | ICR_STOP) : 0; + + rc = spacemit_i2c_xfer_read(adap, &buffer[i], ctrl); + if (rc) + return rc; + } + + return 0; +} + +static int spacemit_i2c_init(const void *fdt, int nodeoff, + const struct fdt_match *match) +{ + struct spacemit_i2c_adapter *adapter; + uint64_t base; + int rc; + + adapter = sbi_zalloc(sizeof(*adapter)); + if (!adapter) + return SBI_ENOMEM; + + rc = fdt_get_node_addr_size(fdt, nodeoff, 0, &base, NULL); + if (rc) { + sbi_free(adapter); + return rc; + } + + adapter->base = base; + adapter->adapter.id = nodeoff; + adapter->adapter.write = spacemit_i2c_adapter_write; + adapter->adapter.read = spacemit_i2c_adapter_read; + rc = i2c_adapter_add(&adapter->adapter); + if (rc) { + sbi_free(adapter); + return rc; + } + + return 0; +} + +static const struct fdt_match spacemit_i2c_match[] = { + { .compatible = "spacemit,k1-i2c" }, + { }, +}; + +const struct fdt_driver fdt_i2c_adapter_spacemit = { + .match_table = spacemit_i2c_match, + .init = spacemit_i2c_init, +}; diff --git a/lib/utils/i2c/objects.mk b/lib/utils/i2c/objects.mk index d34d6648ae0..91ac17ecec2 100644 --- a/lib/utils/i2c/objects.mk +++ b/lib/utils/i2c/objects.mk @@ -15,6 +15,9 @@ libsbiutils-objs-$(CONFIG_FDT_I2C) += i2c/fdt_i2c_adapter_drivers.carray.o carray-fdt_i2c_adapter_drivers-$(CONFIG_FDT_I2C_SIFIVE) += fdt_i2c_adapter_sifive libsbiutils-objs-$(CONFIG_FDT_I2C_SIFIVE) += i2c/fdt_i2c_sifive.o +carray-fdt_i2c_adapter_drivers-$(CONFIG_FDT_I2C_SPACEMIT) += fdt_i2c_adapter_spacemit +libsbiutils-objs-$(CONFIG_FDT_I2C_SPACEMIT) += i2c/fdt_i2c_spacemit.o + carray-fdt_i2c_adapter_drivers-$(CONFIG_FDT_I2C_DW) += fdt_i2c_adapter_dw libsbiutils-objs-$(CONFIG_FDT_I2C_DW) += i2c/fdt_i2c_dw.o diff --git a/lib/utils/irqchip/aplic.c b/lib/utils/irqchip/aplic.c index 8d0db168158..6c63620e122 100644 --- a/lib/utils/irqchip/aplic.c +++ b/lib/utils/irqchip/aplic.c @@ -297,8 +297,21 @@ int aplic_cold_irqchip_init(struct aplic_data *aplic) return rc; } + if (aplic->num_idc) { + for (i = 0; i < aplic->num_idc; i++) + sbi_hartmask_set_hartindex(aplic->idc_map[i], + &aplic->irqchip.target_harts); + } else { + sbi_hartmask_set_all(&aplic->irqchip.target_harts); + } + /* Register irqchip device */ - sbi_irqchip_add_device(&aplic->irqchip); + aplic->irqchip.id = aplic->unique_id; + aplic->irqchip.caps = SBI_IRQCHIP_CAPS_WIRED; + aplic->irqchip.num_hwirq = aplic->num_source + 1; + rc = sbi_irqchip_add_device(&aplic->irqchip); + if (rc) + return rc; /* Attach to the aplic list */ sbi_list_add_tail(&aplic->node, &aplic_list); diff --git a/lib/utils/irqchip/fdt_irqchip_aplic.c b/lib/utils/irqchip/fdt_irqchip_aplic.c index 81ebe67dbb6..4d7b1e77a38 100644 --- a/lib/utils/irqchip/fdt_irqchip_aplic.c +++ b/lib/utils/irqchip/fdt_irqchip_aplic.c @@ -16,6 +16,43 @@ #include #include +static int irqchip_aplic_update_idc_map(const void *fdt, int nodeoff, + struct aplic_data *pd) +{ + int i, err, count, cpu_offset, cpu_intc_offset; + u32 phandle, hartid, hartindex; + const fdt32_t *val; + + val = fdt_getprop(fdt, nodeoff, "interrupts-extended", &count); + if (!val || count < sizeof(fdt32_t)) + return SBI_EINVAL; + count = count / sizeof(fdt32_t); + + for (i = 0; i < count; i += 2) { + phandle = fdt32_to_cpu(val[i]); + + cpu_intc_offset = fdt_node_offset_by_phandle(fdt, phandle); + if (cpu_intc_offset < 0) + continue; + + cpu_offset = fdt_parent_offset(fdt, cpu_intc_offset); + if (cpu_offset < 0) + continue; + + err = fdt_parse_hart_id(fdt, cpu_offset, &hartid); + if (err) + continue; + + hartindex = sbi_hartid_to_hartindex(hartid); + if (hartindex == -1U) + continue; + + pd->idc_map[i / 2] = hartindex; + } + + return 0; +} + static int irqchip_aplic_cold_init(const void *fdt, int nodeoff, const struct fdt_match *match) { @@ -30,12 +67,27 @@ static int irqchip_aplic_cold_init(const void *fdt, int nodeoff, if (rc) goto fail_free_data; + if (pd->num_idc) { + pd->idc_map = sbi_zalloc(sizeof(*pd->idc_map) * pd->num_idc); + if (!pd->idc_map) { + rc = SBI_ENOMEM; + goto fail_free_data; + } + + rc = irqchip_aplic_update_idc_map(fdt, nodeoff, pd); + if (rc) + goto fail_free_idc_map; + } + rc = aplic_cold_irqchip_init(pd); if (rc) - goto fail_free_data; + goto fail_free_idc_map; return 0; +fail_free_idc_map: + if (pd->num_idc) + sbi_free(pd->idc_map); fail_free_data: sbi_free(pd); return rc; diff --git a/lib/utils/irqchip/fdt_irqchip_plic.c b/lib/utils/irqchip/fdt_irqchip_plic.c index 65a9de158d2..61c68ac9f48 100644 --- a/lib/utils/irqchip/fdt_irqchip_plic.c +++ b/lib/utils/irqchip/fdt_irqchip_plic.c @@ -24,6 +24,11 @@ static int irqchip_plic_update_context_map(const void *fdt, int nodeoff, u32 phandle, hwirq, hartid, hartindex; int i, err, count, cpu_offset, cpu_intc_offset; + for (i = 0; i < sbi_hart_count(); i++) { + pd->context_map[i][PLIC_M_CONTEXT] = -1; + pd->context_map[i][PLIC_S_CONTEXT] = -1; + } + val = fdt_getprop(fdt, nodeoff, "interrupts-extended", &count); if (!val || count < sizeof(fdt32_t)) return SBI_EINVAL; diff --git a/lib/utils/irqchip/imsic.c b/lib/utils/irqchip/imsic.c index d72ef7945b6..0e7f3e6bd22 100644 --- a/lib/utils/irqchip/imsic.c +++ b/lib/utils/irqchip/imsic.c @@ -147,7 +147,7 @@ int imsic_get_target_file(u32 hartindex) return imsic_get_hart_file(scratch); } -static int imsic_external_irqfn(void) +static int imsic_process_hwirqs(struct sbi_irqchip_device *chip) { ulong mirq; @@ -346,9 +346,17 @@ int imsic_data_check(struct imsic_data *imsic) return 0; } +static int imsic_hwirq_setup(struct sbi_irqchip_device *chip, u32 hwirq, u32 hwirq_flags) +{ + if (hwirq_flags != SBI_HWIRQ_FLAGS_NONE) + return SBI_ENOTSUPP; + return 0; +} + static struct sbi_irqchip_device imsic_device = { .warm_init = imsic_warm_irqchip_init, - .irq_handle = imsic_external_irqfn, + .process_hwirqs = imsic_process_hwirqs, + .hwirq_setup = imsic_hwirq_setup, }; int imsic_cold_irqchip_init(struct imsic_data *imsic) @@ -391,7 +399,18 @@ int imsic_cold_irqchip_init(struct imsic_data *imsic) } /* Register irqchip device */ - sbi_irqchip_add_device(&imsic_device); + imsic_device.id = imsic->unique_id; + imsic_device.caps = SBI_IRQCHIP_CAPS_MSI; + imsic_device.num_hwirq = imsic->num_ids + 1; + sbi_hartmask_set_all(&imsic_device.target_harts); + rc = sbi_irqchip_add_device(&imsic_device); + if (rc) + return rc; + + /* Mark hwirq 0 and IPI hwirq as reserved */ + rc = sbi_irqchip_register_reserved(&imsic_device, 0, IMSIC_IPI_ID + 1); + if (rc) + return rc; /* Register IPI device */ sbi_ipi_add_device(&imsic_ipi_device); diff --git a/lib/utils/irqchip/plic.c b/lib/utils/irqchip/plic.c index 7989a962d05..2d721724e81 100644 --- a/lib/utils/irqchip/plic.c +++ b/lib/utils/irqchip/plic.c @@ -276,11 +276,12 @@ int plic_cold_irqchip_init(struct plic_data *plic) continue; plic_set_hart_data_ptr(sbi_hartindex_to_scratch(i), plic); + sbi_hartmask_set_hartindex(i, &plic->irqchip.target_harts); } /* Register irqchip device */ + plic->irqchip.id = plic->unique_id; + plic->irqchip.num_hwirq = plic->num_src + 1; plic->irqchip.warm_init = plic_warm_irqchip_init; - sbi_irqchip_add_device(&plic->irqchip); - - return 0; + return sbi_irqchip_add_device(&plic->irqchip); } diff --git a/lib/utils/mpxy/Kconfig b/lib/utils/mpxy/Kconfig index 507b83712d2..a3ed1c22af8 100644 --- a/lib/utils/mpxy/Kconfig +++ b/lib/utils/mpxy/Kconfig @@ -18,6 +18,10 @@ config FDT_MPXY_RPMI_CLOCK bool "MPXY driver for RPMI clock service group" default n +config FDT_MPXY_RPMI_LOGGING + bool "MPXY driver for RPMI logging service group" + default n + config FDT_MPXY_RPMI_SYSMSI bool "MPXY driver for RPMI system MSI service group" default n @@ -34,6 +38,10 @@ config FDT_MPXY_RPMI_PERFORMANCE bool "MPXY driver for RPMI performance service group" default n +config FDT_MPXY_RPMI_MM + bool "MPXY driver for RPMI MM service group" + default n + endif endmenu diff --git a/lib/utils/mpxy/fdt_mpxy_rpmi_logging.c b/lib/utils/mpxy/fdt_mpxy_rpmi_logging.c new file mode 100644 index 00000000000..e8f085ebb8d --- /dev/null +++ b/lib/utils/mpxy/fdt_mpxy_rpmi_logging.c @@ -0,0 +1,51 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 Qualcomm, Inc + * + * Authors: + * Subrahmanya Lingappa + */ + +#include + +static struct mpxy_rpmi_service_data logging_services[] = { + [0] = { + .id = RPMI_MM_SRV_ENABLE_NOTIFICATION, + .min_tx_len = sizeof(struct rpmi_enable_notification_req), + .max_tx_len = sizeof(struct rpmi_enable_notification_req), + .min_rx_len = sizeof(struct rpmi_enable_notification_resp), + .max_rx_len = sizeof(struct rpmi_enable_notification_resp), + }, + [1] = { + .id = RPMI_LOGGING_SRV_LOG_DATA, + .min_tx_len = sizeof(u32) * 2, + .max_tx_len = sizeof(struct rpmi_logging_log_data_req), + .min_rx_len = sizeof(struct rpmi_logging_log_data_resp), + .max_rx_len = sizeof(struct rpmi_logging_log_data_resp), + }, + /* + * Keep a local terminator for safe lookup because only service ID 0x02 + * is intentionally exposed by this MPXY service group handler. + */ + [2] = { + .id = RPMI_LOGGING_SRV_MAX_COUNT, + }, +}; + +static const struct mpxy_rpmi_mbox_data logging_data = { + .servicegrp_id = RPMI_SRVGRP_LOGGING, + .num_services = RPMI_LOGGING_SRV_MAX_COUNT, + .service_data = logging_services, +}; + +static const struct fdt_match logging_match[] = { + { .compatible = "riscv,rpmi-mpxy-logging", .data = &logging_data }, + { }, +}; + +const struct fdt_driver fdt_mpxy_rpmi_logging = { + .match_table = logging_match, + .init = mpxy_rpmi_mbox_init, + .experimental = true, +}; diff --git a/lib/utils/mpxy/fdt_mpxy_rpmi_mm.c b/lib/utils/mpxy/fdt_mpxy_rpmi_mm.c new file mode 100644 index 00000000000..d7176aca06a --- /dev/null +++ b/lib/utils/mpxy/fdt_mpxy_rpmi_mm.c @@ -0,0 +1,116 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 Qualcomm Technologies, Inc. + * + * Authors: + * Ranbir Singh + * Sunil V L + */ + +#include +#include + +static struct rpmi_mm_get_attributes_rsp rsp; + +static struct mpxy_rpmi_service_data mm_srvcdata[] = { + [0] = { + .id = RPMI_MM_SRV_GET_ATTRIBUTES, + .min_tx_len = 0, + .max_tx_len = 0, + .min_rx_len = sizeof(struct rpmi_mm_get_attributes_rsp), + .max_rx_len = sizeof(struct rpmi_mm_get_attributes_rsp), + }, + [1] = { + .id = RPMI_MM_SRV_COMMUNICATE, + .min_tx_len = sizeof(struct rpmi_mm_communicate_req), + .max_tx_len = sizeof(struct rpmi_mm_communicate_req), + .min_rx_len = sizeof(struct rpmi_mm_communicate_rsp), + .max_rx_len = sizeof(struct rpmi_mm_communicate_rsp), + }, +}; + +static int mpxy_rpmi_mm_setup(void **context, struct mbox_chan *chan, + const struct mpxy_rpmi_mbox_data *data) +{ + unsigned long mm_region_addr = 0; + unsigned long mm_region_size = 0; + unsigned long mm_region_flags; + int rc = 0; + + rc = rpmi_normal_request_with_status(chan, RPMI_MM_SRV_GET_ATTRIBUTES, + NULL, 0, 0, &rsp, + rpmi_u32_count(rsp), + rpmi_u32_count(rsp)); + if (rc) + return rc; + +#if __riscv_xlen == 32 + mm_region_addr = rsp.mma.shmem_addr_lo; +#else + mm_region_addr = ((unsigned long)(rsp.mma.shmem_addr_hi) << 32) | + rsp.mma.shmem_addr_lo; +#endif + + mm_region_size = rsp.mma.shmem_size; + mm_region_flags = SBI_DOMAIN_MEMREGION_SHARED_SURW_MRW; + + rc = sbi_domain_root_add_memrange(mm_region_addr, mm_region_size, + PAGE_SIZE, mm_region_flags); + return rc; +} + +static int mpxy_rpmi_mm_xfer(void *context, struct mbox_chan *chan, + struct mbox_xfer *xfer) +{ + struct rpmi_message_args *args = xfer->args; + int rc = 0; + + if (!xfer->rx || (args->type != RPMI_MSG_NORMAL_REQUEST)) + return 0; + + switch (args->service_id) { + case RPMI_MM_SRV_GET_ATTRIBUTES: + ((u32 *)xfer->rx)[0] = cpu_to_le32(RPMI_SUCCESS); + ((u32 *)xfer->rx)[1] = cpu_to_le32(rsp.mma.mm_version); + ((u32 *)xfer->rx)[2] = cpu_to_le32(rsp.mma.shmem_addr_lo); + ((u32 *)xfer->rx)[3] = cpu_to_le32(rsp.mma.shmem_addr_hi); + ((u32 *)xfer->rx)[4] = cpu_to_le32(rsp.mma.shmem_size); + args->rx_data_len = 5 * sizeof(u32); + break; + + case RPMI_MM_SRV_COMMUNICATE: + rc = mbox_chan_xfer(chan, xfer); + break; + + default: + ((u32 *)xfer->rx)[0] = cpu_to_le32(RPMI_ERR_NOTSUPP); + args->rx_data_len = sizeof(u32); + break; + }; + + return rc; +} + +static const struct mpxy_rpmi_mbox_data mm_data = { + .servicegrp_id = RPMI_SRVGRP_MANAGEMENT_MODE, + .num_services = RPMI_MM_SRV_MAX_COUNT, + .service_data = mm_srvcdata, + .setup_group = mpxy_rpmi_mm_setup, + .xfer_group = mpxy_rpmi_mm_xfer, +}; + +/* one extra blank entry for loop termination while matching */ +static const struct fdt_match mm_match[] = { + { + .compatible = "riscv,rpmi-mpxy-mm", + .data = &mm_data, + }, + {}, +}; + +const struct fdt_driver fdt_mpxy_rpmi_mm = { + .experimental = true, + .match_table = mm_match, + .init = mpxy_rpmi_mbox_init, +}; diff --git a/lib/utils/mpxy/objects.mk b/lib/utils/mpxy/objects.mk index bbc998afdf1..1f1bf0a0fa1 100644 --- a/lib/utils/mpxy/objects.mk +++ b/lib/utils/mpxy/objects.mk @@ -26,3 +26,9 @@ libsbiutils-objs-$(CONFIG_FDT_MPXY_RPMI_VOLTAGE) += mpxy/fdt_mpxy_rpmi_voltage.o carray-fdt_mpxy_drivers-$(CONFIG_FDT_MPXY_RPMI_DEVICE_POWER) += fdt_mpxy_rpmi_device_power libsbiutils-objs-$(CONFIG_FDT_MPXY_RPMI_DEVICE_POWER) += mpxy/fdt_mpxy_rpmi_device_power.o + +carray-fdt_mpxy_drivers-$(CONFIG_FDT_MPXY_RPMI_MM) += fdt_mpxy_rpmi_mm +libsbiutils-objs-$(CONFIG_FDT_MPXY_RPMI_MM) += mpxy/fdt_mpxy_rpmi_mm.o + +carray-fdt_mpxy_drivers-$(CONFIG_FDT_MPXY_RPMI_LOGGING) += fdt_mpxy_rpmi_logging +libsbiutils-objs-$(CONFIG_FDT_MPXY_RPMI_LOGGING) += mpxy/fdt_mpxy_rpmi_logging.o diff --git a/lib/utils/regmap/fdt_regmap_syscon.c b/lib/utils/regmap/fdt_regmap_syscon.c index 1309a55cd9c..59cd5ee508d 100644 --- a/lib/utils/regmap/fdt_regmap_syscon.c +++ b/lib/utils/regmap/fdt_regmap_syscon.c @@ -190,7 +190,7 @@ static int regmap_syscon_init(const void *fdt, int nodeoff, srm->rmap.id = nodeoff; srm->rmap.reg_shift = 0; - srm->rmap.reg_stride = srm->reg_io_width * 8; + srm->rmap.reg_stride = srm->reg_io_width; srm->rmap.reg_base = 0; srm->rmap.reg_max = size / srm->reg_io_width; switch (srm->reg_io_width) { diff --git a/lib/utils/reset/Kconfig b/lib/utils/reset/Kconfig index 68e667163b5..b2ac120e893 100644 --- a/lib/utils/reset/Kconfig +++ b/lib/utils/reset/Kconfig @@ -11,7 +11,7 @@ if FDT_RESET config FDT_RESET_ATCWDT200 bool "Andes WDT FDT reset driver" - depends on SYS_ATCSMU + depends on FDT_HSM_ANDES_ATCSMU default n config FDT_RESET_GPIO @@ -24,6 +24,11 @@ config FDT_RESET_HTIF select SYS_HTIF default n +config FDT_RESET_LITEX + bool "LITEX SoC reset driver" + depends on FDT_GPIO + default n + config FDT_RESET_RPMI bool "RPMI FDT reset driver" depends on FDT_MAILBOX && RPMI_MAILBOX @@ -33,6 +38,10 @@ config FDT_RESET_SG2042_HWMON_MCU bool "Sophgo SG2042 hwmon MCU FDT reset driver" default n +config FDT_RESET_SPACEMIT_P1 + bool "SpacemiT P1 reset driver" + default n + config FDT_RESET_SUNXI_WDT bool "Sunxi WDT FDT reset driver" default n diff --git a/lib/utils/reset/fdt_reset_atcwdt200.c b/lib/utils/reset/fdt_reset_atcwdt200.c index 2304582a951..5dad8ac96d4 100644 --- a/lib/utils/reset/fdt_reset_atcwdt200.c +++ b/lib/utils/reset/fdt_reset_atcwdt200.c @@ -1,10 +1,7 @@ /* * SPDX-License-Identifier: BSD-2-Clause * - * Copyright (c) 2022 Andes Technology Corporation - * - * Authors: - * Yu Chien Peter Lin + * Copyright (c) 2025 Andes Technology Corporation */ #include @@ -15,7 +12,7 @@ #include #include #include -#include +#include #define ATCWDT200_WP_NUM 0x5aa5 #define WREN_REG 0x18 @@ -41,8 +38,9 @@ #define CLK_PCLK (1 << 1) #define WDT_EN (1 << 0) +#define AE350_FLASH_BASE 0x80000000 + static volatile char *wdt_addr = NULL; -static struct smu_data smu = { 0 }; static int ae350_system_reset_check(u32 type, u32 reason) { @@ -59,7 +57,7 @@ static int ae350_system_reset_check(u32 type, u32 reason) static void ae350_system_reset(u32 type, u32 reason) { sbi_for_each_hartindex(i) - if (smu_set_reset_vector(&smu, FLASH_BASE, i)) + if (atcsmu_set_reset_vector(AE350_FLASH_BASE, i)) goto fail; /* Program WDT control register */ @@ -88,16 +86,6 @@ static int atcwdt200_reset_init(const void *fdt, int nodeoff, return SBI_ENODEV; wdt_addr = (volatile char *)(unsigned long)reg_addr; - - /* - * The reset device requires smu to program the reset - * vector for each hart. - */ - if (fdt_parse_compat_addr(fdt, ®_addr, "andestech,atcsmu")) - return SBI_ENODEV; - - smu.addr = (unsigned long)reg_addr; - sbi_system_reset_add_device(&atcwdt200_reset); return 0; diff --git a/lib/utils/reset/fdt_reset_litex.c b/lib/utils/reset/fdt_reset_litex.c new file mode 100644 index 00000000000..287745f094f --- /dev/null +++ b/lib/utils/reset/fdt_reset_litex.c @@ -0,0 +1,66 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 Inochi Amaoto + */ + +#include +#include +#include +#include +#include +#include + +#define RESET_CTRL 0x0 + +static volatile u32 *litex_soc_base; + +static int litex_reset_check(u32 type, u32 reason) +{ + switch (type) { + case SBI_SRST_RESET_TYPE_COLD_REBOOT: + case SBI_SRST_RESET_TYPE_WARM_REBOOT: + return 255; + } + + return 0; +} + +static void litex_do_reset(u32 type, u32 reason) +{ + writel_relaxed(0x1, litex_soc_base + RESET_CTRL); +} + +static struct sbi_system_reset_device litex_reset = { + .name = "litex-reset", + .system_reset_check = litex_reset_check, + .system_reset = litex_do_reset +}; + +static int litex_reset_init(const void *fdt, int nodeoff, + const struct fdt_match *match) +{ + uint64_t reg_addr; + int rc; + + rc = fdt_get_node_addr_size(fdt, nodeoff, 0, ®_addr, NULL); + if (rc < 0 || !reg_addr) + return SBI_ENODEV; + + + litex_soc_base = (volatile u32 *)(unsigned long)reg_addr; + + sbi_system_reset_add_device(&litex_reset); + + return 0; +} + +static const struct fdt_match litex_reset_match[] = { + { .compatible = "litex,soc-controller" }, + { }, +}; + +const struct fdt_driver fdt_reset_litex = { + .match_table = litex_reset_match, + .init = litex_reset_init, +}; diff --git a/lib/utils/reset/fdt_reset_sg2042_hwmon_mcu.c b/lib/utils/reset/fdt_reset_sg2042_hwmon_mcu.c index 94ac416252b..bb18026847e 100644 --- a/lib/utils/reset/fdt_reset_sg2042_hwmon_mcu.c +++ b/lib/utils/reset/fdt_reset_sg2042_hwmon_mcu.c @@ -104,7 +104,7 @@ static int sg2042_mcu_reset_init(const void *fdt, int nodeoff, } static const struct fdt_match sg2042_mcu_reset_match[] = { - { .compatible = "sophgo,sg2042-hwmon-mcu", .data = (void *)true}, + { .compatible = "sophgo,sg2042-hwmon-mcu" }, { }, }; diff --git a/lib/utils/reset/fdt_reset_spacemit_p1.c b/lib/utils/reset/fdt_reset_spacemit_p1.c new file mode 100644 index 00000000000..d153f68b1db --- /dev/null +++ b/lib/utils/reset/fdt_reset_spacemit_p1.c @@ -0,0 +1,112 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 Aurelien Jarno + * Authors: + * Aurelien Jarno + */ + +#include +#include +#include +#include +#include +#include +#include + +/* SpacemiT P1 Power Control Register 2 */ +#define PWR_CTRL2 0x7e +#define PWR_CTRL2_SHUTDOWN BIT(2) /* Shutdown request */ +#define PWR_CTRL2_RST BIT(1) /* Reset request */ + +static struct i2c_adapter *p1_adapter = NULL; +static uint32_t p1_reg = 0; + +static int p1_system_reset_check(uint32_t type, uint32_t reason) +{ + switch (type) { + case SBI_SRST_RESET_TYPE_SHUTDOWN: + return 1; + case SBI_SRST_RESET_TYPE_COLD_REBOOT: + case SBI_SRST_RESET_TYPE_WARM_REBOOT: + return 255; + } + + return 0; +} + +static void p1_ops(uint32_t type) +{ + uint8_t byte; + int rc; + + rc = i2c_adapter_reg_read(p1_adapter, p1_reg, PWR_CTRL2, &byte); + if (rc) { + sbi_printf("%s: cannot read P1 Power Control Register 2\n", __func__); + return; + } + + if (type == SBI_SRST_RESET_TYPE_SHUTDOWN) + byte |= PWR_CTRL2_SHUTDOWN; + else + byte |= PWR_CTRL2_RST; + + rc = i2c_adapter_reg_write(p1_adapter, p1_reg, PWR_CTRL2, byte); + if (rc) + sbi_printf("%s: cannot write P1 Power Control Register 2\n", __func__); +} + +static void p1_system_reset(uint32_t type, uint32_t reason) +{ + switch (type) { + case SBI_SRST_RESET_TYPE_SHUTDOWN: + case SBI_SRST_RESET_TYPE_COLD_REBOOT: + case SBI_SRST_RESET_TYPE_WARM_REBOOT: + p1_ops(type); + break; + } +} + +static struct sbi_system_reset_device p1_reset = { + .name = "spacemit-p1-reset", + .system_reset_check = p1_system_reset_check, + .system_reset = p1_system_reset +}; + +static int p1_reset_init(const void *fdt, int nodeoff, + const struct fdt_match *match) +{ + int rc, i2c_bus; + uint64_t addr; + + /* we are spacemit,p1 node */ + rc = fdt_get_node_addr_size(fdt, nodeoff, 0, &addr, NULL); + if (rc) + return rc; + + p1_reg = addr; + + /* find i2c bus parent node */ + i2c_bus = fdt_parent_offset(fdt, nodeoff); + if (i2c_bus < 0) + return i2c_bus; + + /* i2c adapter get */ + rc = fdt_i2c_adapter_get(fdt, i2c_bus, &p1_adapter); + if (rc) + return rc; + + sbi_system_reset_add_device(&p1_reset); + + return 0; +} + +static const struct fdt_match p1_reset_match[] = { + { .compatible = "spacemit,p1" }, + { }, +}; + +const struct fdt_driver fdt_reset_spacemit_p1 = { + .match_table = p1_reset_match, + .init = p1_reset_init, +}; diff --git a/lib/utils/reset/objects.mk b/lib/utils/reset/objects.mk index ac38b49df05..38b4e3065c4 100644 --- a/lib/utils/reset/objects.mk +++ b/lib/utils/reset/objects.mk @@ -17,9 +17,15 @@ libsbiutils-objs-$(CONFIG_FDT_RESET_GPIO) += reset/fdt_reset_gpio.o carray-fdt_early_drivers-$(CONFIG_FDT_RESET_HTIF) += fdt_reset_htif libsbiutils-objs-$(CONFIG_FDT_RESET_HTIF) += reset/fdt_reset_htif.o +carray-fdt_early_drivers-$(CONFIG_FDT_RESET_LITEX) += fdt_reset_litex +libsbiutils-objs-$(CONFIG_FDT_RESET_LITEX) += reset/fdt_reset_litex.o + carray-fdt_early_drivers-$(CONFIG_FDT_RESET_SG2042_HWMON_MCU) += fdt_reset_sg2042_mcu libsbiutils-objs-$(CONFIG_FDT_RESET_SG2042_HWMON_MCU) += reset/fdt_reset_sg2042_hwmon_mcu.o +carray-fdt_early_drivers-$(CONFIG_FDT_RESET_SPACEMIT_P1) += fdt_reset_spacemit_p1 +libsbiutils-objs-$(CONFIG_FDT_RESET_SPACEMIT_P1) += reset/fdt_reset_spacemit_p1.o + carray-fdt_early_drivers-$(CONFIG_FDT_RESET_SUNXI_WDT) += fdt_reset_sunxi_wdt libsbiutils-objs-$(CONFIG_FDT_RESET_SUNXI_WDT) += reset/fdt_reset_sunxi_wdt.o diff --git a/lib/utils/serial/Kconfig b/lib/utils/serial/Kconfig index e3589cac103..21619d0da1b 100644 --- a/lib/utils/serial/Kconfig +++ b/lib/utils/serial/Kconfig @@ -9,6 +9,11 @@ config FDT_SERIAL if FDT_SERIAL +config FDT_SERIAL_ALTR_JUART + bool "Altera JTAG UART FDT driver" + select SERIAL_ALTR_JUART + default n + config FDT_SERIAL_CADENCE bool "Cadence UART FDT driver" select SERIAL_CADENCE @@ -56,6 +61,10 @@ config FDT_SERIAL_XILINX_UARTLITE endif +config SERIAL_ALTR_JUART + bool "Altera JTAG UART support" + default n + config SERIAL_CADENCE bool "Cadence UART support" default n diff --git a/lib/utils/serial/altr-juart.c b/lib/utils/serial/altr-juart.c new file mode 100644 index 00000000000..a230a561f2a --- /dev/null +++ b/lib/utils/serial/altr-juart.c @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 ISCAS + * + * Authors: + * Icenowy Zheng + */ + +#include +#include +#include + +/* clang-format off */ + +#define JUART_DATA_OFFSET 0x00 +# define JUART_DATA_DATA_MASK 0x000000ff +# define JUART_DATA_RVALID (1 << 15) +# define JUART_DATA_RAVAIL_MASK 0xffff0000 +# define JUART_DATA_RAVAIL_SHIFT 16 +#define JUART_CTRL_OFFSET 0x04 +# define JUART_CTRL_RE (1 << 0) +# define JUART_CTRL_WE (1 << 1) +# define JUART_CTRL_RI (1 << 8) +# define JUART_CTRL_WI (1 << 9) +# define JUART_CTRL_AC (1 << 10) +# define JUART_CTRL_WSPACE_MASK 0xffff0000 +# define JUART_CTRL_WSPACE_SHIFT 16 + +/* clang-format on */ + +static volatile char *altr_juart_base; + +static u32 get_reg(u32 offset) +{ + return readl(altr_juart_base + offset); +} + +static void set_reg(u32 offset, u32 val) +{ + writel(val, altr_juart_base + offset); +} + +static void altr_juart_putc(char ch) +{ + while(!(get_reg(JUART_CTRL_OFFSET) & JUART_CTRL_WSPACE_MASK)) + ; + + set_reg(JUART_DATA_OFFSET, (unsigned char)ch); +} + +static int altr_juart_getc(void) +{ + u32 reg = get_reg(JUART_DATA_OFFSET); + if (reg & JUART_DATA_RVALID) + return reg & JUART_DATA_DATA_MASK; + + return -1; +} + +static struct sbi_console_device altr_juart_console = { + .name = "altr-juart", + .console_putc = altr_juart_putc, + .console_getc = altr_juart_getc +}; + +int altr_juart_init(unsigned long base) +{ + altr_juart_base = (volatile char *)base; + + sbi_console_set_device(&altr_juart_console); + + return 0; +} diff --git a/lib/utils/serial/fdt_serial_altr_juart.c b/lib/utils/serial/fdt_serial_altr_juart.c new file mode 100644 index 00000000000..7901aa57441 --- /dev/null +++ b/lib/utils/serial/fdt_serial_altr_juart.c @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 ISCAS + * + * Authors: + * Icenowy Zheng + */ + +#include +#include +#include +#include + +static int serial_altr_juart_init(const void *fdt, int nodeoff, + const struct fdt_match *match) +{ + uint64_t reg_addr, reg_size; + int rc; + + if (nodeoff < 0 || !fdt) + return SBI_ENODEV; + + rc = fdt_get_node_addr_size(fdt, nodeoff, 0, ®_addr, ®_size); + /* Two 32-bit registers */ + if (rc < 0 || !reg_addr || !reg_size || reg_size < 0x8) + return SBI_ENODEV; + + return altr_juart_init(reg_addr); +} + +static const struct fdt_match serial_altr_juart_match[] = { + { .compatible = "altr,juart-1.0" }, + { }, +}; + +const struct fdt_driver fdt_serial_altr_juart = { + .match_table = serial_altr_juart_match, + .init = serial_altr_juart_init, +}; diff --git a/lib/utils/serial/objects.mk b/lib/utils/serial/objects.mk index 7c973c8385b..94cf3a4b880 100644 --- a/lib/utils/serial/objects.mk +++ b/lib/utils/serial/objects.mk @@ -10,6 +10,9 @@ libsbiutils-objs-$(CONFIG_FDT_SERIAL) += serial/fdt_serial.o libsbiutils-objs-$(CONFIG_FDT_SERIAL) += serial/fdt_serial_drivers.carray.o +carray-fdt_serial_drivers-$(CONFIG_FDT_SERIAL_ALTR_JUART) += fdt_serial_altr_juart +libsbiutils-objs-$(CONFIG_FDT_SERIAL_ALTR_JUART) += serial/fdt_serial_altr_juart.o + carray-fdt_serial_drivers-$(CONFIG_FDT_SERIAL_CADENCE) += fdt_serial_cadence libsbiutils-objs-$(CONFIG_FDT_SERIAL_CADENCE) += serial/fdt_serial_cadence.o @@ -37,6 +40,7 @@ libsbiutils-objs-$(CONFIG_FDT_SERIAL_UART8250) += serial/fdt_serial_uart8250.o carray-fdt_serial_drivers-$(CONFIG_FDT_SERIAL_XILINX_UARTLITE) += fdt_serial_xlnx_uartlite libsbiutils-objs-$(CONFIG_FDT_SERIAL_XILINX_UARTLITE) += serial/fdt_serial_xlnx_uartlite.o +libsbiutils-objs-$(CONFIG_SERIAL_ALTR_JUART) += serial/altr-juart.o libsbiutils-objs-$(CONFIG_SERIAL_CADENCE) += serial/cadence-uart.o libsbiutils-objs-$(CONFIG_SERIAL_GAISLER) += serial/gaisler-uart.o libsbiutils-objs-$(CONFIG_SERIAL_RENESAS_SCIF) += serial/renesas_scif.o diff --git a/lib/utils/suspend/Kconfig b/lib/utils/suspend/Kconfig index 6c09d95253d..8d05aff67ac 100644 --- a/lib/utils/suspend/Kconfig +++ b/lib/utils/suspend/Kconfig @@ -9,6 +9,11 @@ config FDT_SUSPEND if FDT_SUSPEND +config FDT_SUSPEND_ANDES_ATCSMU + bool "FDT Andes ATCSMU suspend driver" + depends on FDT_HSM_ANDES_ATCSMU && FDT_CACHE_ANDES_LLCACHE + default n + config FDT_SUSPEND_RPMI bool "FDT RPMI suspend driver" depends on FDT_MAILBOX && RPMI_MAILBOX diff --git a/lib/utils/suspend/fdt_suspend_andes_atcsmu.c b/lib/utils/suspend/fdt_suspend_andes_atcsmu.c new file mode 100644 index 00000000000..072accc031a --- /dev/null +++ b/lib/utils/suspend/fdt_suspend_andes_atcsmu.c @@ -0,0 +1,119 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 Andes Technology Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int check_secondary_harts_sleep(u32 hartid, bool deep_sleep) +{ + const struct sbi_domain *dom = &root; + unsigned long i; + u32 target; + + /* Ensure the secondary harts entering the corresponding sleep state */ + sbi_hartmask_for_each_hartindex(i, dom->possible_harts) { + target = sbi_hartindex_to_hartid(i); + if (target != hartid && !atcsmu_pcs_is_sleep(target, deep_sleep)) + return SBI_EFAIL; + } + + return SBI_OK; +} + +static int ae350_system_suspend_check(u32 sleep_type) +{ + return (sleep_type == SBI_SUSP_SLEEP_TYPE_SUSPEND || + sleep_type == SBI_SUSP_AE350_LIGHT_SLEEP) ? SBI_OK : SBI_EINVAL; +} + +static int ae350_system_suspend(u32 sleep_type, unsigned long addr) +{ + u32 hartid = current_hartid(); + int rc; + + /* Prevent the core leaving the WFI mode unexpectedly */ + csr_write(CSR_MIE, 0); + + /* + * Only allow the S-mode external interrupts (UART2 and RTC alarm) to + * wake up the primary hart + */ + csr_set(CSR_SIE, MIP_SEIP); + atcsmu_set_wakeup_events(PCS_WAKEUP_RTC_ALARM_MASK | PCS_WAKEUP_UART2_MASK, hartid); + + if (sleep_type == SBI_SUSP_AE350_LIGHT_SLEEP) { + rc = check_secondary_harts_sleep(hartid, false); + if (rc) + return rc; + + atcsmu_set_command(LIGHT_SLEEP_CMD, hartid); + } else if (sleep_type == SBI_SUSP_SLEEP_TYPE_SUSPEND) { + rc = check_secondary_harts_sleep(hartid, true); + if (rc) + return rc; + + atcsmu_set_command(DEEP_SLEEP_CMD, hartid); + rc = atcsmu_set_reset_vector((ulong)ae350_enable_coherency_warmboot, hartid); + if (rc) + return rc; + + ae350_non_ret_save(sbi_scratch_thishart_ptr()); + fdt_cmo_llc_enable(false); + rc = fdt_cmo_llc_flush_all(); + if (rc) + return rc; + } + + ae350_disable_coherency(); + wfi(); + + /* Light sleep resumes here */ + ae350_enable_coherency(); + + return SBI_OK; +} + +static void ae350_system_resume(void) +{ + u32 hartid = current_hartid(); + u32 sleep_type = atcsmu_get_sleep_type(hartid); + + if (sleep_type == SBI_SUSP_SLEEP_TYPE_SUSPEND) + fdt_cmo_llc_enable(true); +} + +static struct sbi_system_suspend_device suspend_andes_atcsmu = { + .name = "andes_atcsmu", + .system_suspend_check = ae350_system_suspend_check, + .system_suspend = ae350_system_suspend, + .system_resume = ae350_system_resume, +}; + +static int suspend_andes_atcsmu_probe(const void *fdt, int nodeoff, const struct fdt_match *match) +{ + sbi_system_suspend_set_device(&suspend_andes_atcsmu); + return 0; +} + +static const struct fdt_match suspend_andes_atcsmu_match[] = { + { .compatible = "andestech,atcsmu-sys" }, + { }, +}; + +const struct fdt_driver fdt_suspend_andes_atcsmu = { + .match_table = suspend_andes_atcsmu_match, + .init = suspend_andes_atcsmu_probe, +}; diff --git a/lib/utils/suspend/objects.mk b/lib/utils/suspend/objects.mk index 1fb29b5e794..fb1ad9ae34c 100644 --- a/lib/utils/suspend/objects.mk +++ b/lib/utils/suspend/objects.mk @@ -7,6 +7,9 @@ # Anup Patel # +carray-fdt_early_drivers-$(CONFIG_FDT_SUSPEND_ANDES_ATCSMU) += fdt_suspend_andes_atcsmu +libsbiutils-objs-$(CONFIG_FDT_SUSPEND_ANDES_ATCSMU) += suspend/fdt_suspend_andes_atcsmu.o + carray-fdt_early_drivers-$(CONFIG_FDT_SUSPEND_RPMI) += fdt_suspend_rpmi libsbiutils-objs-$(CONFIG_FDT_SUSPEND_RPMI) += suspend/fdt_suspend_rpmi.o diff --git a/lib/utils/sys/Kconfig b/lib/utils/sys/Kconfig index a22191cd7c2..fc388665524 100644 --- a/lib/utils/sys/Kconfig +++ b/lib/utils/sys/Kconfig @@ -2,10 +2,6 @@ menu "System Device Support" -config SYS_ATCSMU - bool "Andes System Management Unit (SMU) support" - default n - config SYS_HTIF bool "Host transfere interface (HTIF) support" default n diff --git a/lib/utils/sys/atcsmu.c b/lib/utils/sys/atcsmu.c deleted file mode 100644 index 2cba0eb73e7..00000000000 --- a/lib/utils/sys/atcsmu.c +++ /dev/null @@ -1,89 +0,0 @@ -/* - * SPDX-License-Identifier: BSD-3-Clause - * - * Copyright (c) 2023 Andes Technology Corporation - * - * Authors: - * Yu Chien Peter Lin - */ - -#include -#include -#include -#include -#include - -inline int smu_set_wakeup_events(struct smu_data *smu, u32 events, u32 hartid) -{ - if (smu) { - writel(events, (void *)(smu->addr + PCSm_WE_OFFSET(hartid))); - return 0; - } else - return SBI_EINVAL; -} - -inline bool smu_support_sleep_mode(struct smu_data *smu, u32 sleep_mode, - u32 hartid) -{ - u32 pcs_cfg; - - if (!smu) { - sbi_printf("%s(): Failed to access smu_data\n", __func__); - return false; - } - - pcs_cfg = readl((void *)(smu->addr + PCSm_CFG_OFFSET(hartid))); - - switch (sleep_mode) { - case LIGHTSLEEP_MODE: - if (EXTRACT_FIELD(pcs_cfg, PCS_CFG_LIGHT_SLEEP) == 0) { - sbi_printf("SMU: hart%d (PCS%d) does not support light sleep mode\n", - hartid, hartid + 3); - return false; - } - break; - case DEEPSLEEP_MODE: - if (EXTRACT_FIELD(pcs_cfg, PCS_CFG_DEEP_SLEEP) == 0) { - sbi_printf("SMU: hart%d (PCS%d) does not support deep sleep mode\n", - hartid, hartid + 3); - return false; - } - break; - } - - return true; -} - -inline int smu_set_command(struct smu_data *smu, u32 pcs_ctl, u32 hartid) -{ - if (smu) { - writel(pcs_ctl, (void *)(smu->addr + PCSm_CTL_OFFSET(hartid))); - return 0; - } else - return SBI_EINVAL; -} - -inline int smu_set_reset_vector(struct smu_data *smu, ulong wakeup_addr, - u32 hartid) -{ - u32 vec_lo, vec_hi; - u64 reset_vector; - - if (!smu) - return SBI_EINVAL; - - writel(wakeup_addr, (void *)(smu->addr + HARTn_RESET_VEC_LO(hartid))); - writel((u64)wakeup_addr >> 32, - (void *)(smu->addr + HARTn_RESET_VEC_HI(hartid))); - - vec_lo = readl((void *)(smu->addr + HARTn_RESET_VEC_LO(hartid))); - vec_hi = readl((void *)(smu->addr + HARTn_RESET_VEC_HI(hartid))); - reset_vector = ((u64)vec_hi << 32) | vec_lo; - - if (reset_vector != (u64)wakeup_addr) { - sbi_printf("hart%d (PCS%d): Failed to program the reset vector.\n", - hartid, hartid + 3); - return SBI_EFAIL; - } else - return 0; -} diff --git a/lib/utils/sys/objects.mk b/lib/utils/sys/objects.mk index 409d7e8c7ef..d9c67077a3a 100644 --- a/lib/utils/sys/objects.mk +++ b/lib/utils/sys/objects.mk @@ -8,4 +8,3 @@ # libsbiutils-objs-$(CONFIG_SYS_HTIF) += sys/htif.o -libsbiutils-objs-$(CONFIG_SYS_ATCSMU) += sys/atcsmu.o diff --git a/platform/generic/Kconfig b/platform/generic/Kconfig index 716fab42a7b..d594b140a31 100644 --- a/platform/generic/Kconfig +++ b/platform/generic/Kconfig @@ -36,7 +36,6 @@ config PLATFORM_ALLWINNER_D1 config PLATFORM_ANDES_AE350 bool "Andes AE350 support" - select SYS_ATCSMU select ANDES_PMU select ANDES_PMA default n @@ -67,6 +66,10 @@ config PLATFORM_RENESAS_RZFIVE select ANDES_PMU default n +config PLATFORM_SIFIVE_DEV + bool "SiFive development platform support" + default n + config PLATFORM_SIFIVE_FU540 bool "SiFive FU540 support" default n @@ -86,23 +89,37 @@ config PLATFORM_STARFIVE_JH7110 bool "StarFive JH7110 support" default n +config PLATFORM_TENSTORRENT_ATLANTIS + bool "Tenstorrent Atlantis support" + select CPU_TENSTORRENT_ASCALON + default n + config PLATFORM_THEAD bool "THEAD C9xx support" select THEAD_C9XX_ERRATA select THEAD_C9XX_PMU default n -config PLATFORM_MIPS_P8700 - bool "MIPS P8700 support" - default n - config PLATFORM_SPACEMIT_K1 bool "Spacemit K1 support" select FDT_HSM_SPACEMIT default n +config CPU_MIPS_P8700 + bool + default n + +config PLATFORM_MIPS_P8700_EYEQ7H + select CPU_MIPS_P8700 + bool "EyeQ7H" + +config PLATFORM_MIPS_P8700_BOSTON + select CPU_MIPS_P8700 + bool "Boston" + source "$(OPENSBI_SRC_DIR)/platform/generic/andes/Kconfig" source "$(OPENSBI_SRC_DIR)/platform/generic/eswin/Kconfig" +source "$(OPENSBI_SRC_DIR)/platform/generic/tenstorrent/Kconfig" source "$(OPENSBI_SRC_DIR)/platform/generic/thead/Kconfig" endif diff --git a/platform/generic/allwinner/sun20i-d1.c b/platform/generic/allwinner/sun20i-d1.c index 666473c1ac5..4ea9015818d 100644 --- a/platform/generic/allwinner/sun20i-d1.c +++ b/platform/generic/allwinner/sun20i-d1.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -197,11 +198,12 @@ static int sun20i_d1_final_init(bool cold_boot) return generic_final_init(cold_boot); } -static int sun20i_d1_extensions_init(struct sbi_hart_features *hfeatures) +static int sun20i_d1_extensions_init(bool cold_boot) { + struct sbi_hart_features *hfeatures; int rc; - rc = generic_extensions_init(hfeatures); + rc = generic_extensions_init(cold_boot); if (rc) return rc; @@ -209,6 +211,7 @@ static int sun20i_d1_extensions_init(struct sbi_hart_features *hfeatures) /* auto-detection doesn't work on t-head c9xx cores */ /* D1 has 29 mhpmevent csrs, but only 3-9,13-17 have valid value */ + hfeatures = sbi_hart_features_ptr(sbi_scratch_thishart_ptr()); hfeatures->mhpm_mask = 0x0003e3f8; hfeatures->mhpm_bits = 64; diff --git a/platform/generic/andes/ae350.c b/platform/generic/andes/ae350.c index 0808065745a..4cf7e26def5 100644 --- a/platform/generic/andes/ae350.c +++ b/platform/generic/andes/ae350.c @@ -1,121 +1,86 @@ /* * SPDX-License-Identifier: BSD-2-Clause * - * Copyright (c) 2022 Andes Technology Corporation - * - * Authors: - * Yu Chien Peter Lin + * Copyright (c) 2025 Andes Technology Corporation */ -#include +#include #include -#include -#include -#include +#include +#include +#include #include -#include -#include -#include -#include #include -#include -#include +#include +#include +#include -static struct smu_data smu = { 0 }; -extern void __ae350_enable_coherency_warmboot(void); -extern void __ae350_disable_coherency(void); +static unsigned long andes_hart_data_offset; +extern void _start_warm(void); -static int ae350_hart_start(u32 hartid, ulong saddr) +void ae350_non_ret_save(struct sbi_scratch *scratch) { - u32 hartindex = sbi_hartid_to_hartindex(hartid); - - /* - * Don't send wakeup command when: - * 1) boot-time - * 2) the target hart is non-sleepable 25-series hart0 - */ - if (!sbi_init_count(hartindex) || (is_andes(25) && hartid == 0)) - return sbi_ipi_raw_send(hartindex, false); - - /* Write wakeup command to the sleep hart */ - smu_set_command(&smu, WAKEUP_CMD, hartid); - - return 0; + struct andes_hart_data *andes_hdata = sbi_scratch_offset_ptr(scratch, + andes_hart_data_offset); + + andes_hdata->mcache_ctl = csr_read(CSR_MCACHE_CTL); + andes_hdata->mmisc_ctl = csr_read(CSR_MMISC_CTL); + andes_hdata->mpft_ctl = csr_read(CSR_MPFT_CTL); + andes_hdata->mslideleg = csr_read(CSR_MSLIDELEG); + andes_hdata->mxstatus = csr_read(CSR_MXSTATUS); + andes_hdata->slie = csr_read(CSR_SLIE); + andes_hdata->slip = csr_read(CSR_SLIP); + andes_hdata->pmacfg0 = csr_read(CSR_PMACFG0); + andes_hdata->pmacfg2 = csr_read_num(CSR_PMACFG0 + 2); + for (int i = 0; i < 16; i++) + andes_hdata->pmaaddrX[i] = csr_read_num(CSR_PMAADDR0 + i); } -static int ae350_hart_stop(void) +void ae350_non_ret_restore(struct sbi_scratch *scratch) { - int rc; - u32 hartid = current_hartid(); - - /** - * For Andes AX25MP, the hart0 shares power domain with - * L2-cache, instead of turning it off, it should fall - * through and jump to warmboot_addr. - */ - if (is_andes(25) && hartid == 0) - return SBI_ENOTSUPP; - - if (!smu_support_sleep_mode(&smu, DEEPSLEEP_MODE, hartid)) - return SBI_ENOTSUPP; - - /** - * disable all events, the current hart will be - * woken up from reset vector when other hart - * writes its PCS (power control slot) control - * register - */ - smu_set_wakeup_events(&smu, 0x0, hartid); - smu_set_command(&smu, DEEP_SLEEP_CMD, hartid); - - rc = smu_set_reset_vector(&smu, - (ulong)__ae350_enable_coherency_warmboot, - hartid); - if (rc) - goto fail; - - __ae350_disable_coherency(); - - wfi(); - -fail: - /* It should never reach here */ - sbi_hart_hang(); - return 0; + struct andes_hart_data *andes_hdata = sbi_scratch_offset_ptr(scratch, + andes_hart_data_offset); + + csr_write(CSR_MCACHE_CTL, andes_hdata->mcache_ctl); + csr_write(CSR_MMISC_CTL, andes_hdata->mmisc_ctl); + csr_write(CSR_MPFT_CTL, andes_hdata->mpft_ctl); + csr_write(CSR_MSLIDELEG, andes_hdata->mslideleg); + csr_write(CSR_MXSTATUS, andes_hdata->mxstatus); + csr_write(CSR_SLIE, andes_hdata->slie); + csr_write(CSR_SLIP, andes_hdata->slip); + csr_write(CSR_PMACFG0, andes_hdata->pmacfg0); + csr_write_num(CSR_PMACFG0 + 2, andes_hdata->pmacfg2); + for (int i = 0; i < 16; i++) + csr_write_num(CSR_PMAADDR0 + i, andes_hdata->pmaaddrX[i]); } -static const struct sbi_hsm_device andes_smu = { - .name = "andes_smu", - .hart_start = ae350_hart_start, - .hart_stop = ae350_hart_stop, -}; - -static void ae350_hsm_device_init(const void *fdt) +void ae350_enable_coherency_warmboot(void) { - int rc; - - rc = fdt_parse_compat_addr(fdt, (uint64_t *)&smu.addr, - "andestech,atcsmu"); - - if (!rc) { - sbi_hsm_set_device(&andes_smu); - } + ae350_enable_coherency(); + _start_warm(); } -static int ae350_final_init(bool cold_boot) +static int ae350_early_init(bool cold_boot) { - if (cold_boot) { - const void *fdt = fdt_get_address(); + u32 hartid = current_hartid(); + u32 sleep_type = atcsmu_get_sleep_type(hartid); - ae350_hsm_device_init(fdt); + if (cold_boot) { + andes_hart_data_offset = sbi_scratch_alloc_offset(sizeof(struct andes_hart_data)); + if (!andes_hart_data_offset) + return SBI_ENOMEM; } - return generic_final_init(cold_boot); + /* Don't restore Andes CSRs during boot or wake up from light sleep */ + if (sbi_init_count(current_hartindex()) && sleep_type == SBI_SUSP_SLEEP_TYPE_SUSPEND) + ae350_non_ret_restore(sbi_scratch_thishart_ptr()); + + return generic_early_init(cold_boot); } static int ae350_platform_init(const void *fdt, int nodeoff, const struct fdt_match *match) { - generic_platform_ops.final_init = ae350_final_init; + generic_platform_ops.early_init = ae350_early_init; generic_platform_ops.extensions_init = andes_pmu_extensions_init; generic_platform_ops.pmu_init = andes_pmu_init; generic_platform_ops.vendor_ext_provider = andes_sbi_vendor_ext_provider; diff --git a/platform/generic/andes/andes_pmu.c b/platform/generic/andes/andes_pmu.c index 9eee4edc8dc..b3e49e8d771 100644 --- a/platform/generic/andes/andes_pmu.c +++ b/platform/generic/andes/andes_pmu.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -58,12 +59,12 @@ static struct sbi_pmu_device andes_pmu = { .hw_counter_filter_mode = andes_hw_counter_filter_mode }; -int andes_pmu_extensions_init(struct sbi_hart_features *hfeatures) +int andes_pmu_extensions_init(bool cold_boot) { struct sbi_scratch *scratch = sbi_scratch_thishart_ptr(); int rc; - rc = generic_extensions_init(hfeatures); + rc = generic_extensions_init(cold_boot); if (rc) return rc; diff --git a/platform/generic/andes/objects.mk b/platform/generic/andes/objects.mk index f85ad48162b..660546c312f 100644 --- a/platform/generic/andes/objects.mk +++ b/platform/generic/andes/objects.mk @@ -3,7 +3,7 @@ # carray-platform_override_modules-$(CONFIG_PLATFORM_ANDES_AE350) += andes_ae350 -platform-objs-$(CONFIG_PLATFORM_ANDES_AE350) += andes/ae350.o andes/sleep.o +platform-objs-$(CONFIG_PLATFORM_ANDES_AE350) += andes/ae350.o carray-platform_override_modules-$(CONFIG_PLATFORM_ANDES_QILAI) += andes_qilai platform-objs-$(CONFIG_PLATFORM_ANDES_QILAI) += andes/qilai.o diff --git a/platform/generic/andes/sleep.S b/platform/generic/andes/sleep.S deleted file mode 100644 index 361aff3a602..00000000000 --- a/platform/generic/andes/sleep.S +++ /dev/null @@ -1,70 +0,0 @@ -/* - * SPDX-License-Identifier: BSD-2-Clause - * - * Copyright (c) 2023 Andes Technology Corporation - * - * Authors: - * Yu Chien Peter Lin - */ - -#include -#include -#include - - .section .text, "ax", %progbits - .align 3 - .global __ae350_disable_coherency -__ae350_disable_coherency: - /* flush d-cache */ - csrw CSR_MCCTLCOMMAND, 0x6 - /* disable i/d-cache */ - csrc CSR_MCACHE_CTL, 0x3 - /* disable d-cache coherency */ - lui t1, 0x80 - csrc CSR_MCACHE_CTL, t1 - /* - * wait for mcache_ctl.DC_COHSTA to be cleared, - * the bit is hard-wired 0 on platforms w/o CM - * (Coherence Manager) - */ -check_cm_disabled: - csrr t1, CSR_MCACHE_CTL - srli t1, t1, 20 - andi t1, t1, 0x1 - bnez t1, check_cm_disabled - - ret - - .section .text, "ax", %progbits - .align 3 - .global __ae350_enable_coherency -__ae350_enable_coherency: - /* enable d-cache coherency */ - lui t1, 0x80 - csrs CSR_MCACHE_CTL, t1 - /* - * mcache_ctl.DC_COHEN is hard-wired 0 on platforms - * w/o CM support - */ - csrr t1, CSR_MCACHE_CTL - srli t1, t1, 19 - andi t1, t1, 0x1 - beqz t1, enable_L1_cache - /* wait for mcache_ctl.DC_COHSTA to be set */ -check_cm_enabled: - csrr t1, CSR_MCACHE_CTL - srli t1, t1, 20 - andi t1, t1, 0x1 - beqz t1, check_cm_enabled -enable_L1_cache: - /* enable i/d-cache */ - csrs CSR_MCACHE_CTL, 0x3 - - ret - - .section .text, "ax", %progbits - .align 3 - .global __ae350_enable_coherency_warmboot -__ae350_enable_coherency_warmboot: - call ra, __ae350_enable_coherency - j _start_warm diff --git a/platform/generic/configs/defconfig b/platform/generic/configs/defconfig index aab1560f239..1e4104b6fe0 100644 --- a/platform/generic/configs/defconfig +++ b/platform/generic/configs/defconfig @@ -5,14 +5,20 @@ CONFIG_PLATFORM_ESWIN_EIC770X=y CONFIG_PLATFORM_OPENHWGROUP_ARIANE=y CONFIG_PLATFORM_OPENHWGROUP_OPENPITON=y CONFIG_PLATFORM_RENESAS_RZFIVE=y +CONFIG_PLATFORM_SIFIVE_DEV=y CONFIG_PLATFORM_SIFIVE_FU540=y CONFIG_PLATFORM_SIFIVE_FU740=y CONFIG_PLATFORM_SOPHGO_SG2042=y CONFIG_PLATFORM_STARFIVE_JH7110=y +CONFIG_PLATFORM_TENSTORRENT_ATLANTIS=y CONFIG_PLATFORM_THEAD=y -CONFIG_PLATFORM_MIPS_P8700=y +CONFIG_PLATFORM_MIPS_P8700_EYEQ7H=y +CONFIG_PLATFORM_MIPS_P8700_BOSTON=y CONFIG_PLATFORM_SPACEMIT_K1=y CONFIG_FDT_CACHE=y +CONFIG_FDT_CACHE_ANDES_LLCACHE=y +CONFIG_SBI_ISA_EXT_EMU=y +CONFIG_EMU_ZVBB_VLEN_128=y CONFIG_FDT_CACHE_SIFIVE_CCACHE=y CONFIG_FDT_CACHE_SIFIVE_EC=y CONFIG_FDT_CACHE_SIFIVE_PL2=y @@ -23,10 +29,12 @@ CONFIG_FDT_GPIO_DESIGNWARE=y CONFIG_FDT_GPIO_SIFIVE=y CONFIG_FDT_GPIO_STARFIVE=y CONFIG_FDT_HSM=y +CONFIG_FDT_HSM_ANDES_ATCSMU=y CONFIG_FDT_HSM_RPMI=y CONFIG_FDT_HSM_SIFIVE_TMC0=y CONFIG_FDT_I2C=y CONFIG_FDT_I2C_SIFIVE=y +CONFIG_FDT_I2C_SPACEMIT=y CONFIG_FDT_I2C_DW=y CONFIG_FDT_IPI=y CONFIG_FDT_IPI_MSWI=y @@ -43,12 +51,15 @@ CONFIG_FDT_REGMAP_SYSCON=y CONFIG_FDT_RESET=y CONFIG_FDT_RESET_ATCWDT200=y CONFIG_FDT_RESET_GPIO=y +CONFIG_FDT_RESET_LITEX=y CONFIG_FDT_RESET_HTIF=y CONFIG_FDT_RESET_RPMI=y CONFIG_FDT_RESET_SG2042_HWMON_MCU=y +CONFIG_FDT_RESET_SPACEMIT_P1=y CONFIG_FDT_RESET_SUNXI_WDT=y CONFIG_FDT_RESET_SYSCON=y CONFIG_FDT_SERIAL=y +CONFIG_FDT_SERIAL_ALTR_JUART=y CONFIG_FDT_SERIAL_CADENCE=y CONFIG_FDT_SERIAL_GAISLER=y CONFIG_FDT_SERIAL_HTIF=y @@ -60,6 +71,7 @@ CONFIG_FDT_SERIAL_UART8250=y CONFIG_FDT_SERIAL_XILINX_UARTLITE=y CONFIG_SERIAL_SEMIHOSTING=y CONFIG_FDT_SUSPEND=y +CONFIG_FDT_SUSPEND_ANDES_ATCSMU=y CONFIG_FDT_SUSPEND_RPMI=y CONFIG_FDT_SUSPEND_SIFIVE_SMC0=y CONFIG_FDT_TIMER=y @@ -72,3 +84,5 @@ CONFIG_FDT_MPXY_RPMI_VOLTAGE=y CONFIG_FDT_MPXY_RPMI_DEVICE_POWER=y CONFIG_FDT_MPXY_RPMI_PERFORMANCE=y CONFIG_FDT_MPXY_RPMI_SYSMSI=y +CONFIG_FDT_MPXY_RPMI_MM=y +CONFIG_FDT_MPXY_RPMI_LOGGING=y diff --git a/platform/generic/eswin/eic770x.c b/platform/generic/eswin/eic770x.c index 7330df9fbdc..3fe3d090696 100644 --- a/platform/generic/eswin/eic770x.c +++ b/platform/generic/eswin/eic770x.c @@ -10,14 +10,110 @@ #include #include #include +#include +#include +#include #include #include +#include #include #include static struct sbi_hart_protection eswin_eic7700_pmp_protection; +static volatile bool eic770x_power_down = false; -static int eic770x_system_reset_check(u32 type, u32 reason) +static int eic770x_hart_start(u32 hartid, ulong saddr) +{ + u32 hartindex = sbi_hartid_to_hartindex(hartid); + + /* + * saddr is ignored intentionally. + * For non-power-down scenarios, eic770x_hart_stop simply + * returns, putting the hart in atomic_read(&hdata->state) + * loop in sbi_hsm_hart_wait. We wake it up if it's in wfi() + */ + return sbi_ipi_raw_send(hartindex, true); +} + +static int eic770x_hart_stop() +{ + /* + * fence to enforce all previous ipi clears are done + * Refer to comments below in eic770x_cease_other_harts + */ + asm volatile ("fence o, r"); + + if (!eic770x_power_down) + return SBI_ENOTSUPP; + + /* + * bit 0: disableDCacheClockGate + * When some or all warm boot harts haven't gone under at least 1 + * cycle of hsm start/stop, (happens if reset is issued in pre- + * boot environment u-boot/UEFI where all warm boot harts are + * pending start), the FEAT0 CSR still holds the SoC reset values, + * and disableDCacheClockGate is set. A CEASE instruction executed + * when disableDCacheClockGate=1 will not properly reflect its + * ceased status in mcput_cease_from_tile_x. Thus, clear it before + * CEASE. + */ + csr_clear(EIC770X_CSR_FEAT0, 0x1); + + sifive_cease(); +} + +void eic770x_cease_other_harts(void) +{ + u32 to_cease[2] = {}; + + eic770x_power_down = true; + sbi_for_each_hartindex(i) { + u32 hartid = sbi_hartindex_to_hartid(i); + u32 die = hart_die(hartid); + u32 core = hart_core(hartid); + + /* Only wait for other harts */ + if (i == current_hartindex()) + continue; + /* + * Bring harts out of WFI in sbi_hsm_hart_wait + * Harts won't miss this IPI, because: + * 1. If hart goes to wfi() in sbi_hsm_hart_wait, + * it must have not observed eic770x_power_down + * 2. If it hasn't observed eic770x_power_down, + * then it must haven't observed the IPI sent, + * given the wmb() in sbi_ipi_raw_send + * 3. Given the fence o, r, any previous ipi_clear + * can't fall-through the read of eic770x_power_down + */ + sbi_ipi_raw_send(i, false); + to_cease[die] |= EIC770X_MC_CEASE_BIT(core); + } + + for (u32 die = 0; die < array_size(to_cease); die++) { + /* + * MCPU status indicates the wfi/debug/halt/cease status + * of each individual harts in the same die. The value + * can change on the fly, but for ceased harts, the cease + * bit remains high until reset + */ + u32 *status = (u32*)EIC770X_MCPU_STATUS(die); + + if (!to_cease[die]) + continue; + + /* Wait for mcput_cease_from_tile_x */ + while ((readl(status) & to_cease[die]) != to_cease[die]); + } +} + +static const struct sbi_hsm_device eswin_eic770x_hsm = { + .name = "eic770x_hsm", + .hart_start = eic770x_hart_start, + .hart_stop = eic770x_hart_stop, +}; + +static int eic7700_system_reset_check(u32 type, u32 reason) { switch (type) { case SBI_SRST_RESET_TYPE_COLD_REBOOT: @@ -28,23 +124,23 @@ static int eic770x_system_reset_check(u32 type, u32 reason) } } -static void eic770x_system_reset(u32 type, u32 reason) +static void eic7700_system_reset(u32 type, u32 reason) { switch (type) { case SBI_SRST_RESET_TYPE_COLD_REBOOT: case SBI_SRST_RESET_TYPE_WARM_REBOOT: - sbi_printf("%s: resetting...\n", __func__); - writel(EIC770X_SYSCRG_RST_VAL, (void *)EIC770X_SYSCRG_RST); + eic770x_cease_other_harts(); + writel(EIC770X_SYSRST_VAL, (void *)EIC770X_SYSCRG_SYSRST); } - sbi_hart_hang(); + sifive_cease(); } static struct sbi_system_reset_device *board_reset = NULL; -static struct sbi_system_reset_device eic770x_reset = { - .name = "eic770x_reset", - .system_reset_check = eic770x_system_reset_check, - .system_reset = eic770x_system_reset, +static struct sbi_system_reset_device eic7700_reset = { + .name = "eic7700_reset", + .system_reset_check = eic7700_system_reset_check, + .system_reset = eic7700_system_reset, }; #define add_root_mem_chk(...) do { \ @@ -145,7 +241,7 @@ static int eswin_eic7700_early_init(bool cold_boot) if (board_reset) sbi_system_reset_add_device(board_reset); - sbi_system_reset_add_device(&eic770x_reset); + sbi_system_reset_add_device(&eic7700_reset); /* Enable bus blocker */ writel(1, (void*)EIC770X_TL64D2D_OUT); @@ -231,6 +327,9 @@ static int eswin_eic7700_final_init(bool cold_boot) int rc; + if (cold_boot) + sbi_hsm_set_device(&eswin_eic770x_hsm); + /** * Do generic_final_init stuff first, because it touchs FDT. * After final_init, we'll block entire memory port with the @@ -254,21 +353,21 @@ static int eswin_eic7700_final_init(bool cold_boot) __func__); return SBI_EFAIL; } - pmp_set(pmp_idx++, sbi_domain_get_oldpmp_flags(reg), - reg->base, reg->order); + sbi_hart_pmp_set(pmp_idx++, sbi_domain_get_oldpmp_flags(reg), + reg->base, reg->order); } - pmp_set(PMP_RESERVED_A, PMP_L, EIC770X_L3_ZERO_REMOTE, - log2roundup(EIC770X_L3_ZERO_SIZE)); + sbi_hart_pmp_set(PMP_RESERVED_A, PMP_L, EIC770X_L3_ZERO_REMOTE, + log2roundup(EIC770X_L3_ZERO_SIZE)); /** * Enable P550 internal + System Port, so OpenSBI can access * CLINT/PLIC/UART. Might be overwritten in pmp_configure. */ - pmp_set(PMP_FREE_A_START + PMP_FREE_A_COUNT - 1, 0, 0, - log2roundup(EIC770X_MEMPORT_BASE)); + sbi_hart_pmp_set(PMP_FREE_A_START + PMP_FREE_A_COUNT - 1, 0, 0, + log2roundup(EIC770X_MEMPORT_BASE)); - pmp_set(PMP_RESERVED_B, PMP_L, 0, - log2roundup(EIC770X_MEMPORT_LIMIT)); + sbi_hart_pmp_set(PMP_RESERVED_B, PMP_L, 0, + log2roundup(EIC770X_MEMPORT_LIMIT)); /** * These must come after the setup of PMP, as we are about to * enable speculation and HW prefetcher bits @@ -321,13 +420,13 @@ static int eswin_eic7700_pmp_configure(struct sbi_scratch *scratch) if (pmp_idx >= pmp_max) goto no_more_pmp; - pmp_set(pmp_idx++, sbi_domain_get_oldpmp_flags(reg), - reg->base, reg->order); + sbi_hart_pmp_set(pmp_idx++, sbi_domain_get_oldpmp_flags(reg), + reg->base, reg->order); prev = reg; } /* Disable the rest */ while (pmp_idx < pmp_max) - pmp_disable(pmp_idx++); + sbi_hart_pmp_disable(pmp_idx++); /* Process the second free range B [7-7] */ pmp_idx = PMP_FREE_B_START, @@ -340,12 +439,12 @@ static int eswin_eic7700_pmp_configure(struct sbi_scratch *scratch) if (pmp_idx >= pmp_max) goto no_more_pmp; - pmp_set(pmp_idx++, sbi_domain_get_oldpmp_flags(reg), - reg->base, reg->order); + sbi_hart_pmp_set(pmp_idx++, sbi_domain_get_oldpmp_flags(reg), + reg->base, reg->order); } /* Disable the rest */ while (pmp_idx < pmp_max) - pmp_disable(pmp_idx++); + sbi_hart_pmp_disable(pmp_idx++); sbi_hart_pmp_fence(); return 0; @@ -357,14 +456,14 @@ static int eswin_eic7700_pmp_configure(struct sbi_scratch *scratch) static void eswin_eic7700_pmp_unconfigure(struct sbi_scratch *scratch) { /* Enable P550 internal + System Port */ - pmp_set(PMP_FREE_A_START + PMP_FREE_A_COUNT - 1, 0, 0, - log2roundup(EIC770X_MEMPORT_BASE)); + sbi_hart_pmp_set(PMP_FREE_A_START + PMP_FREE_A_COUNT - 1, 0, 0, + log2roundup(EIC770X_MEMPORT_BASE)); for (unsigned int i = 0; i < PMP_FREE_A_COUNT - 1; i++) - pmp_disable(i + PMP_FREE_A_START); + sbi_hart_pmp_disable(i + PMP_FREE_A_START); for (unsigned int i = 0; i < PMP_FREE_B_COUNT; i++) - pmp_disable(i + PMP_FREE_B_START); + sbi_hart_pmp_disable(i + PMP_FREE_B_START); } static struct sbi_hart_protection eswin_eic7700_pmp_protection = { diff --git a/platform/generic/eswin/hfp.c b/platform/generic/eswin/hfp.c index eabed191160..a6e73e185ff 100644 --- a/platform/generic/eswin/hfp.c +++ b/platform/generic/eswin/hfp.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -94,6 +95,7 @@ static int hfp_system_reset_check(u32 type, u32 reason) static void hfp_system_reset(u32 type, u32 reason) { + eic770x_cease_other_harts(); switch (type) { case SBI_SRST_RESET_TYPE_SHUTDOWN: hfp_send_bmc_msg(HFP_MSG_NOTIFY, HFP_CMD_POWER_OFF, @@ -104,7 +106,7 @@ static void hfp_system_reset(u32 type, u32 reason) NULL, 0); break; } - sbi_hart_hang(); + sifive_cease(); } static struct sbi_system_reset_device hfp_reset = { diff --git a/platform/generic/include/andes/andes.h b/platform/generic/include/andes/andes.h index bfedf034c90..dd2451718d0 100644 --- a/platform/generic/include/andes/andes.h +++ b/platform/generic/include/andes/andes.h @@ -6,17 +6,25 @@ #ifndef _RISCV_ANDES_H #define _RISCV_ANDES_H +#include +#include + /* Memory and Miscellaneous Registers */ +#define CSR_MPFT_CTL 0x7c5 #define CSR_MCACHE_CTL 0x7ca #define CSR_MCCTLCOMMAND 0x7cc +#define CSR_MMISC_CTL 0x7d0 /* Configuration Control & Status Registers */ #define CSR_MICM_CFG 0xfc0 #define CSR_MDCM_CFG 0xfc1 #define CSR_MMSC_CFG 0xfc2 -/* Machine Trap Related Registers */ +/* Trap Related Registers */ +#define CSR_MXSTATUS 0x7c4 #define CSR_MSLIDELEG 0x7d5 +#define CSR_SLIE 0x9c4 +#define CSR_SLIP 0x9c5 /* Counter Related Registers */ #define CSR_MCOUNTERWEN 0x7ce @@ -43,13 +51,23 @@ #define MMSC_IOCP_OFFSET 47 #define MMSC_IOCP_MASK (1ULL << MMSC_IOCP_OFFSET) +#define MCACHE_CTL_IC_EN_MASK BIT(0) +#define MCACHE_CTL_DC_EN_MASK BIT(1) #define MCACHE_CTL_CCTL_SUEN_OFFSET 8 #define MCACHE_CTL_CCTL_SUEN_MASK (1 << MCACHE_CTL_CCTL_SUEN_OFFSET) +#define MCACHE_CTL_DC_COHEN_MASK BIT(19) +#define MCACHE_CTL_DC_COHSTA_MASK BIT(20) /* Performance monitor */ #define MMSC_CFG_PMNDS_MASK (1 << 15) #define MIP_PMOVI (1 << 18) +/* Cache control commands */ +#define MCCTLCOMMAND_L1D_WBINVAL_ALL 6 + +/* AE350 platform specific sleep types */ +#define SBI_SUSP_AE350_LIGHT_SLEEP SBI_SUSP_PLATFORM_SLEEP_START + #ifndef __ASSEMBLER__ #define is_andes(series) \ @@ -67,4 +85,68 @@ #endif /* __ASSEMBLER__ */ +struct andes_hart_data { + unsigned long mcache_ctl; + unsigned long mmisc_ctl; + unsigned long mpft_ctl; + unsigned long mslideleg; + unsigned long mxstatus; + unsigned long slie; + unsigned long slip; + unsigned long pmacfg0; + unsigned long pmacfg2; + unsigned long pmaaddrX[16]; +}; + +void ae350_non_ret_save(struct sbi_scratch *scratch); +void ae350_non_ret_restore(struct sbi_scratch *scratch); +void ae350_enable_coherency_warmboot(void); + +/* + * On Andes 4X-series CPUs, disabling the L1 data cache causes the CPU to fetch + * data directly from RAM. However, L1 cache flushes write data back to the + * Last Level Cache (LLC). This discrepancy can lead to return address + * corruption on the stack. To prevent this, the following functions must + * be inlined. + */ +static inline void ae350_disable_coherency(void) +{ + /* + * To disable cache coherency of a core in AE350 platform, follow below steps: + * + * 1) Disable I/D-Cache + * 2) Write back and invalidate D-Cache + * 3) Disable D-Cache coherency + * 4) Wait for D-Cache disengaged from the coherence management + */ + csr_clear(CSR_MCACHE_CTL, MCACHE_CTL_IC_EN_MASK | MCACHE_CTL_DC_EN_MASK); + csr_write(CSR_MCCTLCOMMAND, MCCTLCOMMAND_L1D_WBINVAL_ALL); + csr_clear(CSR_MCACHE_CTL, MCACHE_CTL_DC_COHEN_MASK); + while (csr_read(CSR_MCACHE_CTL) & MCACHE_CTL_DC_COHSTA_MASK) + ; +} + +static inline void ae350_enable_coherency(void) +{ + /* + * To enable cache coherency of a core in AE350 platform, follow below steps: + * + * 1) Enable D-Cache coherency + * 2) Wait for D-Cache engaging in the coherence management + * 3) Enable I/D-Cache + */ + csr_set(CSR_MCACHE_CTL, MCACHE_CTL_DC_COHEN_MASK); + + /* + * mcache_ctl.DC_COHEN is hardwired to 0 if there is no coherence + * manager. In such situation, just enable the I/D-Cache to prevent + * permanently being stuck in the while loop. + */ + if (csr_read(CSR_MCACHE_CTL) & MCACHE_CTL_DC_COHEN_MASK) + while (!(csr_read(CSR_MCACHE_CTL) & MCACHE_CTL_DC_COHSTA_MASK)) + ; + + csr_set(CSR_MCACHE_CTL, MCACHE_CTL_IC_EN_MASK | MCACHE_CTL_DC_EN_MASK); +} + #endif /* _RISCV_ANDES_H */ diff --git a/platform/generic/include/andes/andes_pmu.h b/platform/generic/include/andes/andes_pmu.h index ad4564a6a75..e4446364b16 100644 --- a/platform/generic/include/andes/andes_pmu.h +++ b/platform/generic/include/andes/andes_pmu.h @@ -7,9 +7,9 @@ #ifndef _RISCV_ANDES_PMU_H #define _RISCV_ANDES_PMU_H -#include +#include int andes_pmu_init(void); -int andes_pmu_extensions_init(struct sbi_hart_features *hfeatures); +int andes_pmu_extensions_init(bool cold_boot); #endif /* _RISCV_ANDES_PMU_H */ diff --git a/platform/generic/include/eswin/eic770x.h b/platform/generic/include/eswin/eic770x.h index 67764ec0540..9c80b589071 100644 --- a/platform/generic/include/eswin/eic770x.h +++ b/platform/generic/include/eswin/eic770x.h @@ -14,6 +14,8 @@ struct eic770x_board_override { struct sbi_system_reset_device *reset_dev; }; +void eic770x_cease_other_harts(void); + /* CSRs */ #define EIC770X_CSR_BRPREDICT 0x7c0 #define EIC770X_CSR_FEAT0 0x7c1 @@ -55,11 +57,16 @@ struct eic770x_board_override { #define EIC770X_UART_REG_SHIFT 2 #define EIC770X_UART_REG_WIDTH 4 -#define EIC770X_SYSCRG (EIC770X_SYSPORT_LOCAL + 0x11828000UL) -#define EIC770X_SYSCRG_LSPCLK0 (EIC770X_SYSCRG + 0x200UL) -#define EIC770X_SYSCRG_SYSCLK (EIC770X_SYSCRG + 0x20cUL) -#define EIC770X_SYSCRG_RST (EIC770X_SYSCRG + 0x300UL) -#define EIC770X_SYSCRG_RST_VAL 0x1AC0FFE6UL +#define EIC770X_SYSCON(d) (EIC770X_SYSPORT_BASE(d) + 0x11810000UL) +#define EIC770X_MCPU_STATUS(d) (EIC770X_SYSCON(d) + 0x608UL) +#define EIC770X_MC_CEASE_BIT(c) (1UL << (15 - c)) + +#define EIC770X_SYSCRG(d) (EIC770X_SYSPORT_BASE(d) + 0x11828000UL) +#define EIC770X_SYSCRG_LOCAL (EIC770X_SYSPORT_LOCAL + 0x11828000UL) +#define EIC770X_SYSCRG_LSPCLK0 (EIC770X_SYSCRG_LOCAL + 0x200UL) +#define EIC770X_SYSCRG_MCCLK(d) (EIC770X_SYSCRG(d) + 0x208UL) +#define EIC770X_SYSCRG_SYSCLK (EIC770X_SYSCRG_LOCAL + 0x20cUL) +#define EIC770X_SYSCRG_SYSRST (EIC770X_SYSCRG_LOCAL + 0x300UL) /* Memory Ports */ #define EIC770X_MEMPORT_BASE 0x0080000000UL // 2G @@ -98,4 +105,7 @@ struct eic770x_board_override { divisor > 2 ? divisor : 2; \ }) +/* Reset definitions */ +#define EIC770X_SYSRST_VAL 0x1AC0FFE6UL + #endif diff --git a/platform/generic/include/mips/board.h b/platform/generic/include/mips/board.h deleted file mode 100644 index 6fe7b8b68ff..00000000000 --- a/platform/generic/include/mips/board.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * SPDX-License-Identifier: BSD-2-Clause - * - * Copyright (c) 2025 MIPS - * - */ - -#ifndef __BOARD_H__ -#define __BOARD_H__ - -/* Please review all defines to change for your board. */ - -/* Use in stw.S, p8700.c, p8700.h, mips-cm.h */ -#define CM_BASE 0x16100000 - -/* Use in mips-cm.h, p8700.c */ -#define CLUSTERS_IN_PLATFORM 1 -#if CLUSTERS_IN_PLATFORM > 1 -/* Define global CM bases for cluster 0, 1, 2, and more. */ -#define GLOBAL_CM_BASE0 0 -#define GLOBAL_CM_BASE1 0 -#define GLOBAL_CM_BASE2 0 -#endif - -/* Use in stw.S */ -#define TIMER_ADDR (CM_BASE + 0x8050) - -/* Use in cps-vec.S */ -#define DRAM_ADDRESS 0x80000000 -#define DRAM_SIZE 0x80000000 -#define DRAM_PMP_ADDR ((DRAM_ADDRESS >> 2) | ((DRAM_SIZE - 1) >> 3)) - -#endif diff --git a/platform/generic/include/mips/mips-cm.h b/platform/generic/include/mips/mips-cm.h index 419e8407d3f..d61cf104b88 100644 --- a/platform/generic/include/mips/mips-cm.h +++ b/platform/generic/include/mips/mips-cm.h @@ -14,45 +14,37 @@ /* Define 1 to print out CM read and write info */ #define DEBUG_CM 0 -#if CLUSTERS_IN_PLATFORM > 1 -static long GLOBAL_CM_BASE[CLUSTERS_IN_PLATFORM] = {GLOBAL_CM_BASE0, GLOBAL_CM_BASE1, GLOBAL_CM_BASE2}; -#else -static long GLOBAL_CM_BASE[CLUSTERS_IN_PLATFORM] = {CM_BASE}; -#endif - -#define CPS_ACCESSOR_R(unit, sz, base, off, name) \ -static inline u##sz read_##unit##_##name(u32 hartid, bool local_p) \ +#define CPS_ACCESSOR_R(unit, sz, off, name) \ +static inline u##sz read_##unit##_##name(u32 hartid) \ { \ u##sz value; \ - long cmd_reg; \ - int cl, co; \ - cl = cpu_cluster(hartid); \ - co = cpu_core(hartid); \ - cmd_reg = (local_p ? (base) : ((base) - CM_BASE + GLOBAL_CM_BASE[cl])) \ - + (co << CM_BASE_CORE_SHIFT) \ + int cl = cpu_cluster(hartid); \ + int co = cpu_core(hartid); \ + long cmd_reg = p8700_cm_info->gcr_base[cl] \ + + (co << CM_BASE_CORE_SHIFT) \ + off; \ if (DEBUG_CM) \ - sbi_printf("CM READ%d cmd_reg=%lx\n", sz, cmd_reg); \ + sbi_printf("CM_READ%d(0x%lx) ...\n", sz, cmd_reg); \ if (sz == 32) \ asm volatile("lw %0,0(%1)":"=r"(value):"r"(cmd_reg)); \ else if (sz == 64) \ asm volatile("ld %0,0(%1)":"=r"(value):"r"(cmd_reg)); \ asm volatile("fence"); \ + if (DEBUG_CM) \ + sbi_printf("CM_READ%d(0x%lx) -> 0x%lx\n", sz, cmd_reg, (unsigned long)value); \ return value; \ } -#define CPS_ACCESSOR_W(unit, sz, base, off, name) \ -static inline void write_##unit##_##name(u32 hartid, u##sz value, bool local_p) \ +#define CPS_ACCESSOR_W(unit, sz, off, name) \ +static inline void write_##unit##_##name(u32 hartid, u##sz value) \ { \ - long cmd_reg; \ - int cl, co; \ - cl = cpu_cluster(hartid); \ - co = cpu_core(hartid); \ - cmd_reg = (local_p ? (base) : ((base) - CM_BASE + GLOBAL_CM_BASE[cl])) \ - + (co << CM_BASE_CORE_SHIFT) \ + int cl = cpu_cluster(hartid); \ + int co = cpu_core(hartid); \ + long cmd_reg = p8700_cm_info->gcr_base[cl] \ + + (co << CM_BASE_CORE_SHIFT) \ + off; \ if (DEBUG_CM) \ - sbi_printf("CM WRITE%d cmd_reg=%lx value=%lx\n", sz, \ + sbi_printf("CM_WRITE%d(0x%lx, 0x%lx)\n", sz, \ cmd_reg, (unsigned long)value); \ if (sz == 32) \ asm volatile("sw %0,0(%1)"::"r"(value),"r"(cmd_reg)); \ @@ -61,18 +53,19 @@ static inline void write_##unit##_##name(u32 hartid, u##sz value, bool local_p) asm volatile("fence"); \ } -#define CPS_ACCESSOR_RW(unit, sz, base, off, name) \ - CPS_ACCESSOR_R(unit, sz, base, off, name) \ - CPS_ACCESSOR_W(unit, sz, base, off, name) +#define CPS_ACCESSOR_RW(unit, sz, off, name) \ + CPS_ACCESSOR_R(unit, sz, off, name) \ + CPS_ACCESSOR_W(unit, sz, off, name) #define CPC_CX_ACCESSOR_RW(sz, off, name) \ - CPS_ACCESSOR_RW(cpc, sz, CPC_BASE, CPC_OFF_LOCAL + (off), co_##name) + CPS_ACCESSOR_RW(cpc, sz, CPC_OFFSET + CPC_OFF_LOCAL + (off), co_##name) #define GCR_CX_ACCESSOR_RW(sz, off, name) \ - CPS_ACCESSOR_RW(gcr, sz, CM_BASE, GCR_OFF_LOCAL + (off), co_##name) + CPS_ACCESSOR_RW(gcr, sz, GCR_OFF_LOCAL + (off), co_##name) GCR_CX_ACCESSOR_RW(64, cpu_hart(hartid) << CM_BASE_HART_SHIFT, reset_base) GCR_CX_ACCESSOR_RW(32, GCR_CORE_COH_EN, coherence) +GCR_CX_ACCESSOR_RW(64, GCR_BASE_OFFSET, base) CPC_CX_ACCESSOR_RW(32, CPC_Cx_VP_RUN, vp_run) CPC_CX_ACCESSOR_RW(32, CPC_Cx_VP_STOP, vp_stop) @@ -80,9 +73,11 @@ CPC_CX_ACCESSOR_RW(32, CPC_Cx_CMD, cmd) CPC_CX_ACCESSOR_RW(32, CPC_Cx_STAT_CONF, stat_conf) #define CPC_ACCESSOR_RW(sz, off, name) \ - CPS_ACCESSOR_RW(cpc, sz, CPC_BASE, off, name) + CPS_ACCESSOR_RW(cpc, sz, CPC_OFFSET + (off), name) CPC_ACCESSOR_RW(32, CPC_PWRUP_CTL, pwrup_ctl) +CPC_ACCESSOR_RW(64, CPC_TIMECTL, timectl) +CPC_ACCESSOR_RW(64, CPC_HRTIME, hrtime) CPC_ACCESSOR_RW(32, CPC_CM_STAT_CONF, cm_stat_conf) #endif diff --git a/platform/generic/include/mips/p8700.h b/platform/generic/include/mips/p8700.h index b02aaed4b81..e64c62e1703 100644 --- a/platform/generic/include/mips/p8700.h +++ b/platform/generic/include/mips/p8700.h @@ -8,19 +8,29 @@ #ifndef __P8700_H__ #define __P8700_H__ -#include +/** Coherence manager information + * + * @num_cm: Number of coherence manager + * @gcr_base: Array of base address of the CM + */ +struct p8700_cm_info { + unsigned int num_cm; + unsigned long *gcr_base; +}; + +extern const struct p8700_cm_info *p8700_cm_info; /* PMA */ -#define CSR_MIPSPMACFG0 0x7e0 -#define CSR_MIPSPMACFG1 0x7e1 -#define CSR_MIPSPMACFG2 0x7e2 -#define CSR_MIPSPMACFG3 0x7e3 -#define CSR_MIPSPMACFG4 0x7e4 -#define CSR_MIPSPMACFG5 0x7e5 -#define CSR_MIPSPMACFG6 0x7e6 -#define CSR_MIPSPMACFG7 0x7e7 -#define CSR_MIPSPMACFG8 0x7e8 -#define CSR_MIPSPMACFG9 0x7e9 +#define CSR_MIPSPMACFG0 0x7e0 +#define CSR_MIPSPMACFG1 0x7e1 +#define CSR_MIPSPMACFG2 0x7e2 +#define CSR_MIPSPMACFG3 0x7e3 +#define CSR_MIPSPMACFG4 0x7e4 +#define CSR_MIPSPMACFG5 0x7e5 +#define CSR_MIPSPMACFG6 0x7e6 +#define CSR_MIPSPMACFG7 0x7e7 +#define CSR_MIPSPMACFG8 0x7e8 +#define CSR_MIPSPMACFG9 0x7e9 #define CSR_MIPSPMACFG10 0x7ea #define CSR_MIPSPMACFG11 0x7eb #define CSR_MIPSPMACFG12 0x7ec @@ -35,11 +45,10 @@ /* MIPS CSR */ #define CSR_MIPSTVEC 0x7c0 -#define CSR_MIPSCONFIG0 0x7d0 +#define CSR_MIPSCACHEERR 0x7c5 +#define CSR_MIPSERRCTL 0x7c6 +#define CSR_MIPSDIAGDATA 0x7c8 #define CSR_MIPSCONFIG1 0x7d1 -#define CSR_MIPSCONFIG2 0x7d2 -#define CSR_MIPSCONFIG3 0x7d3 -#define CSR_MIPSCONFIG4 0x7d4 #define CSR_MIPSCONFIG5 0x7d5 #define CSR_MIPSCONFIG6 0x7d6 #define CSR_MIPSCONFIG7 0x7d7 @@ -48,33 +57,45 @@ #define CSR_MIPSCONFIG10 0x7da #define CSR_MIPSCONFIG11 0x7db +/* fields for CSR_MIPSCACHEERR */ +#define MIPSCACHEERR_STATE GENMASK(31,30) +#define MIPSCACHEERR_ARRAY GENMASK(29,26) +#define MIPSCACHEERR_ERR_BIT GENMASK(25,20) /* for correctable */ +#define MIPSCACHEERR_F2 BIT(23) /* for uncorrectable */ +#define MIPSCACHEERR_F BIT(22) /* for uncorrectable */ +#define MIPSCACHEERR_P BIT(21) /* for uncorrectable */ +#define MIPSCACHEERR_S BIT(20) /* for uncorrectable */ +#define MIPSCACHEERR_WAY GENMASK(19,17) +#define MIPSCACHEERR_INDEX GENMASK(16,4) +#define MIPSCACHEERR_WORD GENMASK(3,0) + +/* fields for CSR_MIPSERRCTL */ +#define MIPSERRCTL_PE BIT(31) +#define MIPSERRCTL_BUS_TO GENMASK(19,10) + +/* fields for CSR_MIPSCONFIG1 */ +#define MIPSCONFIG1_L2C BIT(31) +#define MIPSCONFIG1_IS GENMASK(24,22) +#define MIPSCONFIG1_IL GENMASK(21,19) +#define MIPSCONFIG1_IA GENMASK(18,16) +#define MIPSCONFIG1_DS GENMASK(15,13) +#define MIPSCONFIG1_DL GENMASK(12,10) +#define MIPSCONFIG1_DA GENMASK(9,7) + #define MIPSCONFIG5_MTW 4 -#define GEN_MASK(h, l) (((1ul << ((h) + 1 - (l))) - 1) << (l)) -#define EXT(val, mask) (((val) & (mask)) >> (__builtin_ffs(mask) - 1)) +/* mhartID structure */ +#define P8700_HARTID_CLUSTER GENMASK(19, 16) +#define P8700_HARTID_CORE GENMASK(11, 4) +#define P8700_HARTID_HART GENMASK(3, 0) +#define cpu_cluster(i) EXTRACT_FIELD(i, P8700_HARTID_CLUSTER) +#define cpu_core(i) EXTRACT_FIELD(i, P8700_HARTID_CORE) +#define cpu_hart(i) EXTRACT_FIELD(i, P8700_HARTID_HART) -/* - * We allocate the number of bits to encode clusters, cores, and harts - * from the original mhartid to a new dense index. - */ -#define NUM_OF_BITS_FOR_CLUSTERS 4 -#define NUM_OF_BITS_FOR_CORES 12 -#define NUM_OF_BITS_FOR_HARTS 4 - -/* To get the field from new/hashed mhartid */ -#define NEW_CLUSTER_SHIFT (NUM_OF_BITS_FOR_CORES + NUM_OF_BITS_FOR_HARTS) -#define NEW_CLUSTER_MASK ((1 << NUM_OF_BITS_FOR_CLUSTERS) - 1) -#define NEW_CORE_SHIFT NUM_OF_BITS_FOR_HARTS -#define NEW_CORE_MASK ((1 << NUM_OF_BITS_FOR_CORES) - 1) -#define NEW_HART_MASK ((1 << NUM_OF_BITS_FOR_HARTS) - 1) -#define cpu_cluster(i) (((i) >> NEW_CLUSTER_SHIFT) & NEW_CLUSTER_MASK) -#define cpu_core(i) (((i) >> NEW_CORE_SHIFT) & NEW_CORE_MASK) -#define cpu_hart(i) ((i) & NEW_HART_MASK) - -#define CPC_BASE (CM_BASE + 0x8000) +#define CPC_OFFSET (0x8000) #define SIZE_FOR_CPC_MTIME 0x10000 /* The size must be 2^order */ -#define AIA_BASE (CM_BASE + 0x40000) +#define AIA_OFFSET (0x40000) #define SIZE_FOR_AIA_M_MODE 0x20000 /* The size must be 2^order */ #define P8700_ALIGN 0x10000 @@ -85,6 +106,17 @@ /* GCR Block offsets */ #define GCR_OFF_LOCAL 0x2000 +#define GCR_GLOBAL_CONFIG 0x0000 +#define GCR_GC_NUM_CORES GENMASK(7, 0) +#define GCR_GC_NUM_IOCUS GENMASK(11, 8) +#define GCR_GC_NUM_MMIOS GENMASK(19, 16) +#define GCR_GC_NUM_AUX GENMASK(22, 20) +#define GCR_GC_NUM_CLUSTERS GENMASK(29, 23) +#define GCR_GC_HAS_ITU BIT(31) +#define GCR_GC_CL_ID GENMASK(39, 32) +#define GCR_GC_HAS_DBU BIT(40) +#define GCR_GC_NOC GENMASK(43, 41) + #define GCR_BASE_OFFSET 0x0008 #define GCR_CORE_COH_EN 0x00f8 #define GCR_CORE_COH_EN_EN (0x1 << 0) @@ -92,8 +124,69 @@ #define L2_PFT_CONTROL_OFFSET 0x0300 #define L2_PFT_CONTROL_B_OFFSET 0x0308 +#define GCR_L2_CONFIG 0x0130 +#define GCR_L2_ASSOC GENMASK(7, 0) +#define GCR_L2_LINE_SIZE GENMASK(11, 8) +#define GCR_L2_SET_SIZE GENMASK(15, 12) +#define GCR_L2_BYPASS BIT(20) +#define GCR_L2_COP_DATA_ECC_WE BIT(24) +#define GCR_L2_COP_TAG_ECC_WE BIT(25) +#define GCR_L2_COP_LRU_WE BIT(26) +#define GCR_L2_REG_EXISTS BIT(31) + +#define GCR_L2_TAG_ADDR 0x0600 +#define GCR_L2_TAG_STATE 0x0608 +#define GCR_L2_DATA 0x0610 +#define GCR_L2_ECC 0x0618 + +#define GCR_L2SM_COP 0x0620 +#define GCR_L2SM_COP_CMD GENMASK(1, 0) +#define L2SM_COP_CMD_NOP 0 +#define L2SM_COP_CMD_START 1 +#define L2SM_COP_CMD_ABORT 3 +#define GCR_L2SM_COP_TYPE GENMASK(4, 2) +#define L2SM_COP_TYPE_IDX_WBINV 0 +#define L2SM_COP_TYPE_IDX_STORETAG 1 +#define L2SM_COP_TYPE_IDX_STORETAGDATA 2 +#define L2SM_COP_TYPE_HIT_INV 4 +#define L2SM_COP_TYPE_HIT_WBINV 5 +#define L2SM_COP_TYPE_HIT_WB 6 +#define L2SM_COP_TYPE_FETCHLOCK 7 +#define GCR_L2SM_COP_RUNNING BIT(5) +#define GCR_L2SM_COP_RESULT GENMASK(8, 6) +#define L2SM_COP_RESULT_DONTCARE 0 +#define L2SM_COP_RESULT_DONE_OK 1 +#define L2SM_COP_RESULT_DONE_ERROR 2 +#define L2SM_COP_RESULT_ABORT_OK 3 +#define L2SM_COP_RESULT_ABORT_ERROR 4 +#define GCR_L2SM_COP_PRESENT BIT(31) +/* MMIO regions. Actual count in GCR_GLOBAL_CONFIG.GCR_GC_NUM_MMIOS */ +#define GCR_MMIO_BOTTOM(n) (0x700 + (n) * 0x10) /* n = 0..7 */ +#define GCR_MMIO_TOP(n) (0x708 + (n) * 0x10) /* n = 0..7 */ +#define GCR_MMIO_ADDR GENMASK(47, 16) /* both top and bottom */ +#define GCR_MMIO_BOTTOM_CCA GENMASK(9, 8) +#define GCR_MMIO_BOTTOM_FORCE_NC BIT(6) +/* + * 15:12 - reserved + * 11 - AUX3 + * 10 - AUX2 + * 9 - AUX1 + * 8 - AUX0 + * 7:1 - reserved + * 0 - Main memory port; MEM + */ +#define GCR_MMIO_BOTTOM_PORT GENMASK(5, 2) +#define GCR_MMIO_BOTTOM_DIS_RQ_LIM BIT(1) +#define GCR_MMIO_BOTTOM_EN BIT(0) + /* CPC Block offsets */ #define CPC_PWRUP_CTL 0x0030 +#define CPC_TIMECTL 0x0058 +#define TIMECTL_HARMED BIT(3) +#define TIMECTL_HSTOP BIT(2) +#define TIMECTL_MARMED BIT(1) +#define TIMECTL_MSTOP BIT(0) +#define CPC_HRTIME 0x0090 #define CPC_CM_STAT_CONF 0x1008 #define CPC_OFF_LOCAL 0x2000 @@ -106,8 +199,29 @@ #define CPC_Cx_CMD_RESET 0x4 #define CPC_Cx_STAT_CONF 0x0008 -#define CPC_Cx_STAT_CONF_SEQ_STATE GEN_MASK(22, 19) +#define CPC_Cx_STAT_CONF_SEQ_STATE GENMASK(22, 19) #define CPC_Cx_STAT_CONF_SEQ_STATE_U5 6 #define CPC_Cx_STAT_CONF_SEQ_STATE_U6 7 +extern const struct p8700_cm_info *p8700_cm_info; +void mips_p8700_dump_mmio(void); +void mips_p8700_pmp_set(unsigned int n, unsigned long flags, + unsigned long prot, unsigned long addr, + unsigned long log2len); +void mips_p8700_power_up_other_cluster(u32 hartid); +int mips_p8700_hart_start(u32 hartid, ulong saddr); +int mips_p8700_hart_stop(void); + +struct p8700_cache_info { + u32 line; + u32 assoc_ways; + u32 sets; +}; + +void mips_p8700_cache_info(struct p8700_cache_info *l1d, struct p8700_cache_info *l1i, + struct p8700_cache_info *l2); +int mips_p8700_add_memranges(void); +struct fdt_match; +int mips_p8700_platform_init(const void *fdt, int nodeoff, const struct fdt_match *match); + #endif diff --git a/platform/generic/include/platform_override.h b/platform/generic/include/platform_override.h index 3e6e50445fc..1b8127cb712 100644 --- a/platform/generic/include/platform_override.h +++ b/platform/generic/include/platform_override.h @@ -10,7 +10,6 @@ #ifndef __PLATFORM_OVERRIDE_H__ #define __PLATFORM_OVERRIDE_H__ -#include #include #include #include @@ -19,7 +18,7 @@ bool generic_cold_boot_allowed(u32 hartid); int generic_nascent_init(void); int generic_early_init(bool cold_boot); int generic_final_init(bool cold_boot); -int generic_extensions_init(struct sbi_hart_features *hfeatures); +int generic_extensions_init(bool cold_boot); int generic_domains_init(void); int generic_pmu_init(void); uint64_t generic_pmu_xlate_to_mhpmevent(uint32_t event_idx, uint64_t data); diff --git a/platform/generic/include/spacemit/k1.h b/platform/generic/include/spacemit/k1.h index bd666346564..7095bc089ba 100644 --- a/platform/generic/include/spacemit/k1.h +++ b/platform/generic/include/spacemit/k1.h @@ -25,33 +25,23 @@ #define PMU_AP_BASE 0xd4282800 -#define PMU_AP_CORE0_WAKEUP_OFFSET (PMU_AP_BASE + 0x12c) -#define PMU_AP_CORE4_WAKEUP_OFFSET (PMU_AP_BASE + 0x324) -#define PMU_AP_CLUSTER0_WAKEUP_OFFSET(index) (PMU_AP_CORE0_WAKEUP_OFFSET + index * 4) -#define PMU_AP_CLUSTER1_WAKEUP_OFFSET(index) (PMU_AP_CORE4_WAKEUP_OFFSET + index * 4) - -#define PMU_AP_CORE0_IDLE_CFG_OFFSET (PMU_AP_BASE + 0x124) -#define PMU_AP_CORE4_IDLE_CFG_OFFSET (PMU_AP_BASE + 0x304) -#define PMU_AP_CLUSTER0_IDLE_CFG_OFFSET(index) (PMU_AP_CORE0_IDLE_CFG_OFFSET + index * 4) -#define PMU_AP_CLUSTER1_IDLE_CFG_OFFSET(index) (PMU_AP_CORE4_IDLE_CFG_OFFSET + index * 4) - -#define PMU_AP_CORE0_WAKEUP PMU_AP_CLUSTER0_WAKEUP_OFFSET(0) -#define PMU_AP_CORE1_WAKEUP PMU_AP_CLUSTER0_WAKEUP_OFFSET(1) -#define PMU_AP_CORE2_WAKEUP PMU_AP_CLUSTER0_WAKEUP_OFFSET(2) -#define PMU_AP_CORE3_WAKEUP PMU_AP_CLUSTER0_WAKEUP_OFFSET(3) -#define PMU_AP_CORE4_WAKEUP PMU_AP_CLUSTER1_WAKEUP_OFFSET(0) -#define PMU_AP_CORE5_WAKEUP PMU_AP_CLUSTER1_WAKEUP_OFFSET(1) -#define PMU_AP_CORE6_WAKEUP PMU_AP_CLUSTER1_WAKEUP_OFFSET(2) -#define PMU_AP_CORE7_WAKEUP PMU_AP_CLUSTER1_WAKEUP_OFFSET(3) - -#define PMU_AP_CORE0_IDLE_CFG PMU_AP_CLUSTER0_IDLE_CFG_OFFSET(0) -#define PMU_AP_CORE1_IDLE_CFG PMU_AP_CLUSTER0_IDLE_CFG_OFFSET(1) -#define PMU_AP_CORE2_IDLE_CFG PMU_AP_CLUSTER0_IDLE_CFG_OFFSET(2) -#define PMU_AP_CORE3_IDLE_CFG PMU_AP_CLUSTER0_IDLE_CFG_OFFSET(3) -#define PMU_AP_CORE4_IDLE_CFG PMU_AP_CLUSTER1_IDLE_CFG_OFFSET(0) -#define PMU_AP_CORE5_IDLE_CFG PMU_AP_CLUSTER1_IDLE_CFG_OFFSET(1) -#define PMU_AP_CORE6_IDLE_CFG PMU_AP_CLUSTER1_IDLE_CFG_OFFSET(2) -#define PMU_AP_CORE7_IDLE_CFG PMU_AP_CLUSTER1_IDLE_CFG_OFFSET(3) +#define PMU_AP_CORE0_WAKEUP (PMU_AP_BASE + 0x12c) +#define PMU_AP_CORE1_WAKEUP (PMU_AP_BASE + 0x130) +#define PMU_AP_CORE2_WAKEUP (PMU_AP_BASE + 0x134) +#define PMU_AP_CORE3_WAKEUP (PMU_AP_BASE + 0x138) +#define PMU_AP_CORE4_WAKEUP (PMU_AP_BASE + 0x324) +#define PMU_AP_CORE5_WAKEUP (PMU_AP_BASE + 0x328) +#define PMU_AP_CORE6_WAKEUP (PMU_AP_BASE + 0x32c) +#define PMU_AP_CORE7_WAKEUP (PMU_AP_BASE + 0x330) + +#define PMU_AP_CORE0_IDLE_CFG (PMU_AP_BASE + 0x124) +#define PMU_AP_CORE1_IDLE_CFG (PMU_AP_BASE + 0x128) +#define PMU_AP_CORE2_IDLE_CFG (PMU_AP_BASE + 0x160) +#define PMU_AP_CORE3_IDLE_CFG (PMU_AP_BASE + 0x164) +#define PMU_AP_CORE4_IDLE_CFG (PMU_AP_BASE + 0x304) +#define PMU_AP_CORE5_IDLE_CFG (PMU_AP_BASE + 0x308) +#define PMU_AP_CORE6_IDLE_CFG (PMU_AP_BASE + 0x30c) +#define PMU_AP_CORE7_IDLE_CFG (PMU_AP_BASE + 0x310) /* power down */ #define PMU_AP_IDLE_PWRDWN BIT(0) @@ -68,7 +58,7 @@ #define C0_RVBADDR_LO_ADDR 0xd4282db0 #define C0_RVBADDR_HI_ADDR 0xd4282db4 #define C1_RVBADDR_LO_ADDR 0xd4282eb0 -#define C1_RVBADDR_HI_ADDR 0xd4282c04 +#define C1_RVBADDR_HI_ADDR 0xd4282eb4 #define CCI_550_PLATFORM_CCI_ADDR 0xd8500000 diff --git a/platform/generic/include/tenstorrent/ascalon.h b/platform/generic/include/tenstorrent/ascalon.h new file mode 100644 index 00000000000..5d7b7635d09 --- /dev/null +++ b/platform/generic/include/tenstorrent/ascalon.h @@ -0,0 +1,12 @@ +/* + * SPDX-FileCopyrightText: (c) 2025-2026 Tenstorrent USA, Inc. + * SPDX-License-Identifier: BSD-2-Clause + */ + +#ifndef __TENSTORRENT_ASCALON_H__ +#define __TENSTORRENT_ASCALON_H__ + +void tt_ascalon_discover_pmas_from_boot_hart(void); +void tt_ascalon_verify_pmas_nonboot_hart(void); + +#endif diff --git a/platform/generic/include/tenstorrent/pma.h b/platform/generic/include/tenstorrent/pma.h new file mode 100644 index 00000000000..db8310e6017 --- /dev/null +++ b/platform/generic/include/tenstorrent/pma.h @@ -0,0 +1,17 @@ +/* + * SPDX-FileCopyrightText: (c) 2025-2026 Tenstorrent USA, Inc. + * SPDX-License-Identifier: BSD-2-Clause + */ + +#ifndef __TENSTORRENT_PMA_H__ +#define __TENSTORRENT_PMA_H__ + +/* Max number of PMAs for devices (CPU, IOMMU) for Tenstorrent platforms. */ +#define TT_MAX_PMAS 32 + +u64 tt_pma_get(unsigned int n); +void tt_pma_set(unsigned int n, u64 pma); +bool tt_pma_validate(unsigned int i, u64 pma); +void tt_pma_print(unsigned int i, u64 pma); + +#endif diff --git a/platform/generic/include/thead/c9xx_errata.h b/platform/generic/include/thead/c9xx_errata.h index 40c1587bcf7..7a419e4169f 100644 --- a/platform/generic/include/thead/c9xx_errata.h +++ b/platform/generic/include/thead/c9xx_errata.h @@ -8,6 +8,7 @@ */ #define THEAD_QUIRK_ERRATA_TLB_FLUSH BIT(0) #define THEAD_QUIRK_ERRATA_THEAD_PMU BIT(1) +#define THEAD_QUIRK_ERRATA_JTLB BIT(2) void thead_register_tlb_flush_trap_handler(void); diff --git a/platform/generic/mips/boston.c b/platform/generic/mips/boston.c new file mode 100644 index 00000000000..87cb328dc4e --- /dev/null +++ b/platform/generic/mips/boston.c @@ -0,0 +1,150 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 MIPS + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Use in nascent init - not have DTB yet */ +#define DRAM_ADDRESS 0x80000000 +#define DRAM_SIZE 0x80000000 +#define DRAM_PMP_ADDR ((DRAM_ADDRESS >> 2) | ((DRAM_SIZE - 1) >> 3)) + +static const struct sbi_hsm_device mips_hsm = { + .name = "mips_hsm", + .hart_start = mips_p8700_hart_start, + .hart_stop = mips_p8700_hart_stop, +}; + +static int boston_final_init(bool cold_boot) +{ + if (cold_boot) + sbi_hsm_set_device(&mips_hsm); + + return generic_final_init(cold_boot); +} + +static int boston_early_init(bool cold_boot) +{ + int rc; + + rc = generic_early_init(cold_boot); + if (rc) + return rc; + + if (!cold_boot) + return 0; + + rc = mips_p8700_add_memranges(); + + return rc; +} + +static int boston_nascent_init(void) +{ + u64 hartid = current_hartid(); + unsigned long cm_base = p8700_cm_info->gcr_base[0]; + int i; + + /* Coherence enable for every core */ + if (cpu_hart(hartid) == 0) { + cm_base += (cpu_core(hartid) << CM_BASE_CORE_SHIFT); + __raw_writeq(GCR_CORE_COH_EN_EN, + (void *)(cm_base + GCR_OFF_LOCAL + + GCR_CORE_COH_EN)); + mb(); + } + + /* Set up pmp for DRAM */ + csr_write(CSR_PMPADDR14, DRAM_PMP_ADDR); + /* All from 0x0 */ + csr_write(CSR_PMPADDR15, 0x1fffffffffffffff); + csr_write(CSR_PMPCFG2, ((PMP_A_NAPOT|PMP_R|PMP_W|PMP_X)<<56)| + ((PMP_A_NAPOT|PMP_R|PMP_W|PMP_X)<<48)); + /* Set cacheable for pmp6, uncacheable for pmp7 */ + csr_write(CSR_MIPSPMACFG2, ((u64)CCA_CACHE_DISABLE << 56)| + ((u64)CCA_CACHE_ENABLE << 48)); + /* Reset pmpcfg0 */ + csr_write(CSR_PMPCFG0, 0); + /* Reset pmacfg0 */ + csr_write(CSR_MIPSPMACFG0, 0); + mb(); + + /* Per cluster set up */ + if (cpu_core(hartid) == 0 && cpu_hart(hartid) == 0) { + /* Enable L2 prefetch */ + __raw_writel(0xfffff110, + (void *)(cm_base + L2_PFT_CONTROL_OFFSET)); + __raw_writel(0x15ff, + (void *)(cm_base + L2_PFT_CONTROL_B_OFFSET)); + } + + /* Per core set up */ + if (cpu_hart(hartid) == 0) { + /* Enable load pair, store pair, and HTW */ + csr_clear(CSR_MIPSCONFIG7, (1<<12)|(1<<13)|(1<<7)); + + /* Disable noRFO, misaligned load/store */ + csr_set(CSR_MIPSCONFIG7, (1<<25)|(1<<9)); + + /* Enable L1-D$ Prefetch */ + csr_write(CSR_MIPSCONFIG11, 0xff); + + for (i = 0; i < 8; i++) { + csr_set(CSR_MIPSCONFIG8, 4 + 0x100 * i); + csr_set(CSR_MIPSCONFIG9, 8); + mb(); + RISCV_FENCE_I; + } + } + + /* Per hart set up */ + /* Enable AMO and RDTIME illegal instruction exceptions. */ + csr_set(CSR_MIPSCONFIG6, (1<<2)|(1<<1)); + + return 0; +} + +static int boston_platform_init(const void *fdt, int nodeoff, const struct fdt_match *match) +{ + int rc = mips_p8700_platform_init(fdt, nodeoff, match); + + if (rc) + return rc; + generic_platform_ops.early_init = boston_early_init; + generic_platform_ops.final_init = boston_final_init; + generic_platform_ops.nascent_init = boston_nascent_init; + generic_platform_ops.pmp_set = mips_p8700_pmp_set; + + return 0; +} + +static unsigned long boston_gcr_base[] = { + 0x16100000, +}; + +static struct p8700_cm_info boston_cm_info = { + .num_cm = array_size(boston_gcr_base), + .gcr_base = boston_gcr_base, +}; + +static const struct fdt_match boston_match[] = { + { .compatible = "mips,p8700-boston", .data = &boston_cm_info }, + { }, +}; + +const struct fdt_driver mips_p8700_boston = { + .match_table = boston_match, + .init = boston_platform_init, +}; diff --git a/platform/generic/mips/eyeq7h.c b/platform/generic/mips/eyeq7h.c new file mode 100644 index 00000000000..30af096c14f --- /dev/null +++ b/platform/generic/mips/eyeq7h.c @@ -0,0 +1,484 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 Mobileye + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MIPS_OLB1 0x67046000 +#define MIPS_OLB2 0x67047000 +#define MIPS_CM_CTL0 (0x14) +#define MIPS_CTL0_CM_PWR_UP BIT(0) +#define MIPS_CTL0_DBU_PWR_UP BIT(1) +#define MIPS_CTL0_CM_RST_HOLD BIT(2) +#define MIPS_CTL0_DBU_RST_HOLD BIT(3) +#define MIPS_CTL0_DBU_COLD_PWR_UP GENMASK(5, 4) /* after cold rst: 00 - pwr down, 01 -clk off */ +#define MIPS_CTL0_PARITY_EN BIT(6) +#define MIPS_CTL0_DBG_RST_DASRT BIT(7) +#define MIPS_CTL0_CACHE_HW_INIT_INHIBIT BIT(16) +#define MIPS_CTL0_SW_RESET_N BIT(17) +#define MIPS_CTL0_CORE_CLK_STS(n) BIT(28 + (n)) /* n = 0..3 */ + +#define OLB_ACC0 0x45000000 +#define OLB_ACC1 0x65000000 +#define OLB_XNN0 0x43600000 +#define OLB_XNN1 0x63600000 +#define NCORE 0x67800000 +#define OLB_WEST 0x48600000 +#define OLB_WEST_TSTCSR 0x60 +#define TSTCSR_PALLADIUM BIT(0) +#define TSTCSR_DDR_STUB BIT(1) +#define TSTCSR_MIPS12_PRESENT GENMASK(3, 2) +#define TSTCSR_ACC_PRESENT GENMASK(5, 4) + +#define OLB_WEST_CFG 0x68 +#define WEST_CFG_MIPS_MTIME_START BIT(8) + +/* Use in nascent init - not have DTB yet */ +#define DRAM_ADDRESS 0x800000000UL +#define DRAM_SIZE 0x800000000UL +#define DRAM_PMP_ADDR ((DRAM_ADDRESS >> 2) | ((DRAM_SIZE - 1) >> 3)) + +#define MMIO_BASE 0x00000000 +#define MMIO_SIZE 0x80000000 + +static int eyeq7h_active_clusters = 1; + +static long MIPS_OLB_ADDR[3] = {0, MIPS_OLB1, MIPS_OLB2}; + +static void eyeq7h_powerup_olb(u32 hartid) +{ + int cl = cpu_cluster(hartid); + volatile void *cmd; + u32 temp; + + if (cl < 1 || cl >= p8700_cm_info->num_cm || cl >= array_size(MIPS_OLB_ADDR)) + return; + + /* Get the MIPS_CM_CTL0 address */ + cmd = (volatile void *)(MIPS_OLB_ADDR[cl] + MIPS_CM_CTL0); + + /* set reset value. Value may be wrong after JTAG debug session */ + temp = MIPS_CTL0_CACHE_HW_INIT_INHIBIT | + INSERT_FIELD(0, MIPS_CTL0_DBU_COLD_PWR_UP, 2) | + MIPS_CTL0_DBU_PWR_UP | MIPS_CTL0_CM_PWR_UP; + writel(temp, cmd); + wmb(); + sbi_timer_udelay(10); + /* Enable HW cache init */ + temp = temp & ~MIPS_CTL0_CACHE_HW_INIT_INHIBIT; + /* deassert reset */ + temp = temp | MIPS_CTL0_SW_RESET_N; + writel(temp, cmd); + wmb(); + /* TODO: using CPU clock ready as reset complete indication, is it correct? */ + while(!(readl(cmd) & MIPS_CTL0_CORE_CLK_STS(0))) + cpu_relax(); +} + +static void eyeq7h_power_up_other_cluster(u32 hartid) +{ + unsigned int cl = cpu_cluster(hartid); + unsigned long cm_base = p8700_cm_info->gcr_base[cl]; + + /* Power up MIPS OLB */ + eyeq7h_powerup_olb(hartid); + /* remap local cluster address to its global address */ + writeq(cm_base, (void*)cm_base + GCR_BASE_OFFSET); + wmb(); + mips_p8700_power_up_other_cluster(hartid); +} + +static int hart_start(u32 hartid, ulong saddr) +{ + if (cpu_cluster(hartid) >= eyeq7h_active_clusters) { + sbi_printf("Requested CPU 0x%x in inactive/nonexistent cluster\n", hartid); + return SBI_EINVALID_ADDR; + } + + return mips_p8700_hart_start(hartid, saddr); +} + +static const struct sbi_hsm_device eyeq7h_hsm = { + .name = "eyeq7h_hsm", + .hart_start = hart_start, + .hart_stop = mips_p8700_hart_stop, +}; + +static struct sbi_domain_memregion *find_last_memregion(const struct sbi_domain *dom) +{ + struct sbi_domain_memregion *reg; + + sbi_domain_for_each_memregion(dom, reg) {} + return --reg; +} + +static int fixup_dram_region(const struct sbi_domain *dom, + struct sbi_domain_memregion *reg) +{ + const void *fdt = fdt_get_address(); + int node; + int ret; + uint64_t mem_addr, mem_size; + static const char mem_str[] = "memory"; + + if (!reg || !fdt) + return SBI_EINVAL; + + /* Find the memory range */ + node = fdt_node_offset_by_prop_value(fdt, -1, "device_type", + mem_str, sizeof(mem_str)); + ret = fdt_get_node_addr_size(fdt, node, 0, &mem_addr, &mem_size); + if (ret) + return ret; + reg->flags = SBI_DOMAIN_MEMREGION_MMIO; /* disable cache & prefetch */ + return sbi_domain_root_add_memrange(mem_addr, mem_size, mem_size, + (SBI_DOMAIN_MEMREGION_SU_READABLE | + SBI_DOMAIN_MEMREGION_SU_WRITABLE | + SBI_DOMAIN_MEMREGION_SU_EXECUTABLE)); +} + +static void fdt_disable_by_compat(void *fdt, const char *compatible) +{ + int node = 0; + + while ((node = fdt_node_offset_by_compatible(fdt, node, compatible)) >= 0) + fdt_setprop_string(fdt, node, "status", "disabled"); +} + +/** + * p8700_acc_clusters_do_fixup() - detect present accelerator clusters + * + * Detect what accelerator clusters are actually present in design and + * disable missed ones. Same bit indicates presence of the ACC and XNN + * clusters + */ +static void eyeq7h_acc_clusters_do_fixup(struct fdt_general_fixup *f, void *fdt) +{ + u32 tstcsr = readl((void*)OLB_WEST + OLB_WEST_TSTCSR); + u32 acc01_present = EXTRACT_FIELD(tstcsr, TSTCSR_ACC_PRESENT); + static const char YN[2] = {'N', 'Y'}; + + sbi_dprintf("OLB indicates ACC clusters[01] = [%c%c]\n", + YN[acc01_present & BIT(0)], + YN[(acc01_present >> 1) & BIT(0)]); + + /* if accelerators present, correspondend OLBS present too */ + /* deassert cluster resets for accelerators and XNN */ + if (!(acc01_present & BIT(0))) { + sbi_dprintf("Disable ACC0\n"); + fdt_disable_by_compat(fdt, "mobileye,eyeq7h-acc0-olb"); + fdt_disable_by_compat(fdt, "mobileye,eyeq7h-xnn0-olb"); + } else { + writel(0xff, (void*)OLB_ACC0 + 0x60); + writel(0xff, (void*)OLB_ACC0 + 0x64); + writel(0x7f, (void*)OLB_XNN0 + 0x60); + writel(0x7f, (void*)OLB_XNN0 + 0x64); + } + if (!(acc01_present & BIT(1))) { + sbi_dprintf("Disable ACC1\n"); + fdt_disable_by_compat(fdt, "mobileye,eyeq7h-acc1-olb"); + fdt_disable_by_compat(fdt, "mobileye,eyeq7h-xnn1-olb"); + } else { + writel(0xff, (void*)OLB_ACC1 + 0x60); + writel(0xff, (void*)OLB_ACC1 + 0x64); + writel(0x7f, (void*)OLB_XNN1 + 0x60); + writel(0x7f, (void*)OLB_XNN1 + 0x64); + } +} + +static struct fdt_general_fixup eyeq7h_acc_clusters_fixup = { + .name = "acc-clusters-fixup", + .do_fixup = eyeq7h_acc_clusters_do_fixup, +}; + +static void eyeq7h_cache_do_fixup(struct fdt_general_fixup *f, void *fdt) +{ + struct p8700_cache_info l1d, l1i, l2; + mips_p8700_cache_info(&l1d, &l1i, &l2); + + sbi_dprintf("Cache geometry:\n" + " D: %4d Kbytes line %3d bytes %2d ways %5d sets\n" + " I: %4d Kbytes line %3d bytes %2d ways %5d sets\n", + l1d.assoc_ways * l1d.line * l1d.sets / 1024, + l1d.line, l1d.assoc_ways, l1d.sets, + l1i.assoc_ways * l1i.line * l1i.sets / 1024, + l1i.line, l1i.assoc_ways, l1i.sets); + if (l2.line) { + sbi_dprintf(" L2: %4d Kbytes line %3d bytes %2d ways %5d sets\n", + l2.assoc_ways * l2.line * l2.sets / 1024, + l2.line, l2.assoc_ways, l2.sets); + } else { + sbi_dprintf(" L2: not present\n"); + } +} + +static struct fdt_general_fixup eyeq7h_cache_fixup = { + .name = "cache-fixup", + .do_fixup = eyeq7h_cache_do_fixup, +}; + +static int eyeq7h_final_init(bool cold_boot) +{ + if (!cold_boot) + return 0; + + sbi_hsm_set_device(&eyeq7h_hsm); + fdt_register_general_fixup(&eyeq7h_acc_clusters_fixup); + fdt_register_general_fixup(&eyeq7h_cache_fixup); + + return generic_final_init(cold_boot); +} + +/** + * There's 2 sources of information what clusters are present: + * - GCR_CONFIG register from the cluster 0 GCR + * - TSTCSR_MIPS12_PRESENT from the TSTCSR reg in OLB_WEST + * check that both indicates presence of clusters + */ +static void eyeq7h_init_clusters(void) +{ + unsigned long cm_base = p8700_cm_info->gcr_base[0]; + u64 gcr_config = readq((void*)cm_base + GCR_GLOBAL_CONFIG); + int num_clusters = EXTRACT_FIELD(gcr_config, GCR_GC_NUM_CLUSTERS); + u32 tstcsr = readl((void*)OLB_WEST + OLB_WEST_TSTCSR); + u32 mips12_present = EXTRACT_FIELD(tstcsr, TSTCSR_MIPS12_PRESENT); + /** + * total clusters, by mips[12] encoding. + * Don't support only mips2 present, consider as 1 total cluster + */ + static const int olb_clusters[4] = {1, 2, 1, 3}; + int num_olb_clusters = olb_clusters[mips12_present]; + static const char YN[2] = {'N', 'Y'}; + + sbi_dprintf("GCR_CONFIG reports %d clusters\n", num_clusters); + sbi_dprintf("OLB indicates %d clusters, mips[12] = [%c%c]\n", + num_olb_clusters, + YN[mips12_present & BIT(0)], + YN[mips12_present >> 1 & BIT(0)]); + if (num_clusters > num_olb_clusters) + num_clusters = num_olb_clusters; + sbi_dprintf("Use %d clusters\n", num_clusters); + /* Power up other clusters in the platform. */ + for (int i = 1; i < num_clusters; i++) { + eyeq7h_power_up_other_cluster(INSERT_FIELD(0, P8700_HARTID_CLUSTER, i)); + } + eyeq7h_active_clusters = num_clusters; + /** + * sync timers in all clusters. EQ7 have counters restart pins for clusters + * connected to the OLB. + * Stop/arm all counters, then restart all at once + */ + for (int i = 0; i < num_clusters; i++) { + write_cpc_timectl(INSERT_FIELD(0, P8700_HARTID_CLUSTER, i), + TIMECTL_HARMED | TIMECTL_HSTOP | TIMECTL_MARMED | TIMECTL_MSTOP); + } + { + u32 cfg = readl((void*)OLB_WEST + OLB_WEST_CFG); + + writel(cfg | WEST_CFG_MIPS_MTIME_START, (void*)OLB_WEST + OLB_WEST_CFG); + } +} + +static int eyeq7h_early_init(bool cold_boot) +{ + const struct sbi_domain *dom; + struct sbi_domain_memregion *reg; + int rc; + unsigned long cm_base; + + rc = generic_early_init(cold_boot); + if (rc) + return rc; + + if (!cold_boot) + return 0; + + cm_base = p8700_cm_info->gcr_base[0]; + sbi_dprintf("Remap Cluster %d CM 0x%lx -> 0x%lx\n", 0, + readq((void*)cm_base + GCR_BASE_OFFSET), + cm_base); + writeq(cm_base, (void*)cm_base + GCR_BASE_OFFSET); + wmb(); + eyeq7h_init_clusters(); +/** + * Memory map: + * 0x00_20080000 0x00_20100000 M:IRW- S:---- GCR local access (CM_BASE) + * 0x00_40000000 0x00_70000000 M:IRW- S:IRW- Peripherals + * 0x00_48700000 0x00_48780000 M:IRW- S:---- GCR cluster 0 + * 0x00_67480000 0x00_67500000 M:IRW- S:---- GCR cluster 1 + * 0x00_67500000 0x00_67580000 M:IRW- S:---- GCR cluster 2 + * 0x00_67800000 0x00_67900000 M:IRW- S:---- Ncore + * 0x00_70000000 0x00_80000000 M:---- S:IRW- PCI32 BARs, NOT USED - 32-bit mode + * 0x01_00000000 0x08_00000000 M:---- S:IRW- PCI64 BARs, NOT USED - PCI2PCI + * 0x08_00000000 0x10_00000000 M:---- S:-RWX DDR64 + * 0x10_00000000 0x20_00000000 M:---- S:IRW- PCI64 BARs + */ + rc = mips_p8700_add_memranges(); + if (rc) + return rc; + /* the rest of MMIO - shared with S-mode */ + rc = sbi_domain_root_add_memrange(MMIO_BASE, MMIO_SIZE, MMIO_SIZE, + SBI_DOMAIN_MEMREGION_MMIO | + SBI_DOMAIN_MEMREGION_SHARED_SURW_MRW); + if (rc) + return rc; + /* PCIE BARs - MMIO S-mode */ + rc = sbi_domain_root_add_memrange(0x1000000000UL, 0x1000000000UL, 0x1000000000UL, + SBI_DOMAIN_MEMREGION_MMIO | + SBI_DOMAIN_MEMREGION_SU_READABLE | + SBI_DOMAIN_MEMREGION_SU_WRITABLE); + + /* + * sbi_domain_init adds last "all-inclusive" memory region + * 0 .. ~0 RWX + * Find this region (it is the last one) and update size according to DRAM + */ + dom = sbi_domain_thishart_ptr(); + reg = find_last_memregion(dom); + return fixup_dram_region(dom, reg); +} + +static int eyeq7h_nascent_init(void) +{ + unsigned hartid = current_hartid(); + unsigned cl = cpu_cluster(hartid); + unsigned long cm_base = p8700_cm_info->gcr_base[cl]; + int i; + + /* Coherence enable for every core */ + if (cpu_hart(hartid) == 0) { + cm_base += (cpu_core(hartid) << CM_BASE_CORE_SHIFT); + __raw_writeq(GCR_CORE_COH_EN_EN, + (void *)(cm_base + GCR_OFF_LOCAL + + GCR_CORE_COH_EN)); + mb(); + } + + /** + * Boot code set PMP14 and PMP15 to allow basic cacheable and uncacheable + * access. + * To avoid hang during PMP count detection, set up PMP13 same as PMP14 + * Point is, PMP count detection procedure tries to write every PMP + * entry with maximum allowed value, then return original value. + * If memory covered only by PMP14, when it is written, next instruction + * fetch will fail. Use PMP13 as a back-up for PMP14. When PMP13 tested, + * PMP14 will serve the memory access; when PMP14 tested, PMP13 will + * provide memory access + */ + /* Set up pmp for DRAM */ + csr_write(CSR_PMPADDR13, DRAM_PMP_ADDR); + csr_write(CSR_PMPADDR14, DRAM_PMP_ADDR); + /* + * FIXME: for unknown reason if I copy PMP14 to PMP13 using + * csr_write(CSR_PMPADDR13, csr_read(CSR_PMPADDR14)); + * instead of writing as above, system hangs on late Linux boot + */ + /* All from 0x0 */ + csr_write(CSR_PMPADDR15, 0x1fffffffffffffff); + csr_write(CSR_PMPCFG2, ((PMP_A_NAPOT|PMP_R|PMP_W|PMP_X)<<56) | + ((PMP_A_NAPOT|PMP_R|PMP_W|PMP_X)<<48) | + ((PMP_A_NAPOT|PMP_R|PMP_W|PMP_X)<<40)); + /* Set cacheable for pmp13/pmp14, uncacheable for pmp15 */ + csr_write(CSR_MIPSPMACFG2, ((u64)CCA_CACHE_DISABLE << 56) | + ((u64)CCA_CACHE_ENABLE << 48) | + ((u64)CCA_CACHE_ENABLE << 40)); + /* Reset pmpcfg0 */ + csr_write(CSR_PMPCFG0, 0); + /* Reset pmacfg0 */ + csr_write(CSR_MIPSPMACFG0, 0); + mb(); + + /* Per cluster set up */ + if (cpu_core(hartid) == 0 && cpu_hart(hartid) == 0) { + /* Enable L2 prefetch */ + __raw_writel(0xfffff110, + (void *)(cm_base + L2_PFT_CONTROL_OFFSET)); + __raw_writel(0x15ff, + (void *)(cm_base + L2_PFT_CONTROL_B_OFFSET)); + /* + * Remove access to NCORE CSRs from mmio region 1 + * which is routed to AUX. NCORE to use default route through MEM. + */ + __raw_writeq(NCORE-1, (void *)(cm_base + GCR_MMIO_TOP(1))); + mb(); + mips_p8700_dump_mmio(); + } + + /* Per core set up */ + if (cpu_hart(hartid) == 0) { + /* Enable load pair, store pair, and HTW */ + csr_clear(CSR_MIPSCONFIG7, (1<<12)|(1<<13)|(1<<7)); + + /* Disable noRFO, misaligned load/store */ + csr_set(CSR_MIPSCONFIG7, (1<<25)|(1<<9)); + + /* Enable L1-D$ Prefetch */ + csr_write(CSR_MIPSCONFIG11, 0xff); + + for (i = 0; i < 8; i++) { + csr_set(CSR_MIPSCONFIG8, 4 + 0x100 * i); + csr_set(CSR_MIPSCONFIG9, 8); + mb(); + RISCV_FENCE_I; + } + } + + /* Per hart set up */ + /* Enable AMO and RDTIME illegal instruction exceptions. */ + csr_set(CSR_MIPSCONFIG6, (1<<2)|(1<<1)); + /* enable ECC for L1 I/D and FTLB */ + csr_set(CSR_MIPSERRCTL, MIPSERRCTL_PE); + + return 0; +} + +static int eyeq7h_platform_init(const void *fdt, int nodeoff, const struct fdt_match *match) +{ + int rc = mips_p8700_platform_init(fdt, nodeoff, match); + + if (rc) + return rc; + generic_platform_ops.early_init = eyeq7h_early_init; + generic_platform_ops.final_init = eyeq7h_final_init; + generic_platform_ops.nascent_init = eyeq7h_nascent_init; + generic_platform_ops.pmp_set = mips_p8700_pmp_set; + + return 0; +} + +static unsigned long eyeq7h_gcr_base[] = { + 0x48700000, + 0x67480000, + 0x67500000, +}; + +static struct p8700_cm_info eyeq7h_cm_info = { + .num_cm = array_size(eyeq7h_gcr_base), + .gcr_base = eyeq7h_gcr_base, +}; + +static const struct fdt_match eyeq7h_match[] = { + { .compatible = "mobileye,eyeq7h", .data = &eyeq7h_cm_info }, + { }, +}; + +const struct fdt_driver mips_p8700_eyeq7h = { + .match_table = eyeq7h_match, + .init = eyeq7h_platform_init, +}; diff --git a/platform/generic/mips/objects.mk b/platform/generic/mips/objects.mk index a08c4c63299..14524aadc12 100644 --- a/platform/generic/mips/objects.mk +++ b/platform/generic/mips/objects.mk @@ -3,6 +3,9 @@ # ifeq ($(PLATFORM_RISCV_XLEN), 64) -carray-platform_override_modules-$(CONFIG_PLATFORM_MIPS_P8700) += mips_p8700 -platform-objs-$(CONFIG_PLATFORM_MIPS_P8700) += mips/p8700.o mips/mips_warm_boot.o +carray-platform_override_modules-$(CONFIG_PLATFORM_MIPS_P8700_EYEQ7H) += mips_p8700_eyeq7h +carray-platform_override_modules-$(CONFIG_PLATFORM_MIPS_P8700_BOSTON) += mips_p8700_boston +platform-objs-$(CONFIG_CPU_MIPS_P8700) += mips/p8700.o mips/mips_warm_boot.o +platform-objs-$(CONFIG_PLATFORM_MIPS_P8700_EYEQ7H) += mips/eyeq7h.o +platform-objs-$(CONFIG_PLATFORM_MIPS_P8700_BOSTON) += mips/boston.o endif diff --git a/platform/generic/mips/p8700.c b/platform/generic/mips/p8700.c index a25610a1165..ff13d37bf1f 100644 --- a/platform/generic/mips/p8700.c +++ b/platform/generic/mips/p8700.c @@ -5,8 +5,6 @@ * */ -#include -#include #include #include #include @@ -16,11 +14,74 @@ #include #include -extern void mips_warm_boot(void); +const struct p8700_cm_info *p8700_cm_info; + +void mips_p8700_dump_mmio(void) +{ + struct sbi_scratch *scratch = sbi_scratch_thishart_ptr(); + unsigned hartid = current_hartid(); + unsigned cl = cpu_cluster(hartid); + unsigned long cm_base = p8700_cm_info->gcr_base[cl]; + u64 gcr_config = readq((void*)cm_base + GCR_GLOBAL_CONFIG); + int num_mmios = EXTRACT_FIELD(gcr_config, GCR_GC_NUM_MMIOS); + static const char *ports[16] = { + [0] = "MEM", + [1] = "? 1", + [2] = "? 2", + [3] = "? 3", + [4] = "? 4", + [5] = "? 5", + [6] = "? 6", + [7] = "? 7", + [8] = "AUX0", + [9] = "AUX1", + [10] = "AUX2", + [11] = "AUX3", + [12] = "? 12", + [13] = "? 13", + [14] = "? 14", + [15] = "? 15", + }; + static const char *ccas[4] = { + [0] = "ANY", + [1] = "UC", + [2] = "UCA", + [3] = "UC|UCA", + }; + static const char *ncs[2] = { + [0] = " ", + [1] = "NC", + }; + static const char *dis_rqs[2] = { + [0] = " ", + [1] = "DIS_RQ_LIM", + }; + if (!(scratch->options & SBI_SCRATCH_DEBUG_PRINTS)) + return; + + sbi_printf("Cluster %d: %d MMIO regions\n", cl, num_mmios); + for (int i = 0; i < num_mmios; i++) { + u64 b = readq((void*)cm_base + GCR_MMIO_BOTTOM(i)); + u64 t = readq((void*)cm_base + GCR_MMIO_TOP(i)); + if (b & GCR_MMIO_BOTTOM_EN) { + ulong ta = EXTRACT_FIELD(t, GCR_MMIO_ADDR) << 16; + ulong ba = EXTRACT_FIELD(b, GCR_MMIO_ADDR) << 16; + int cca = EXTRACT_FIELD(b, GCR_MMIO_BOTTOM_CCA); + int nc = EXTRACT_FIELD(b, GCR_MMIO_BOTTOM_FORCE_NC); + int port = EXTRACT_FIELD(b, GCR_MMIO_BOTTOM_PORT); + int dis_rq_lim = EXTRACT_FIELD(b, GCR_MMIO_BOTTOM_DIS_RQ_LIM); + + sbi_printf(" [%d] : 0x%016lx-0x%016lx %4s %6s %s %s\n", i, + ba, ta, ports[port], ccas[cca], ncs[nc], dis_rqs[dis_rq_lim]); + } else { + sbi_printf(" [%d] : --disabled--\n", i); + } + } +} -static void mips_p8700_pmp_set(unsigned int n, unsigned long flags, - unsigned long prot, unsigned long addr, - unsigned long log2len) +void mips_p8700_pmp_set(unsigned int n, unsigned long flags, + unsigned long prot, unsigned long addr, + unsigned long log2len) { int pmacfg_csr, pmacfg_shift; unsigned long cfgmask; @@ -38,137 +99,187 @@ static void mips_p8700_pmp_set(unsigned int n, unsigned long flags, csr_write_num(pmacfg_csr, pmacfg); } -#if CLUSTERS_IN_PLATFORM > 1 -static void power_up_other_cluster(u32 hartid) +static void mips_p8700_sync_hrtimer(unsigned int cl) { - unsigned int stat; - unsigned int timeout; - bool local_p = (cpu_cluster(current_hartid()) == cpu_cluster(hartid)); + u64 v1, v2, mv, delta; + volatile u64 *my_timer = (volatile u64 *)(p8700_cm_info->gcr_base[cl] + CPC_OFFSET + CPC_HRTIME); + volatile u64 *ref_timer = (volatile u64 *)(p8700_cm_info->gcr_base[0] + CPC_OFFSET + CPC_HRTIME); + + v1 = readq_relaxed(my_timer); + mv = readq_relaxed(ref_timer); + v2 = readq_relaxed(my_timer); + delta = mv - ((v1 / 2) + (v2 / 2)); + writeq_relaxed(readq_relaxed(my_timer) + delta, my_timer); +} - /* Power up cluster cl core 0 hart 0 */ - write_cpc_pwrup_ctl(hartid, 1, local_p); +void mips_p8700_power_up_other_cluster(u32 hartid) +{ + unsigned int cl = cpu_cluster(hartid); + + /* Power up CM in cluster */ + write_cpc_pwrup_ctl(hartid, 1); /* Wait for the CM to start up */ - timeout = 100; - while (true) { - stat = read_cpc_cm_stat_conf(hartid, local_p); - stat = EXT(stat, CPC_Cx_STAT_CONF_SEQ_STATE); - if (stat == CPC_Cx_STAT_CONF_SEQ_STATE_U5) - break; - - /* Delay a little while before we start warning */ - if (timeout) { - sbi_dprintf("Delay a little while before we start warning\n"); - timeout--; - } - else { - sbi_printf("Waiting for cluster %u CM to power up... STAT_CONF=0x%x\n", - cpu_cluster(hartid), stat); - break; + for (int i = 100; i > 0; i--) { + u32 stat = read_cpc_cm_stat_conf(hartid); + + stat = EXTRACT_FIELD(stat, CPC_Cx_STAT_CONF_SEQ_STATE); + if (stat == CPC_Cx_STAT_CONF_SEQ_STATE_U5) { + if (cl) /* sync high-res timer to cluster 0 */ + mips_p8700_sync_hrtimer(cl); + return; } + cpu_relax(); } + sbi_printf("ERROR: Fail to power up cluster %u\n", cl); } -#endif -static int mips_hart_start(u32 hartid, ulong saddr) +extern void mips_warm_boot(void); + +struct mips_boot_params { + u32 hartid; + u32 target_state; +}; + +static bool mips_hart_reached_state(void *arg) { - unsigned int stat; - unsigned int timeout; - bool local_p = (cpu_cluster(current_hartid()) == cpu_cluster(hartid)); + struct mips_boot_params *p = arg; + u32 stat = read_cpc_co_stat_conf(p->hartid); + stat = EXTRACT_FIELD(stat, CPC_Cx_STAT_CONF_SEQ_STATE); + return stat == p->target_state; +} + +int mips_p8700_hart_start(u32 hartid, ulong saddr) +{ /* Hart 0 is the boot hart, and we don't use the CPC cmd to start. */ if (hartid == 0) return SBI_ENOTSUPP; /* Change reset base to mips_warm_boot */ - write_gcr_co_reset_base(hartid, (unsigned long)mips_warm_boot, local_p); + write_gcr_co_reset_base(hartid, (unsigned long)mips_warm_boot); if (cpu_hart(hartid) == 0) { + unsigned int const timeout_ms = 10; + bool booted; + struct mips_boot_params p = { + .hartid = hartid, + .target_state = CPC_Cx_STAT_CONF_SEQ_STATE_U6, + }; + /* Ensure its coherency is disabled */ - write_gcr_co_coherence(hartid, 0, local_p); + write_gcr_co_coherence(hartid, 0); /* Start cluster cl core co hart 0 */ - write_cpc_co_vp_run(hartid, 1 << cpu_hart(hartid), local_p); + write_cpc_co_vp_run(hartid, 1 << cpu_hart(hartid)); /* Reset cluster cl core co hart 0 */ - write_cpc_co_cmd(hartid, CPC_Cx_CMD_RESET, local_p); - - timeout = 100; - while (true) { - stat = read_cpc_co_stat_conf(hartid, local_p); - stat = EXT(stat, CPC_Cx_STAT_CONF_SEQ_STATE); - if (stat == CPC_Cx_STAT_CONF_SEQ_STATE_U6) - break; - - /* Delay a little while before we start warning */ - if (timeout) { - sbi_timer_mdelay(10); - timeout--; - } - else { - sbi_printf("Waiting for cluster %u core %u hart %u to start... STAT_CONF=0x%x\n", - cpu_cluster(hartid), - cpu_core(hartid), cpu_hart(hartid), - stat); - break; - } + write_cpc_co_cmd(hartid, CPC_Cx_CMD_RESET); + + booted = sbi_timer_waitms_until(mips_hart_reached_state, &p, timeout_ms); + if (!booted) { + sbi_printf("ERROR: failed to boot hart 0x%x in %d ms\n", + hartid, timeout_ms); + return -SBI_ETIMEDOUT; } - } - else { - write_cpc_co_vp_run(hartid, 1 << cpu_hart(hartid), local_p); + } else { + write_cpc_co_vp_run(hartid, 1 << cpu_hart(hartid)); } return 0; } -static int mips_hart_stop() +int mips_p8700_hart_stop() { u32 hartid = current_hartid(); - bool local_p = (cpu_cluster(current_hartid()) == cpu_cluster(hartid)); /* Hart 0 is the boot hart, and we don't use the CPC cmd to stop. */ if (hartid == 0) return SBI_ENOTSUPP; - write_cpc_co_vp_stop(hartid, 1 << cpu_hart(hartid), local_p); + write_cpc_co_vp_stop(hartid, 1 << cpu_hart(hartid)); return 0; } -static const struct sbi_hsm_device mips_hsm = { - .name = "mips_hsm", - .hart_start = mips_hart_start, - .hart_stop = mips_hart_stop, -}; - -static int mips_p8700_final_init(bool cold_boot) +void mips_p8700_cache_info(struct p8700_cache_info *l1d, struct p8700_cache_info *l1i, + struct p8700_cache_info *l2) { - if (cold_boot) - sbi_hsm_set_device(&mips_hsm); + u32 mipsconfig1 = csr_read(CSR_MIPSCONFIG1); - return generic_final_init(cold_boot); + if (l1d) { + u32 da = EXTRACT_FIELD(mipsconfig1, MIPSCONFIG1_DA); + u32 dl = EXTRACT_FIELD(mipsconfig1, MIPSCONFIG1_DL); + u32 ds = EXTRACT_FIELD(mipsconfig1, MIPSCONFIG1_DS); + + l1d->line = dl ? 1 << (dl + 1) : 0; + l1d->assoc_ways = da +1; + l1d->sets = ds == 7 ? 32 : 1 << (ds + 6); + } + if (l1i) { + u32 ia = EXTRACT_FIELD(mipsconfig1, MIPSCONFIG1_IA); + u32 il = EXTRACT_FIELD(mipsconfig1, MIPSCONFIG1_IL); + u32 is = EXTRACT_FIELD(mipsconfig1, MIPSCONFIG1_IS); + + l1i->line = il ? 1 << (il + 1) : 0; + l1i->assoc_ways = ia +1; + l1i->sets = is == 7 ? 32 : 1 << (is + 6); + } + if (l2) { + if (mipsconfig1 & MIPSCONFIG1_L2C) { + void *cm_base = (void *)p8700_cm_info->gcr_base[0]; + u32 l2_config = readl(cm_base + GCR_L2_CONFIG); + + if (l2_config & GCR_L2_REG_EXISTS) { + u32 l2a = EXTRACT_FIELD(l2_config, GCR_L2_ASSOC); + u32 l2l = EXTRACT_FIELD(l2_config, GCR_L2_LINE_SIZE); + u32 l2s = EXTRACT_FIELD(l2_config, GCR_L2_SET_SIZE); + + l2->assoc_ways = l2a + 1; + l2->line = 1 << (l2l + 1); + l2->sets = 1 << (l2s + 6); + return; + } + } + l2->line = 0; + l2->assoc_ways = 0; + l2->sets = 0; + } } -static int mips_p8700_early_init(bool cold_boot) +/** + * See CPU cluster memory map in the table below + * To save PMP regions, group areas with M mode access, marked (1) and (2) + * + * GCR_BASE offset | | | Block Name | Description + * 0x00000 - 0x01FFF | M | ^ | GCR.Global | Per-cluster CM registers. + * 0x02000 - 0x05FFF | M | | | GCR.Core | Per-core CM registers. + * 0x06000 - 0x07FFF | - |(1)| Reserved. + * 0x08000 - 0x09FFF | M | | | CPC.Global | Per-cluster CPC registers. + * 0x0A000 - 0x0EFFF | M | | | CPC.Core | Per-core/Per-device CPC registers. + * 0x0F000 - 0x0FFFF | - | v | Reserved. + * 0x10000 - 0x1FFFF | S | | uGCR | Reserved for user defined CM registers. + * 0x20000 - 0x3EFFF | - | | Reserved. + * 0x3F000 - 0x3F0FF | ? | | FDC.Global | FDC.Global registers. + * 0x3F100 - 0x3FFFF | ? | | TRF.Global | TRF.Global registers + * 0x40000 - 0x4BFFF | M | ^ | APLIC.M | APLIC Machine registers. + * 0x4C000 - 0x4CFFF | M |(2)| APLIC.custom | APLIC custom registers. + * 0x4D000 - 0x4FFFF | - | | | Reserved. + * 0x50000 - 0x5FFFF | M | v | ACLINT.M | ACLINT Machine registers. + * 0x60000 - 0x6BFFF | S | | APLIC.S | APLIC Supervisor registers. + * 0x6C000 - 0x6FFFF | S | | ACLINT.S | ACLINT Supervisor registers. + * 0x70000 - 0x7EFFF | - | | Reserved. + * 0x7F000 - 0x7FFFF | S | | GCR.U | User Mode GCRs. + */ +int mips_p8700_add_memranges(void) { - int rc; - - rc = generic_early_init(cold_boot); - if (rc) - return rc; - - if (cold_boot) { -#if CLUSTERS_IN_PLATFORM > 1 - int i; - /* Power up other clusters in the platform. */ - for (i = 1; i < CLUSTERS_IN_PLATFORM; i++) { - power_up_other_cluster(i << NEW_CLUSTER_SHIFT); - } -#endif + int rc = SBI_OK; + for (int i = 0; i < p8700_cm_info->num_cm; i++) { + unsigned long cm_base = p8700_cm_info->gcr_base[i]; - /* For the CPC mtime region, the minimum size is 0x10000. */ - rc = sbi_domain_root_add_memrange(CM_BASE, SIZE_FOR_CPC_MTIME, - P8700_ALIGN, + /* CM and MTIMER */ + rc = sbi_domain_root_add_memrange(cm_base, SIZE_FOR_CPC_MTIME, + SIZE_FOR_CPC_MTIME, (SBI_DOMAIN_MEMREGION_MMIO | SBI_DOMAIN_MEMREGION_M_READABLE | SBI_DOMAIN_MEMREGION_M_WRITABLE)); @@ -176,121 +287,26 @@ static int mips_p8700_early_init(bool cold_boot) return rc; /* For the APLIC and ACLINT m-mode region */ - rc = sbi_domain_root_add_memrange(AIA_BASE, SIZE_FOR_AIA_M_MODE, - P8700_ALIGN, + rc = sbi_domain_root_add_memrange(cm_base + AIA_OFFSET, SIZE_FOR_AIA_M_MODE, + SIZE_FOR_AIA_M_MODE, (SBI_DOMAIN_MEMREGION_MMIO | SBI_DOMAIN_MEMREGION_M_READABLE | SBI_DOMAIN_MEMREGION_M_WRITABLE)); if (rc) return rc; - -#if CLUSTERS_IN_PLATFORM > 1 - for (i = 0; i < CLUSTERS_IN_PLATFORM; i++) { - /* For the CPC mtime region, the minimum size is 0x10000. */ - rc = sbi_domain_root_add_memrange(GLOBAL_CM_BASE[i], SIZE_FOR_CPC_MTIME, - P8700_ALIGN, - (SBI_DOMAIN_MEMREGION_MMIO | - SBI_DOMAIN_MEMREGION_M_READABLE | - SBI_DOMAIN_MEMREGION_M_WRITABLE)); - if (rc) - return rc; - - /* For the APLIC and ACLINT m-mode region */ - rc = sbi_domain_root_add_memrange(AIA_BASE - CM_BASE + GLOBAL_CM_BASE[i], SIZE_FOR_AIA_M_MODE, - P8700_ALIGN, - (SBI_DOMAIN_MEMREGION_MMIO | - SBI_DOMAIN_MEMREGION_M_READABLE | - SBI_DOMAIN_MEMREGION_M_WRITABLE)); - if (rc) - return rc; - } -#endif } - - return 0; + return rc; } -static int mips_p8700_nascent_init(void) +int mips_p8700_platform_init(const void *fdt, int nodeoff, const struct fdt_match *match) { - u64 hartid = current_hartid(); - u64 cm_base = CM_BASE; - int i; + const struct p8700_cm_info *data = match->data; - /* Coherence enable for every core */ - if (cpu_hart(hartid) == 0) { - cm_base += (cpu_core(hartid) << CM_BASE_CORE_SHIFT); - __raw_writeq(GCR_CORE_COH_EN_EN, - (void *)(cm_base + GCR_OFF_LOCAL + - GCR_CORE_COH_EN)); - mb(); - } - - /* Set up pmp for DRAM */ - csr_write(CSR_PMPADDR14, DRAM_PMP_ADDR); - /* All from 0x0 */ - csr_write(CSR_PMPADDR15, 0x1fffffffffffffff); - csr_write(CSR_PMPCFG2, ((PMP_A_NAPOT|PMP_R|PMP_W|PMP_X)<<56)| - ((PMP_A_NAPOT|PMP_R|PMP_W|PMP_X)<<48)); - /* Set cacheable for pmp6, uncacheable for pmp7 */ - csr_write(CSR_MIPSPMACFG2, ((u64)CCA_CACHE_DISABLE << 56)| - ((u64)CCA_CACHE_ENABLE << 48)); - /* Reset pmpcfg0 */ - csr_write(CSR_PMPCFG0, 0); - /* Reset pmacfg0 */ - csr_write(CSR_MIPSPMACFG0, 0); - mb(); - - /* Per cluster set up */ - if (cpu_core(hartid) == 0 && cpu_hart(hartid) == 0) { - /* Enable L2 prefetch */ - __raw_writel(0xfffff110, - (void *)(cm_base + L2_PFT_CONTROL_OFFSET)); - __raw_writel(0x15ff, - (void *)(cm_base + L2_PFT_CONTROL_B_OFFSET)); + if (!data) { + sbi_printf("Missing CM info for %s\n", match->compatible); + return SBI_EINVAL; } - /* Per core set up */ - if (cpu_hart(hartid) == 0) { - /* Enable load pair, store pair, and HTW */ - csr_clear(CSR_MIPSCONFIG7, (1<<12)|(1<<13)|(1<<7)); - - /* Disable noRFO, misaligned load/store */ - csr_set(CSR_MIPSCONFIG7, (1<<25)|(1<<9)); - - /* Enable L1-D$ Prefetch */ - csr_write(CSR_MIPSCONFIG11, 0xff); - - for (i = 0; i < 8; i++) { - csr_set(CSR_MIPSCONFIG8, 4 + 0x100 * i); - csr_set(CSR_MIPSCONFIG9, 8); - mb(); - RISCV_FENCE_I; - } - } - - /* Per hart set up */ - /* Enable AMO and RDTIME illegal instruction exceptions. */ - csr_set(CSR_MIPSCONFIG6, (1<<2)|(1<<1)); - - return 0; -} - -static int mips_p8700_platform_init(const void *fdt, int nodeoff, const struct fdt_match *match) -{ - generic_platform_ops.early_init = mips_p8700_early_init; - generic_platform_ops.final_init = mips_p8700_final_init; - generic_platform_ops.nascent_init = mips_p8700_nascent_init; - generic_platform_ops.pmp_set = mips_p8700_pmp_set; - - return 0; + p8700_cm_info = data; + return SBI_OK; } - -static const struct fdt_match mips_p8700_match[] = { - { .compatible = "mips,p8700" }, - { }, -}; - -const struct fdt_driver mips_p8700 = { - .match_table = mips_p8700_match, - .init = mips_p8700_platform_init, -}; diff --git a/platform/generic/objects.mk b/platform/generic/objects.mk index c4a8fee2d2f..ca7fb8b7a3e 100644 --- a/platform/generic/objects.mk +++ b/platform/generic/objects.mk @@ -35,9 +35,9 @@ FW_JUMP_FDT_OFFSET=0x2200000 FW_PAYLOAD=y ifeq ($(PLATFORM_RISCV_XLEN), 32) # This needs to be 4MB aligned for 32-bit system - FW_PAYLOAD_OFFSET=0x400000 + FW_PAYLOAD_ALIGN=0x400000 else # This needs to be 2MB aligned for 64-bit system - FW_PAYLOAD_OFFSET=0x200000 + FW_PAYLOAD_ALIGN=0x200000 endif FW_PAYLOAD_FDT_OFFSET=$(FW_JUMP_FDT_OFFSET) diff --git a/platform/generic/openhwgroup/ariane.c b/platform/generic/openhwgroup/ariane.c index 70a97f38360..ce2f5512b8e 100644 --- a/platform/generic/openhwgroup/ariane.c +++ b/platform/generic/openhwgroup/ariane.c @@ -24,6 +24,7 @@ CLINT_MTIMER_OFFSET) static struct plic_data plic = { + .unique_id = 0, .addr = ARIANE_PLIC_ADDR, .size = ARIANE_PLIC_SIZE, .num_src = ARIANE_PLIC_NUM_SOURCES, diff --git a/platform/generic/openhwgroup/openpiton.c b/platform/generic/openhwgroup/openpiton.c index 60d719e15ce..1d098da7561 100644 --- a/platform/generic/openhwgroup/openpiton.c +++ b/platform/generic/openhwgroup/openpiton.c @@ -23,6 +23,7 @@ (OPENPITON_DEFAULT_CLINT_ADDR + CLINT_MTIMER_OFFSET) static struct plic_data plic = { + .unique_id = 0, .addr = (unsigned long)OPENPITON_DEFAULT_PLIC_ADDR, .size = OPENPITON_DEFAULT_PLIC_SIZE, .num_src = OPENPITON_DEFAULT_PLIC_NUM_SOURCES, diff --git a/platform/generic/platform.c b/platform/generic/platform.c index 01fa03e0da4..1df0280da61 100644 --- a/platform/generic/platform.c +++ b/platform/generic/platform.c @@ -253,11 +253,13 @@ int generic_final_init(bool cold_boot) return 0; } -int generic_extensions_init(struct sbi_hart_features *hfeatures) +int generic_extensions_init(bool cold_boot) { + if (!cold_boot) + return 0; + /* Parse the ISA string from FDT and enable the listed extensions */ - return fdt_parse_isa_extensions(fdt_get_address(), current_hartid(), - hfeatures->extensions); + return fdt_parse_isa_extensions_all_harts(fdt_get_address()); } int generic_domains_init(void) diff --git a/platform/generic/sifive/objects.mk b/platform/generic/sifive/objects.mk index d75e444abf5..d32e1273dec 100644 --- a/platform/generic/sifive/objects.mk +++ b/platform/generic/sifive/objects.mk @@ -2,6 +2,9 @@ # SPDX-License-Identifier: BSD-2-Clause # +carray-platform_override_modules-$(CONFIG_PLATFORM_SIFIVE_DEV) += sifive_dev_platform +platform-objs-$(CONFIG_PLATFORM_SIFIVE_DEV) += sifive/sifive_dev_platform.o + carray-platform_override_modules-$(CONFIG_PLATFORM_SIFIVE_FU540) += sifive_fu540 platform-objs-$(CONFIG_PLATFORM_SIFIVE_FU540) += sifive/fu540.o diff --git a/platform/generic/sifive/sifive_dev_platform.c b/platform/generic/sifive/sifive_dev_platform.c new file mode 100644 index 00000000000..579d46516bb --- /dev/null +++ b/platform/generic/sifive/sifive_dev_platform.c @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 SiFive Inc. + */ + +#include +#include +#include + +/* + * Instead of allocating individual PMP entries for each M-mode + * MMIO driver, configure SMEPMP regions to grant R/W permission + * to peripheral and system ports for all privilege modes. + * Finer-grained access control is enforced by wgChecker rules. + */ +static int sifive_smepmp_setup(void) +{ + struct sbi_scratch *scratch = sbi_scratch_thishart_ptr(); + int rc; + + if (!sbi_hart_has_extension(scratch, SBI_HART_EXT_SMEPMP)) + return 0; + + /* Peripheral port region: 0 to 2GiB */ + rc = sbi_domain_root_add_memrange(0x0, 0x80000000, 0x80000000, + SBI_DOMAIN_MEMREGION_MMIO | + SBI_DOMAIN_MEMREGION_SHARED_SURW_MRW); + if (rc) + return rc; + +#if __riscv_xlen > 32 + /* System port region: 512GiB to 1TiB */ + rc = sbi_domain_root_add_memrange(0x8000000000ULL, 0x8000000000ULL, 0x8000000000ULL, + SBI_DOMAIN_MEMREGION_MMIO | + SBI_DOMAIN_MEMREGION_SHARED_SURW_MRW); + if (rc) + return rc; +#endif + + return 0; +} + +static int sifive_early_init(bool cold_boot) +{ + int rc; + + if (cold_boot) { + rc = sifive_smepmp_setup(); + if (rc) + return rc; + } + + return generic_early_init(cold_boot); +} + +static int sifive_platform_init(const void *fdt, int nodeoff, + const struct fdt_match *match) +{ + generic_platform_ops.early_init = sifive_early_init; + + return 0; +} + +static const struct fdt_match sifive_dev_platform_match[] = { + { .compatible = "sifive-dev" }, + { }, +}; + +const struct fdt_driver sifive_dev_platform = { + .match_table = sifive_dev_platform_match, + .init = sifive_platform_init, +}; diff --git a/platform/generic/sophgo/sg2042.c b/platform/generic/sophgo/sg2042.c index c2447c3c2df..ac8840e8520 100644 --- a/platform/generic/sophgo/sg2042.c +++ b/platform/generic/sophgo/sg2042.c @@ -49,11 +49,11 @@ static int sophgo_sg2042_early_init(bool cold_boot) return 0; } -static int sophgo_sg2042_extensions_init(struct sbi_hart_features *hfeatures) +static int sophgo_sg2042_extensions_init(bool cold_boot) { int rc; - rc = generic_extensions_init(hfeatures); + rc = generic_extensions_init(cold_boot); if (rc) return rc; diff --git a/platform/generic/starfive/jh7110.c b/platform/generic/starfive/jh7110.c index c1328437eb1..7008386a0be 100644 --- a/platform/generic/starfive/jh7110.c +++ b/platform/generic/starfive/jh7110.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include diff --git a/platform/generic/tenstorrent/Kconfig b/platform/generic/tenstorrent/Kconfig new file mode 100644 index 00000000000..76c7fb329da --- /dev/null +++ b/platform/generic/tenstorrent/Kconfig @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: (c) 2025-2026 Tenstorrent USA, Inc. +# SPDX-License-Identifier: BSD-2-Clause + +config CPU_TENSTORRENT_ASCALON + bool diff --git a/platform/generic/tenstorrent/ascalon.c b/platform/generic/tenstorrent/ascalon.c new file mode 100644 index 00000000000..485144cdbf5 --- /dev/null +++ b/platform/generic/tenstorrent/ascalon.c @@ -0,0 +1,53 @@ +/* + * SPDX-FileCopyrightText: (c) 2025-2026 Tenstorrent USA, Inc. + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include + +#include +#include + +#define CSR_PMACFG0 0x7e0 + +void tt_ascalon_discover_pmas_from_boot_hart(void) +{ + struct sbi_trap_info trap = {0}; + + /* Whisper virtual platform does not implement PMA */ + csr_read_allowed(CSR_PMACFG0, &trap); + if (trap.cause) + return; + + for (unsigned int i = 0; i < TT_MAX_PMAS; i++) { + u64 pma = csr_read_num(CSR_PMACFG0 + i); + if (!tt_pma_validate(i, pma)) { + sbi_printf("HART%d: Bad boot PMA%02d 0x%016lx\n", + current_hartid(), i, pma); + } + tt_pma_set(i, pma); + + if (pma) + tt_pma_print(i, pma); + } +} + +void tt_ascalon_verify_pmas_nonboot_hart(void) +{ + struct sbi_trap_info trap = {0}; + + /* Whisper virtual platform does not implement PMA */ + csr_read_allowed(CSR_PMACFG0, &trap); + if (trap.cause) + return; + + for (unsigned int i = 0; i < TT_MAX_PMAS; i++) { + u64 pma = csr_read_num(CSR_PMACFG0 + i); + if (pma != tt_pma_get(i)) { + sbi_printf("HART%d: Bad boot PMA%02d 0x%016lx does not match boot HART\n", + current_hartid(), i, pma); + } + } +} diff --git a/platform/generic/tenstorrent/atlantis.c b/platform/generic/tenstorrent/atlantis.c new file mode 100644 index 00000000000..4c312f7ef2d --- /dev/null +++ b/platform/generic/tenstorrent/atlantis.c @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: (c) 2025-2026 Tenstorrent USA, Inc. + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +static int tt_atlantis_final_init(bool cold_boot) +{ + if (cold_boot) { + /* Boot firmware sets HART PMAs. Read and verify them. */ + tt_ascalon_discover_pmas_from_boot_hart(); + } else { + /* Verify nonboot HARTs have PMAs matching boot HART */ + tt_ascalon_verify_pmas_nonboot_hart(); + } + + return generic_final_init(cold_boot); +} + +static bool tt_atlantis_single_fw_region(void) +{ + return true; +} + +static int tt_atlantis_platform_init(const void *fdt, int nodeoff, const struct fdt_match *match) +{ + generic_platform_ops.final_init = tt_atlantis_final_init; + generic_platform_ops.single_fw_region = tt_atlantis_single_fw_region; + + return 0; +} + +static const struct fdt_match tt_atlantis_match[] = { + { .compatible = "tenstorrent,atlantis" }, + { }, +}; + +const struct fdt_driver tenstorrent_atlantis = { + .match_table = tt_atlantis_match, + .init = tt_atlantis_platform_init, +}; diff --git a/platform/generic/tenstorrent/objects.mk b/platform/generic/tenstorrent/objects.mk new file mode 100644 index 00000000000..17da2c5e2d7 --- /dev/null +++ b/platform/generic/tenstorrent/objects.mk @@ -0,0 +1,12 @@ +# +# SPDX-FileCopyrightText: (c) 2025-2026 Tenstorrent USA, Inc. +# SPDX-License-Identifier: BSD-2-Clause +# + +ifeq ($(PLATFORM_RISCV_XLEN), 64) +platform-objs-y += tenstorrent/pma.o +platform-objs-$(CONFIG_CPU_TENSTORRENT_ASCALON) += tenstorrent/ascalon.o + +carray-platform_override_modules-$(CONFIG_PLATFORM_TENSTORRENT_ATLANTIS) += tenstorrent_atlantis +platform-objs-$(CONFIG_PLATFORM_TENSTORRENT_ATLANTIS) += tenstorrent/atlantis.o +endif diff --git a/platform/generic/tenstorrent/pma.c b/platform/generic/tenstorrent/pma.c new file mode 100644 index 00000000000..daf60192bfc --- /dev/null +++ b/platform/generic/tenstorrent/pma.c @@ -0,0 +1,138 @@ +/* + * SPDX-FileCopyrightText: (c) 2025-2026 Tenstorrent USA, Inc. + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include + +#include + +/* + * All PMAs in the system should be the same (after boot). The init code + * must have set PMAs for all HARTs. + */ + +/* + * Ascalon CPU and IOMMU PMA layout: + * Field + * [2:0] Permission [0] Read, [1] Write, [2] Execute + * [4:3] Memory type 00: Main memory, 01: IO memory relaxed, + * 10: IO memory channel 0, 11: IO memory channel 1 + * [6:5] AMO type 00: AMONone, 01: AMOSwap, + * 10: AMOLogical, 11: AMOArithmetic + * [7] Cacheability (main memory type) + * 1: Cacheable, 0: Non-cacheable + * Combining Capability (IO memory type) + * 1: Combining allowed, 0: Combining disallowed + * [8] Routing (coherency) + * 1: Coherent network, 0: Non-coherent network + * [11:9] Reserved + * [51:12] Physical address [51:12] base + * [63:58] Size log 2 (number of address LSB to ignore when matching) + * 0 = invalid entry (no match) + */ + +#define PMA_PERMISSION_R 0x1 +#define PMA_PERMISSION_W 0x2 +#define PMA_PERMISSION_X 0x4 +#define PMA_PERMISSION_MASK 0x7 + +#define PMA_TYPE_MAIN_MEMORY 0x0 +#define PMA_TYPE_IO_RELAXED 0x8 +#define PMA_TYPE_IO_ORDERED_0 0x10 +#define PMA_TYPE_IO_ORDERED_1 0x18 +#define PMA_TYPE_MASK 0x18 + +#define PMA_AMO_NONE 0x0 +#define PMA_AMO_SWAP 0x20 +#define PMA_AMO_LOGICAL 0x40 +#define PMA_AMO_ARITHMETIC 0x60 +#define PMA_AMO_MASK 0x60 + +#define PMA_MEMORY_CACHEABLE 0x80 +#define PMA_IO_COMBINING 0x80 +#define PMA_ROUTING_COHERENT 0x100 + +#define PMA_FLAGS_MASK 0x00000000000001ffULL +#define PMA_ADDRESS_MASK 0x000ffffffffff000ULL +#define PMA_SIZE_MASK 0xfc00000000000000ULL +#define PMA_RESERVED_MASK 0x0300000000000e00ULL + +#define PMA_SIZE_SHIFT 58 + +static u64 tt_pma_size(u64 pma) +{ + if ((pma & PMA_SIZE_MASK) == 0) + return 0; + + return 1ULL << ((pma & PMA_SIZE_MASK) >> PMA_SIZE_SHIFT); +} + +static u64 tt_pma_address(u64 pma) +{ + return (pma & PMA_ADDRESS_MASK) & ~((tt_pma_size(pma) - 1)); +} + +bool tt_pma_validate(unsigned int i, u64 pma) +{ + if (!pma) + return true; + + if (pma & PMA_RESERVED_MASK) { + sbi_printf("PMA%02u 0x%016lx contains reserved bits\n", i, pma); + return false; + } + + if (tt_pma_size(pma) < 4096) { + sbi_printf("PMA%02u 0x%016lx size < 4KB\n", i, pma); + return false; + } + + if (tt_pma_address(pma) != (pma & PMA_ADDRESS_MASK)) { + sbi_printf("PMA%02u 0x%016lx address is not aligned to size\n", i, pma); + return false; + } + + return true; +} + +void tt_pma_print(unsigned int i, u64 pma) +{ + sbi_printf("PMA%02d : 0x%016lx-0x%016lx perm:%s%s%s type:%s %s %s amo:%s\n", i, + tt_pma_address(pma), tt_pma_address(pma) + tt_pma_size(pma) - 1, + pma & PMA_PERMISSION_R ? "R" : " ", + pma & PMA_PERMISSION_W ? "W" : " ", + pma & PMA_PERMISSION_X ? "X" : " ", + (pma & PMA_TYPE_MASK) == PMA_TYPE_MAIN_MEMORY ? "main-memory" : + ((pma & PMA_TYPE_MASK) == PMA_TYPE_IO_RELAXED ? "io-relaxed" : + ((pma & PMA_TYPE_MASK) == PMA_TYPE_IO_ORDERED_0 ? "io-ordered-0" : "io-ordered-1")), + (pma & PMA_TYPE_MASK) == PMA_TYPE_MAIN_MEMORY ? + (pma & PMA_MEMORY_CACHEABLE ? "cacheable" : "non-cacheable") : + (pma & PMA_IO_COMBINING ? "combining" : "non-combining"), + pma & PMA_ROUTING_COHERENT ? "coherent" : "non-coherent", + (pma & PMA_AMO_MASK) == PMA_AMO_NONE ? "none" : + ((pma & PMA_AMO_MASK) == PMA_AMO_SWAP ? "swap" : + ((pma & PMA_AMO_MASK) == PMA_AMO_LOGICAL ? "logical" : "arithmetic"))); +} + +static u64 pmas[TT_MAX_PMAS]; + +void tt_pma_set(unsigned int n, u64 pma) +{ + if (n >= TT_MAX_PMAS) + sbi_panic("PMA exceeded TT_MAX_PMAS"); + + pmas[n] = pma; +} + +u64 tt_pma_get(unsigned int n) +{ + if (n >= TT_MAX_PMAS) + sbi_panic("PMA exceeded TT_MAX_PMAS"); + + return pmas[n]; +} diff --git a/platform/generic/thead/thead-generic.c b/platform/generic/thead/thead-generic.c index ddb4f0bf425..ba7a2290487 100644 --- a/platform/generic/thead/thead-generic.c +++ b/platform/generic/thead/thead-generic.c @@ -13,6 +13,7 @@ #include #include #include +#include #include struct thead_generic_quirks { @@ -26,11 +27,11 @@ static int thead_tlb_flush_early_init(bool cold_boot) return generic_early_init(cold_boot); } -static int thead_pmu_extensions_init(struct sbi_hart_features *hfeatures) +static int thead_pmu_extensions_init(bool cold_boot) { int rc; - rc = generic_extensions_init(hfeatures); + rc = generic_extensions_init(cold_boot); if (rc) return rc; @@ -39,6 +40,32 @@ static int thead_pmu_extensions_init(struct sbi_hart_features *hfeatures) return 0; } +static void thead_jtlb_local_sfence_vma(struct sbi_tlb_info *tinfo) +{ + __asm__ __volatile__("sfence.vma"); +} + +static void thead_jtlb_local_hfence_vvma_asid(struct sbi_tlb_info *tinfo) +{ + unsigned long start = tinfo->start; + unsigned long size = tinfo->size; + unsigned long asid = tinfo->asid; + + /* Flush entire MM context for a given ASID */ + if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) { + __asm__ __volatile__("sfence.vma x0, %0" + : + : "r"(asid) + : "memory"); + return; + } + + __asm__ __volatile__("sfence.vma x0, %0" + : + : "r"(asid) + : "memory"); +} + static int thead_generic_platform_init(const void *fdt, int nodeoff, const struct fdt_match *match) { @@ -49,6 +76,11 @@ static int thead_generic_platform_init(const void *fdt, int nodeoff, if (quirks->errata & THEAD_QUIRK_ERRATA_THEAD_PMU) generic_platform_ops.extensions_init = thead_pmu_extensions_init; + if (quirks->errata &THEAD_QUIRK_ERRATA_JTLB) { + generic_platform_ops.local_sfence_vma = thead_jtlb_local_sfence_vma; + generic_platform_ops.local_sfence_vma_asid = thead_jtlb_local_hfence_vvma_asid; + } + return 0; } @@ -60,13 +92,17 @@ static const struct thead_generic_quirks thead_pmu_quirks = { .errata = THEAD_QUIRK_ERRATA_THEAD_PMU, }; +static const struct thead_generic_quirks thead_pmu_jtlb_quirks = { + .errata = THEAD_QUIRK_ERRATA_THEAD_PMU | THEAD_QUIRK_ERRATA_JTLB, +}; + static const struct fdt_match thead_generic_match[] = { { .compatible = "canaan,kendryte-k230", .data = &thead_pmu_quirks }, { .compatible = "sophgo,cv1800b", .data = &thead_pmu_quirks }, { .compatible = "sophgo,cv1812h", .data = &thead_pmu_quirks }, { .compatible = "sophgo,sg2000", .data = &thead_pmu_quirks }, { .compatible = "sophgo,sg2002", .data = &thead_pmu_quirks }, - { .compatible = "sophgo,sg2044", .data = &thead_pmu_quirks }, + { .compatible = "sophgo,sg2044", .data = &thead_pmu_jtlb_quirks }, { .compatible = "thead,th1520", .data = &thead_th1520_quirks }, { }, }; diff --git a/platform/kendryte/k210/Kconfig b/platform/kendryte/k210/Kconfig deleted file mode 100644 index 5bf597315eb..00000000000 --- a/platform/kendryte/k210/Kconfig +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: BSD-2-Clause - -config PLATFORM_KENDRYTE_K210 - bool - select FDT - select IPI_MSWI - select IRQCHIP_PLIC - select SERIAL_SIFIVE - select TIMER_MTIMER - default y diff --git a/platform/kendryte/k210/configs/defconfig b/platform/kendryte/k210/configs/defconfig deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/platform/kendryte/k210/k210.dts b/platform/kendryte/k210/k210.dts deleted file mode 100644 index bcd075bf589..00000000000 --- a/platform/kendryte/k210/k210.dts +++ /dev/null @@ -1,70 +0,0 @@ -/* - * SPDX-License-Identifier: BSD-2-Clause - * - * Copyright (c) 2019 Western Digital Corporation or its affiliates. - * - * Authors: - * Damien Le Moal - */ - -/dts-v1/; -/ { - #address-cells = <2>; - #size-cells = <2>; - compatible = "kendryte,k210"; - - chosen { - bootargs = "console=hvc0 earlycon=sbi"; - }; - - cpus { - #address-cells = <1>; - #size-cells = <0>; - cpu0: cpu@0 { - device_type = "cpu"; - clock-frequency = <390000000>; - i-cache-size = <32768>; - d-cache-size = <32768>; - mmu-type = "none"; - reg = <0>; - riscv,isa = "rv64imafdc"; - status = "okay"; - cpu0_intc: interrupt-controller { - #interrupt-cells = <1>; - compatible = "riscv,cpu-intc"; - interrupt-controller; - }; - }; - cpu1: cpu@1 { - device_type = "cpu"; - clock-frequency = <390000000>; - d-cache-size = <32768>; - i-cache-size = <32768>; - mmu-type = "none"; - reg = <1>; - riscv,isa = "rv64imafdc"; - status = "okay"; - cpu1_intc: interrupt-controller { - #interrupt-cells = <1>; - compatible = "riscv,cpu-intc"; - interrupt-controller; - }; - }; - }; - - memory@80000000 { - /* Bank 0: 4 MB, Bank 1: 2 MB, AI chip SRAM: 2MB */ - device_type = "memory"; - reg = <0x00000000 0x80000000 0x00000000 0x00800000>; - }; - - plic0: interrupt-controller@C000000 { - #interrupt-cells = <1>; - compatible = "riscv,plic0"; - interrupt-controller; - interrupts-extended = - <&cpu0_intc 11 &cpu0_intc 9 - &cpu1_intc 11 &cpu1_intc 9>; - reg = <0x0 0xc000000 0x0 0x4000000>; - }; -}; diff --git a/platform/kendryte/k210/objects.mk b/platform/kendryte/k210/objects.mk deleted file mode 100644 index efac3d2f95e..00000000000 --- a/platform/kendryte/k210/objects.mk +++ /dev/null @@ -1,25 +0,0 @@ -# -# SPDX-License-Identifier: BSD-2-Clause -# -# Copyright (c) 2019 Western Digital Corporation or its affiliates. -# -# Authors: -# Damien Le Moal -# - -# Compiler flags -platform-cppflags-y = -platform-cflags-y = -platform-asflags-y = -platform-ldflags-y = - -# Objects to build -platform-objs-y += platform.o - -platform-objs-y += k210.o -platform-varprefix-k210.o = dt_k210 -platform-padding-k210.o = 2048 - -# Blobs to build -FW_PAYLOAD=y -FW_PAYLOAD_ALIGN=0x1000 diff --git a/platform/kendryte/k210/platform.c b/platform/kendryte/k210/platform.c deleted file mode 100644 index 65f5d497857..00000000000 --- a/platform/kendryte/k210/platform.c +++ /dev/null @@ -1,175 +0,0 @@ -/* - * SPDX-License-Identifier: BSD-2-Clause - * - * Copyright (c) 2019 Western Digital Corporation or its affiliates. - * - * Authors: - * Damien Le Moal - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "platform.h" - -extern const char dt_k210_start[]; - -unsigned long fw_platform_init(unsigned long arg0, unsigned long arg1, - unsigned long arg2, unsigned long arg3, - unsigned long arg4) -{ - return (unsigned long)&dt_k210_start[0]; -} - -static struct plic_data plic = { - .addr = K210_PLIC_BASE_ADDR, - .size = K210_PLIC_BASE_SIZE, - .num_src = K210_PLIC_NUM_SOURCES, - .context_map = { - [0] = { 0, 1 }, - [1] = { 2, 3 }, - }, -}; - -static struct aclint_mswi_data mswi = { - .addr = K210_ACLINT_MSWI_ADDR, - .size = ACLINT_MSWI_SIZE, - .first_hartid = 0, - .hart_count = K210_HART_COUNT, -}; - -static struct aclint_mtimer_data mtimer = { - .mtime_freq = K210_ACLINT_MTIMER_FREQ, - .mtime_addr = K210_ACLINT_MTIMER_ADDR + - ACLINT_DEFAULT_MTIME_OFFSET, - .mtime_size = ACLINT_DEFAULT_MTIME_SIZE, - .mtimecmp_addr = K210_ACLINT_MTIMER_ADDR + - ACLINT_DEFAULT_MTIMECMP_OFFSET, - .mtimecmp_size = ACLINT_DEFAULT_MTIMECMP_SIZE, - .first_hartid = 0, - .hart_count = K210_HART_COUNT, - .has_64bit_mmio = true, -}; - -static u32 k210_get_clk_freq(void) -{ - u32 clksel0, pll0; - u64 pll0_freq, clkr0, clkf0, clkod0, div; - - /* - * If the clock selector is not set, use the base frequency. - * Otherwise, use PLL0 frequency with a frequency divisor. - */ - clksel0 = k210_read_sysreg(K210_CLKSEL0); - if (!(clksel0 & 0x1)) - return K210_CLK0_FREQ; - - /* - * Get PLL0 frequency: - * freq = base frequency * clkf0 / (clkr0 * clkod0) - */ - pll0 = k210_read_sysreg(K210_PLL0); - clkr0 = 1 + (pll0 & 0x0000000f); - clkf0 = 1 + ((pll0 & 0x000003f0) >> 4); - clkod0 = 1 + ((pll0 & 0x00003c00) >> 10); - pll0_freq = clkf0 * K210_CLK0_FREQ / (clkr0 * clkod0); - - /* Get the frequency divisor from the clock selector */ - div = 2ULL << ((clksel0 & 0x00000006) >> 1); - - return pll0_freq / div; -} - -static int k210_system_reset_check(u32 type, u32 reason) -{ - return 1; -} - -static void k210_system_reset(u32 type, u32 reason) -{ - u32 val; - - val = k210_read_sysreg(K210_RESET); - val |= K210_RESET_MASK; - k210_write_sysreg(val, K210_RESET); - - while (1); -} - -static struct sbi_system_reset_device k210_reset = { - .name = "kendryte_k210_reset", - .system_reset_check = k210_system_reset_check, - .system_reset = k210_system_reset -}; - -static int k210_early_init(bool cold_boot) -{ - int rc; - - if (!cold_boot) - return 0; - - sbi_system_reset_add_device(&k210_reset); - - rc = sifive_uart_init(K210_UART_BASE_ADDR, k210_get_clk_freq(), - K210_UART_BAUDRATE); - if (rc) - return rc; - - return aclint_mswi_cold_init(&mswi); -} - -static int k210_final_init(bool cold_boot) -{ - void *fdt; - - if (!cold_boot) - return 0; - - fdt = fdt_get_address_rw(); - - fdt_cpu_fixup(fdt); - fdt_fixups(fdt); - - return 0; -} - -static int k210_irqchip_init(void) -{ - return plic_cold_irqchip_init(&plic); -} - -static int k210_timer_init(void) -{ - return aclint_mtimer_cold_init(&mtimer, NULL); -} - -const struct sbi_platform_operations platform_ops = { - .early_init = k210_early_init, - - .final_init = k210_final_init, - - .irqchip_init = k210_irqchip_init, - - .timer_init = k210_timer_init, -}; - -const struct sbi_platform platform = { - .opensbi_version = OPENSBI_VERSION, - .platform_version = SBI_PLATFORM_VERSION(0x0, 0x01), - .name = "Kendryte K210", - .features = 0, - .hart_count = K210_HART_COUNT, - .hart_stack_size = SBI_PLATFORM_DEFAULT_HART_STACK_SIZE, - .heap_size = - SBI_PLATFORM_DEFAULT_HEAP_SIZE(K210_HART_COUNT), - .platform_ops_addr = (unsigned long)&platform_ops -}; diff --git a/platform/kendryte/k210/platform.h b/platform/kendryte/k210/platform.h deleted file mode 100644 index 9417403d246..00000000000 --- a/platform/kendryte/k210/platform.h +++ /dev/null @@ -1,50 +0,0 @@ -/* - * SPDX-License-Identifier: BSD-2-Clause - * - * Copyright (c) 2019 Western Digital Corporation or its affiliates. - * - * Authors: - * Damien Le Moal - */ -#ifndef _K210_PLATFORM_H_ -#define _K210_PLATFORM_H_ - -#include - -#define K210_HART_COUNT 2 - -#define K210_UART_BAUDRATE 115200 -#define K210_ACLINT_MTIMER_FREQ 7800000 -#define K210_CLK0_FREQ 26000000UL -#define K210_PLIC_NUM_SOURCES 65 - -/* Registers base address */ -#define K210_SYSCTL_BASE_ADDR 0x50440000ULL -#define K210_UART_BASE_ADDR 0x38000000ULL -#define K210_CLINT_BASE_ADDR 0x02000000ULL -#define K210_ACLINT_MSWI_ADDR \ - (K210_CLINT_BASE_ADDR + CLINT_MSWI_OFFSET) -#define K210_ACLINT_MTIMER_ADDR \ - (K210_CLINT_BASE_ADDR + CLINT_MTIMER_OFFSET) -#define K210_PLIC_BASE_ADDR 0x0C000000ULL -#define K210_PLIC_BASE_SIZE (0x200000ULL + (K210_HART_COUNT * 0x1000)) - -/* Registers */ -#define K210_PLL0 0x08 -#define K210_CLKSEL0 0x20 -#define K210_RESET 0x30 - -/* Register bit masks */ -#define K210_RESET_MASK 0x01 - -static inline u32 k210_read_sysreg(u32 reg) -{ - return readl((volatile void *)(K210_SYSCTL_BASE_ADDR + reg)); -} - -static inline void k210_write_sysreg(u32 val, u32 reg) -{ - writel(val, (volatile void *)(K210_SYSCTL_BASE_ADDR + reg)); -} - -#endif /* _K210_PLATFORM_H_ */ diff --git a/platform/nuclei/ux600/platform.c b/platform/nuclei/ux600/platform.c index 14fbaeb6c07..07b279772f2 100644 --- a/platform/nuclei/ux600/platform.c +++ b/platform/nuclei/ux600/platform.c @@ -63,6 +63,7 @@ static u32 ux600_clk_freq = 8000000; static struct plic_data plic = { + .unique_id = 0, .addr = UX600_PLIC_ADDR, .size = UX600_PLIC_SIZE, .num_src = UX600_PLIC_NUM_SOURCES, diff --git a/platform/template/objects.mk b/platform/template/objects.mk index f240a5576eb..9b4bc928d78 100644 --- a/platform/template/objects.mk +++ b/platform/template/objects.mk @@ -85,11 +85,11 @@ FW_PAYLOAD= # This needs to be 4MB aligned for 32-bit support # This needs to be 2MB aligned for 64-bit support ifeq ($(PLATFORM_RISCV_XLEN), 32) -FW_PAYLOAD_OFFSET=0x400000 +FW_PAYLOAD_ALIGN=0x400000 else -FW_PAYLOAD_OFFSET=0x200000 +FW_PAYLOAD_ALIGN=0x200000 endif -# FW_PAYLOAD_ALIGN=0x1000 +# FW_PAYLOAD_OFFSET=0x400000 # FW_PAYLOAD_PATH="path to next boot stage binary image file" # FW_PAYLOAD_FDT_OFFSET=0x2200000 # diff --git a/platform/template/platform.c b/platform/template/platform.c index 38be1b5c06b..1f2a1dce241 100644 --- a/platform/template/platform.c +++ b/platform/template/platform.c @@ -34,6 +34,7 @@ #define PLATFORM_UART_BAUDRATE 115200 static struct plic_data plic = { + .unique_id = 0, .addr = PLATFORM_PLIC_ADDR, .size = PLATFORM_PLIC_SIZE, .num_src = PLATFORM_PLIC_NUM_SOURCES, diff --git a/scripts/create-binary-archive.sh b/scripts/create-binary-archive.sh index 6ea4c9c57b7..7f21178816d 100755 --- a/scripts/create-binary-archive.sh +++ b/scripts/create-binary-archive.sh @@ -100,7 +100,6 @@ build_opensbi() { 64) # Setup 64-bit platform list BUILD_PLATFORM_SUBDIR+=("nuclei/ux600") - BUILD_PLATFORM_SUBDIR+=("kendryte/k210") BUILD_PLATFORM_SUBDIR+=("generic") ;; *)