x86_64/pci: enable support for PCIe enhanced configuration

Author: zyuiop

src/arch/x86_64/kernel/acpi.rs

@@ -44,6 +44,11 @@ const SLP_EN: u16 = 1 << 13;
 
 /// The "Multiple APIC Description Table" (MADT) preserved for get_apic_table().
 static MADT: OnceCell<AcpiTable<'_>> = OnceCell::new();
+
+/// The MCFG table, to address PCI-E configuration space
+#[cfg(feature = "pci")]
+static MCFG: OnceCell<AcpiTable<'_>> = OnceCell::new();
+
 /// The PM1A Control I/O Port for powering off the computer through ACPI.
 static PM1A_CNT_BLK: OnceCell<Port<u16>> = OnceCell::new();
 /// The Sleeping State Type code for powering off the computer through ACPI.
@@ -471,6 +476,11 @@ pub fn get_madt() -> Option<&'static AcpiTable<'static>> {
 	MADT.get()
 }
 
+#[cfg(feature = "pci")]
+pub fn get_mcfg_table() -> Option<&'static AcpiTable<'static>> {
+	MCFG.get()
+}
+
 pub fn poweroff() {
 	if let (Some(mut pm1a_cnt_blk), Some(&slp_typa)) = (PM1A_CNT_BLK.get().cloned(), SLP_TYPA.get())
 	{
@@ -546,6 +556,12 @@ pub fn init() {
 				"SSDT at {table_physical_address:p} has invalid checksum"
 			);
 			parse_ssdt(table);
+		} else if cfg!(feature = "pci") && table.header.signature() == "MCFG" {
+			assert!(
+				verify_checksum(table.header_start_address(), table.header.length as usize).is_ok(),
+				"MCFG at {table_physical_address:p} has invalid checksum"
+			);
+			MCFG.set(table).unwrap();
 		}
 	}
 }

src/arch/x86_64/kernel/mod.rs

@@ -166,14 +166,14 @@ pub fn boot_processor_init() {
 	interrupts::install();
 	systemtime::init();
 
-	if is_uhyve_with_pci() || !is_uhyve() {
-		#[cfg(feature = "pci")]
-		pci::init();
-	}
 	if !env::is_uhyve() {
 		#[cfg(feature = "acpi")]
 		acpi::init();
 	}
+	if is_uhyve_with_pci() || !is_uhyve() {
+		#[cfg(feature = "pci")]
+		pci::init();
+	}
 
 	apic::init();
 	scheduler::install_timer_handler();

src/arch/x86_64/kernel/pci.rs

@@ -12,14 +12,37 @@ const CONFIG_ADDRESS: Port<u32> = Port::new(0xcf8);
 const CONFIG_DATA: Port<u32> = Port::new(0xcfc);
 
 #[derive(Debug, Copy, Clone)]
-pub(crate) struct PciConfigRegion;
+struct PciConfigRegion;
 
 impl PciConfigRegion {
 	pub const fn new() -> Self {
 		Self {}
 	}
 }
 
+#[derive(Debug, Copy, Clone)]
+pub enum PciConfigAccess {
+	PciConfigRegion(PciConfigRegion),
+	#[cfg(feature = "acpi")]
+	PcieConfigRegion(pcie::McfgTableEntry),
+}
+
+impl ConfigRegionAccess for PciConfigAccess {
+	unsafe fn read(&self, address: PciAddress, offset: u16) -> u32 {
+		match self {
+			PciConfigAccess::PciConfigRegion(entry) => entry.read(address, offset),
+			PciConfigAccess::PcieConfigRegion(entry) => entry.read(address, offset),
+		}
+	}
+
+	unsafe fn write(&self, address: PciAddress, offset: u16, value: u32) {
+		match self {
+			PciConfigAccess::PciConfigRegion(entry) => entry.write(address, offset, value),
+			PciConfigAccess::PcieConfigRegion(entry) => entry.write(address, offset, value),
+		}
+	}
+}
+
 impl ConfigRegionAccess for PciConfigRegion {
 	#[inline]
 	unsafe fn read(&self, pci_addr: PciAddress, register: u16) -> u32 {
@@ -59,20 +82,135 @@ impl ConfigRegionAccess for PciConfigRegion {
 pub(crate) fn init() {
 	debug!("Scanning PCI Busses 0 to {}", PCI_MAX_BUS_NUMBER - 1);
 
+	#[cfg(feature = "acpi")]
+	if pcie::init_pcie() { return; }
+
+	enumerate_devices(0, PCI_MAX_BUS_NUMBER, PciConfigAccess::PciConfigRegion(PciConfigRegion::new()))
+}
+
+fn enumerate_devices(bus_start: u8, bus_end: u8, access: PciConfigAccess) {
 	// Hermit only uses PCI for network devices.
 	// Therefore, multifunction devices as well as additional bridges are not scanned.
 	// We also limit scanning to the first 32 buses.
-	let pci_config = PciConfigRegion::new();
-	for bus in 0..PCI_MAX_BUS_NUMBER {
+	for bus in bus_start..bus_end {
 		for device in 0..PCI_MAX_DEVICE_NUMBER {
 			let pci_address = PciAddress::new(0, bus, device, 0);
 			let header = PciHeader::new(pci_address);
 
-			let (device_id, vendor_id) = header.id(pci_config);
+			let (device_id, vendor_id) = header.id(access);
 			if device_id != u16::MAX && vendor_id != u16::MAX {
-				let device = PciDevice::new(pci_address, pci_config);
+				let device = PciDevice::new(pci_address, access);
 				PCI_DEVICES.with(|pci_devices| pci_devices.unwrap().push(device));
 			}
 		}
 	}
 }
+
+#[cfg(feature = "acpi")]
+mod pcie {
+	use core::ptr;
+	use pci_types::{ConfigRegionAccess, PciAddress};
+	use memory_addresses::{PhysAddr, VirtAddr};
+	use super::{PciConfigAccess, PCI_MAX_BUS_NUMBER};
+	use crate::env;
+	use crate::env::kernel::acpi;
+
+	pub fn init_pcie() -> bool {
+		let Some(table) = acpi::get_mcfg_table() else { return false; };
+
+		let mut start_addr: *const McfgTableEntry = core::ptr::with_exposed_provenance(table.table_start_address() + 8);
+		let end_addr: *const McfgTableEntry = core::ptr::with_exposed_provenance(table.table_end_address() + 8);
+
+		if start_addr == end_addr {
+			return false;
+		}
+
+		while start_addr < end_addr {
+			unsafe {
+				let read = ptr::read_unaligned(start_addr);
+				init_pcie_bus(read);
+				start_addr = start_addr.add(1);
+			}
+		}
+
+		true
+	}
+
+	#[derive(Debug)]
+	#[repr(C)]
+	struct PcieDeviceConfig {
+		vendor_id: u16,
+		device_id: u16,
+		_reserved: [u8; 4096 - 8]
+	}
+
+	#[derive(Debug, Copy, Clone)]
+	#[repr(C)]
+	pub(crate) struct McfgTableEntry {
+		pub base_address: u64,
+		pub pci_segment_number: u16,
+		pub start_pci_bus: u8,
+		pub end_pci_bus: u8,
+		_reserved: u32
+	}
+
+	impl McfgTableEntry {
+		pub fn pci_config_space_address(&self, bus_number: u8, device: u8, function: u8) -> PhysAddr {
+			PhysAddr::new(
+				self.base_address +
+					((bus_number as u64) << 20) |
+					(((device as u64) & 0x1f) << 15) |
+					(((function as u64) & 0x7) << 12)
+			)
+		}
+	}
+
+	#[derive(Debug)]
+	#[cfg(feature = "pci")]
+	struct McfgTable(alloc::vec::Vec<McfgTableEntry>);
+
+	impl PcieDeviceConfig {
+		fn get<'a>(physical_address: PhysAddr) -> &'a Self {
+			assert!(env::is_uefi());
+
+			// For UEFI Systems, the tables are already mapped so we only need to return a proper reference to the table
+			let allocated_virtual_address = VirtAddr::new(physical_address.as_u64());
+			let ptr: *const PcieDeviceConfig = allocated_virtual_address.as_ptr();
+
+			unsafe { ptr.as_ref().unwrap() }
+		}
+	}
+
+	impl ConfigRegionAccess for McfgTableEntry {
+		unsafe fn read(&self, address: PciAddress, offset: u16) -> u32 {
+			assert_eq!(address.segment(), self.pci_segment_number);
+			assert!(address.bus() >= self.start_pci_bus);
+			assert!(address.bus() <= self.end_pci_bus);
+
+			let ptr = self.pci_config_space_address(address.bus(), address.device(), address.function()) + offset as u64;
+			let ptr = ptr.as_usize() as *const u32;
+
+			unsafe { *ptr }
+		}
+
+		unsafe fn write(&self, address: PciAddress, offset: u16, value: u32) {
+			assert_eq!(address.segment(), self.pci_segment_number);
+			assert!(address.bus() >= self.start_pci_bus);
+			assert!(address.bus() <= self.end_pci_bus);
+
+			let ptr = self.pci_config_space_address(address.bus(), address.device(), address.function()) + offset as u64;
+			let ptr = ptr.as_usize() as *mut u32;
+
+			unsafe { *ptr = value; }
+		}
+	}
+
+	fn init_pcie_bus(bus_entry: McfgTableEntry) {
+		if bus_entry.start_pci_bus > PCI_MAX_BUS_NUMBER {
+			return;
+		}
+
+		let end = if bus_entry.end_pci_bus > PCI_MAX_BUS_NUMBER { PCI_MAX_BUS_NUMBER } else { bus_entry.end_pci_bus };
+		super::enumerate_devices(bus_entry.start_pci_bus, end, PciConfigAccess::PcieConfigRegion(bus_entry));
+	}
+}

src/drivers/fs/virtio_pci.rs

@@ -2,7 +2,7 @@ use alloc::vec::Vec;
 
 use volatile::VolatileRef;
 
-use crate::arch::pci::PciConfigRegion;
+use crate::arch::pci::PciConfigAccess;
 use crate::drivers::fs::virtio_fs::{FsDevCfg, VirtioFsDriver};
 use crate::drivers::pci::PciDevice;
 use crate::drivers::virtio::error::{self, VirtioError};
@@ -26,7 +26,7 @@ impl VirtioFsDriver {
 	/// configuration structures and moving them into the struct.
 	pub fn new(
 		caps_coll: UniCapsColl,
-		device: &PciDevice<PciConfigRegion>,
+		device: &PciDevice<PciConfigAccess>,
 	) -> Result<Self, error::VirtioFsError> {
 		let device_id = device.device_id();
 
@@ -54,7 +54,7 @@ impl VirtioFsDriver {
 	}
 
 	/// Initializes virtio filesystem device
-	pub fn init(device: &PciDevice<PciConfigRegion>) -> Result<VirtioFsDriver, VirtioError> {
+	pub fn init(device: &PciDevice<PciConfigAccess>) -> Result<VirtioFsDriver, VirtioError> {
 		let mut drv = match pci::map_caps(device) {
 			Ok(caps) => match VirtioFsDriver::new(caps, device) {
 				Ok(driver) => driver,

src/drivers/net/rtl8139.rs

@@ -10,7 +10,7 @@ use pci_types::{Bar, CommandRegister, InterruptLine, MAX_BARS};
 use x86_64::instructions::port::Port;
 
 use crate::arch::kernel::interrupts::*;
-use crate::arch::pci::PciConfigRegion;
+use crate::arch::pci::PciConfigAccess;
 use crate::drivers::Driver;
 use crate::drivers::error::DriverError;
 use crate::drivers::net::{NetworkDriver, mtu};
@@ -423,7 +423,7 @@ impl Drop for RTL8139Driver {
 }
 
 pub(crate) fn init_device(
-	device: &PciDevice<PciConfigRegion>,
+	device: &PciDevice<PciConfigAccess>,
 ) -> Result<RTL8139Driver, DriverError> {
 	let irq = device.get_irq().unwrap();
 	let mut iobase: Option<u32> = None;

src/drivers/net/virtio/pci.rs

@@ -7,7 +7,7 @@ use smoltcp::phy::ChecksumCapabilities;
 use volatile::VolatileRef;
 
 use super::{Init, Uninit};
-use crate::arch::pci::PciConfigRegion;
+use crate::arch::pci::PciConfigAccess;
 use crate::drivers::net::virtio::{NetDevCfg, VirtioNetDriver};
 use crate::drivers::pci::PciDevice;
 use crate::drivers::virtio::error::{self, VirtioError};
@@ -32,7 +32,7 @@ impl VirtioNetDriver<Uninit> {
 	/// configuration structures and moving them into the struct.
 	pub(crate) fn new(
 		caps_coll: UniCapsColl,
-		device: &PciDevice<PciConfigRegion>,
+		device: &PciDevice<PciConfigAccess>,
 	) -> Result<Self, error::VirtioNetError> {
 		let device_id = device.device_id();
 		let UniCapsColl {
@@ -69,7 +69,7 @@ impl VirtioNetDriver<Uninit> {
 	/// Returns a driver instance of
 	/// [VirtioNetDriver](structs.virtionetdriver.html) or an [VirtioError](enums.virtioerror.html).
 	pub(crate) fn init(
-		device: &PciDevice<PciConfigRegion>,
+		device: &PciDevice<PciConfigAccess>,
 	) -> Result<VirtioNetDriver<Init>, VirtioError> {
 		// enable bus master mode
 		device.set_command(CommandRegister::BUS_MASTER_ENABLE);

src/drivers/pci.rs

@@ -16,7 +16,7 @@ use pci_types::{
 	InterruptPin, MAX_BARS, PciAddress, PciHeader, StatusRegister, VendorId,
 };
 
-use crate::arch::pci::PciConfigRegion;
+use crate::arch::pci::PciConfigAccess;
 #[cfg(feature = "fuse")]
 use crate::drivers::fs::virtio_fs::VirtioFsDriver;
 #[cfg(any(feature = "tcp", feature = "udp"))]
@@ -53,7 +53,7 @@ use crate::drivers::{Driver, InterruptHandlerQueue};
 use crate::env;
 use crate::init_cell::InitCell;
 
-pub(crate) static PCI_DEVICES: InitCell<Vec<PciDevice<PciConfigRegion>>> =
+pub(crate) static PCI_DEVICES: InitCell<Vec<PciDevice<PciConfigAccess>>> =
 	InitCell::new(Vec::new());
 static PCI_DRIVERS: InitCell<Vec<PciDriver>> = InitCell::new(Vec::new());
 

src/drivers/virtio/transport/pci.rs

@@ -20,7 +20,7 @@ use volatile::access::ReadOnly;
 use volatile::{VolatilePtr, VolatileRef};
 
 use crate::arch::memory_barrier;
-use crate::arch::pci::PciConfigRegion;
+use crate::arch::pci::PciConfigAccess;
 use crate::drivers::error::DriverError;
 #[cfg(feature = "fuse")]
 use crate::drivers::fs::virtio_fs::VirtioFsDriver;
@@ -675,7 +675,7 @@ impl PciBar {
 ///
 /// Returns ONLY Virtio specific capabilities, which allow to locate the actual capability
 /// structures inside the memory areas, indicated by the BaseAddressRegisters (BAR's).
-fn read_caps(device: &PciDevice<PciConfigRegion>) -> Result<Vec<PciCap>, PciError> {
+fn read_caps(device: &PciDevice<PciConfigAccess>) -> Result<Vec<PciCap>, PciError> {
 	let device_id = device.device_id();
 
 	let capabilities = device
@@ -706,7 +706,7 @@ fn read_caps(device: &PciDevice<PciConfigRegion>) -> Result<Vec<PciCap>, PciErro
 	}
 }
 
-pub(crate) fn map_caps(device: &PciDevice<PciConfigRegion>) -> Result<UniCapsColl, VirtioError> {
+pub(crate) fn map_caps(device: &PciDevice<PciConfigAccess>) -> Result<UniCapsColl, VirtioError> {
 	let device_id = device.device_id();
 
 	// In case caplist pointer is not used, abort as it is essential
@@ -790,7 +790,7 @@ pub(crate) fn map_caps(device: &PciDevice<PciConfigRegion>) -> Result<UniCapsCol
 /// driver with a [`PciDevice<PciConfigRegion>`] reference, allowing access to the capabilities
 /// list of the given device through [map_caps].
 pub(crate) fn init_device(
-	device: &PciDevice<PciConfigRegion>,
+	device: &PciDevice<PciConfigAccess>,
 ) -> Result<VirtioDriver, DriverError> {
 	let device_id = device.device_id();
 

src/drivers/vsock/pci.rs

@@ -1,4 +1,4 @@
-use crate::arch::pci::PciConfigRegion;
+use crate::arch::pci::PciConfigAccess;
 use crate::drivers::pci::PciDevice;
 use crate::drivers::virtio::error::{self, VirtioError};
 use crate::drivers::virtio::transport::pci;
@@ -31,7 +31,7 @@ impl VirtioVsockDriver {
 	/// configuration structures and moving them into the struct.
 	pub fn new(
 		caps_coll: UniCapsColl,
-		device: &PciDevice<PciConfigRegion>,
+		device: &PciDevice<PciConfigAccess>,
 	) -> Result<Self, error::VirtioVsockError> {
 		let device_id = device.device_id();
 
@@ -64,7 +64,7 @@ impl VirtioVsockDriver {
 	///
 	/// Returns a driver instance of VirtioVsockDriver.
 	pub(crate) fn init(
-		device: &PciDevice<PciConfigRegion>,
+		device: &PciDevice<PciConfigAccess>,
 	) -> Result<VirtioVsockDriver, VirtioError> {
 		let mut drv = match pci::map_caps(device) {
 			Ok(caps) => match VirtioVsockDriver::new(caps, device) {