From e34ea14e660da03b2cdcaeb5bb33626c2457cdf7 Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Mon, 5 Aug 2019 00:38:48 +0800 Subject: [PATCH 1/3] --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index 42ef5bc..79b04eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,5 +4,6 @@ version = "0.1.0" authors = ["Samuel Ortiz "] repository = "https://github.com/rust-vmm/vm-device" license = "Apache-2.0" +edition = "2018" [dependencies] From 613228f95c1db6fe4c145e09d41b9f48c939b62e Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Mon, 5 Aug 2019 00:40:08 +0800 Subject: [PATCH 2/3] --- Cargo.toml | 9 +++ src/interrupt/mod.rs | 188 +++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 2 + 3 files changed, 199 insertions(+) create mode 100644 src/interrupt/mod.rs diff --git a/Cargo.toml b/Cargo.toml index 79b04eb..65b3669 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,3 +7,12 @@ license = "Apache-2.0" edition = "2018" [dependencies] +vmm-sys-util = { git = "https://github.com/rust-vmm/vmm-sys-util.git", branch = "master" } + +[features] +default = ["legacy_irq", "generic_msi", "pci_msi", "pci_msix"] +legacy_irq = [] +msi_irq = [] +generic_msi = ["msi_irq"] +pci_msi = ["msi_irq"] +pci_msix = ["msi_irq"] diff --git a/src/interrupt/mod.rs b/src/interrupt/mod.rs new file mode 100644 index 0000000..8b70eba --- /dev/null +++ b/src/interrupt/mod.rs @@ -0,0 +1,188 @@ +//! Traits and Structs to manage interrupt sources for devices. +//! +//! In system programming, an interrupt is a signal to the processor emitted by hardware or +//! software indicating an event that needs immediate attention. An interrupt alerts the processor +//! to a high-priority condition requiring the interruption of the current code the processor is +//! executing. The processor responds by suspending its current activities, saving its state, and +//! executing a function called an interrupt handler (or an interrupt service routine, ISR) to deal +//! with the event. This interruption is temporary, and, after the interrupt handler finishes, +//! unless handling the interrupt has emitted a fatal error, the processor resumes normal +//! activities. +//! +//! Hardware interrupts are used by devices to communicate that they require attention from the +//! operating system, or a bare-metal program running on the CPU if there are no OSes. The act of +//! initiating a hardware interrupt is referred to as an interrupt request (IRQ). Different devices +//! are usually associated with different interrupts using a unique value associated with each +//! interrupt. This makes it possible to know which hardware device caused which interrupts. +//! These interrupt values are often called IRQ lines, or just interrupt lines. +//! +//! Nowadays, IRQ lines is not the only mechanism to delivery device interrupts to processors. +//! MSI [(Message Signalled Interrupt)](https://en.wikipedia.org/wiki/Message_Signaled_Interrupts) +//! is another commonly used alternative in-band method of signaling an interrupt, using special +//! in-band messages to replace traditional out-of-band assertion of dedicated interrupt lines. +//! While more complex to implement in a device, message signaled interrupts have some significant +//! advantages over pin-based out-of-band interrupt signaling. Message signaled interrupts are +//! supported in PCI bus since its version 2.2, and in later available PCI Express bus. Some non-PCI +//! architectures also use message signaled interrupts. +//! +//! On the other hand, IRQ is a term commonly used by Operating Systems when dealing with hardware +//! interrupts. The IRQ number managed by OSes is independent of the IRQ number managed by VMM. +//! To make things easy, `Interrupt Source` is used instead of IRQ for pin-based interrupts and +//! MSI interrupts. +//! +//! A device may support multiple types of interrupts, and each type of interrupt may support one +//! or multiple interrupt sources. For example, a PCI device may support: +//! * Legacy Irq: exactly one interrupt source. +//! * PCI MSI Irq: 1,2,4,8,16,32 interrupt sources. +//! * PCI MSIx Irq: 2^n(n=0-11) interrupt sources. +//! +//! A distinct Interrupt Source Identifier (ISID) will be assigned to each interrupt source. +//! An ID allocator will be used to allocate and free Interrupt Source Identifiers for devices. +//! To decouple the vm-device crate from the ID allocator, the vm-device crate doesn't take the +//! responsibility to allocate/free Interrupt Source IDs but only makes use of assigned IDs. +//! +//! The overall flow to deal with interrupts is: +//! * the VMM creates an interrupt manager +//! * the VMM creates a device manager, passing on an reference to the interrupt manager +//! * the device manager passes on an reference to the interrupt manager to all registered devices +//! * guest kernel loads drivers for virtual devices +//! * guest device driver determines the type and number of interrupts needed, and update the +//! device configuration +//! * the virtual device backend requests the interrupt manager to create an interrupt group +//! according to guest configuration information + +use std::sync::Arc; +use vmm_sys_util::eventfd::EventFd; + +/// Reuse std::io::Result to simplify inter operability. +pub type Result = std::io::Result; + +/// Data type to store an interrupt index. +pub type InterruptIndex = u32; + +#[cfg(feature = "generic_msi")] +/// Maximum number of generic Message Signalled Interrupts supported per device. +pub const MAX_GENERIC_MSI_IRQS: InterruptIndex = 32; + +/// Type of interrupt source. +pub enum InterruptSourceType { + #[cfg(feature = "legacy_irq")] + /// Legacy Pin-based Interrupt. + /// On x86 platforms, legacy interrupts are routed through 82599 PICs and/or IOAPICs. + Legacy, + #[cfg(feature = "generic_msi")] + /// Generic Message Signalled Interrupt. + /// Some non-PCI devices (like HPET on x86) make use of generic MSI in platform specific ways. + GenericMsi, + #[cfg(feature = "pci_msi")] + /// PCI Message Signalled Interrupt. + PciMsi, + #[cfg(feature = "pci_msix")] + /// PCI Message Signalled Interrupt - Second Generation. + PciMsix, +} + +/// Configuration data for an interrupt source. +pub enum InterruptSourceConfig { + /// Configuration data for Legacy interrupts. + Legacy(LegacyIrqSourceConfig), + /// Configuration data for GenericMsi, PciMsi, PciMsix interrupts. + Msi(MsiIrqSourceConfig), +} + +/// Configuration data for legacy interrupts. +/// +/// On x86 platforms, legacy interrupts means those interrupts routed through PICs or IOAPICs. +pub struct LegacyIrqSourceConfig { + /// Bitmask for valid bits in the corresponding interrupt status register. + pub valid_flag_mask: u32, +} + +/// Configuration data for GenericMsi, PciMsi, PciMsix interrupts. +pub struct MsiIrqSourceConfig { + pub high_addr: u32, + pub low_addr: u32, + pub data: u32, +} + +/// Trait to manage [InterruptSourceGroup](trait.InterruptSourceGroup.html) objects for devices. +pub trait InterruptManager { + /// Create an [InterruptSourceGroup](trait.InterruptSourceGroup.html) object for a device. + /// + /// An [InterruptSourceGroup](trait.InterruptSourceGroup.html) object manages all interrupt + /// sources of the same type for a device. + /// + /// # Arguments + /// * ty: type of interrupt source. + /// * base: base Interrupt Source ID to be managed by the group object. + /// * count: number of Interrupt Sources to be managed by the group object. + fn create_group( + &self, + ty: InterruptSourceType, + base: InterruptIndex, + count: InterruptIndex, + ) -> Result>>; + + /// Destroy an [InterruptSourceGroup](trait.InterruptSourceGroup.html) object created by + /// [create_group()](trait.InterruptManager.html#tymethod.create_group). + fn destroy_group(&self, group: Arc>) -> Result<()>; + + /* + /// Enable all created interrupt groups. + /// + /// During early startup stages, the manager may get called to create interrupt groups when + /// the underline system is not ready to handle interrupts yet. So the manager will record + /// those requests and apply them when the underline system is ready. + fn start(&self) -> Result<()>; + */ +} + +/// Trait to manage a group of interrupt sources for a device. +/// +/// A device may support several types of interrupts, and each type of interrupt may contain one or +/// multiple continuous interrupt sources. For example, a PCI device may concurrently support: +/// * Legacy Irq: exactly one interrupt source. +/// * PCI MSI Irq: 1,2,4,8,16,32 interrupt sources. +/// * PCI MSIx Irq: 2^n(n=0-11) interrupt sources. +/// +/// PCI MSI interrupts of a device may not be configured individually, and must configured as a +/// whole block. So all interrupts of the same type of a device are abstracted as an +/// [InterruptSourceGroup](trait.InterruptSourceGroup.html) object, instead of abstracting each +/// interrupt source as a distinct InterruptSource. +#[allow(clippy::len_without_is_empty)] +pub trait InterruptSourceGroup { + /// Get type of the interrupt group. + fn get_type(&self) -> InterruptSourceType; + + /// Get number of assigned Interrupt Sources. + fn len(&self) -> InterruptIndex; + + /// Get base of assigned Interrupt Source Identifier. + fn get_base(&self) -> InterruptIndex; + + /// Get the irqfd to inject interrupts into the guest by using Linux KVM module. + fn get_irqfd(&self, _index: InterruptIndex) -> Option<&EventFd> { + None + } + + /// Configure the interrupt source for generating interrupts for guest. + /// + /// # Arguments + /// * index: sub-index into the group. + /// * config: configuration data for the interrupt source. + fn configure(&self, index: InterruptIndex, config: InterruptSourceConfig) -> Result<()>; + + //fn set_mask(&self, index: InterruptIndex, masked: bool) -> Result<()>; + + /// Inject an interrupt into the guest. + /// + /// If the interrupt has an associated `interrupt_status` register, all bits set in `flag` + /// will be atomically ORed into the `interrupt_status` register. + fn trigger(&self, index: InterruptIndex, flags: u32) -> Result<()>; + + /// Acknowledge that the guest has handled the interrupt. + /// + /// If the interrupt has an associated `interrupt_status` register, all bits set in `flag` + /// will get atomically cleared from the `interrupt_status` register. + fn ack(&self, index: InterruptIndex, flags: u32) -> Result<()>; +} diff --git a/src/lib.rs b/src/lib.rs index 31e1bb2..7991dd7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,5 @@ +pub mod interrupt; + #[cfg(test)] mod tests { #[test] From 85d92ad1a241923357c9013bc236eb4dd1c42f15 Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Mon, 5 Aug 2019 02:14:44 +0800 Subject: [PATCH 3/3] --- Cargo.toml | 6 +- src/interrupt/kvm_irq.rs | 136 +++++++++++++++++++++++++++++++++++++++ src/interrupt/mod.rs | 5 ++ 3 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 src/interrupt/kvm_irq.rs diff --git a/Cargo.toml b/Cargo.toml index 65b3669..7c1b82e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,10 +7,14 @@ license = "Apache-2.0" edition = "2018" [dependencies] +libc = ">=0.2.39" + +kvm-ioctls = { git = "https://github.com/rust-vmm/kvm-ioctls.git", branch = "master", optional = true } vmm-sys-util = { git = "https://github.com/rust-vmm/vmm-sys-util.git", branch = "master" } [features] -default = ["legacy_irq", "generic_msi", "pci_msi", "pci_msix"] +default = ["kvm_irq", "legacy_irq"] +kvm_irq = ["kvm-ioctls"] legacy_irq = [] msi_irq = [] generic_msi = ["msi_irq"] diff --git a/src/interrupt/kvm_irq.rs b/src/interrupt/kvm_irq.rs new file mode 100644 index 0000000..46561a6 --- /dev/null +++ b/src/interrupt/kvm_irq.rs @@ -0,0 +1,136 @@ +//! Manage hardware interrupts for devices based on Linux KVM framework. + +use kvm_ioctls::VmFd; +use vmm_sys_util::eventfd::EventFd; + +use super::*; + +pub struct KvmIrqManager { + vmfd: Arc, +} + +impl KvmIrqManager { + pub fn new(vmfd: Arc) -> Self { + KvmIrqManager { vmfd } + } +} + +impl InterruptManager for KvmIrqManager { + fn create_group( + &self, + ty: InterruptSourceType, + base: u32, + count: u32, + ) -> Result>> { + match ty { + #[cfg(feature = "legacy_irq")] + InterruptSourceType::Legacy => legacy::LegacyIrq::new(self.vmfd.clone(), base, count), + #[cfg(feature = "generic_msi")] + InterruptSourceType::GenericMsi => {} + #[cfg(feature = "pci_msi")] + InterruptSourceType::PciMsi => {} + #[cfg(feature = "pci_msix")] + InterruptSourceType::PciMsix => {} + } + } + + fn destroy_group(&self, group: Arc>) -> Result<()> { + match group.get_type() { + #[cfg(feature = "legacy_irq")] + InterruptSourceType::Legacy => {} + } + Ok(()) + } +} + +#[cfg(feature = "legacy_irq")] +mod legacy { + use super::*; + use std::os::unix::io::AsRawFd; + use std::sync::atomic::{AtomicUsize, Ordering}; + + pub(super) struct LegacyIrq { + base: u32, + vmfd: Arc, + status: AtomicUsize, + irqfd: EventFd, + } + + impl LegacyIrq { + #[allow(clippy::new_ret_no_self)] + pub fn new( + vmfd: Arc, + base: u32, + count: u32, + ) -> Result>> { + if count != 1 { + return Err(std::io::Error::from_raw_os_error(libc::EINVAL)); + } + Ok(Arc::new(Box::new(LegacyIrq { + base, + vmfd, + status: AtomicUsize::new(0), + irqfd: EventFd::new(0).unwrap(), + }))) + } + } + + impl InterruptSourceGroup for LegacyIrq { + fn get_type(&self) -> InterruptSourceType { + InterruptSourceType::Legacy + } + + fn len(&self) -> u32 { + 1 + } + + fn get_base(&self) -> u32 { + self.base + } + + fn get_irqfd(&self, index: InterruptIndex) -> Option<&EventFd> { + if index == 0 { + Some(&self.irqfd) + } else { + None + } + } + + fn configure(&self, index: InterruptIndex, config: InterruptSourceConfig) -> Result<()> { + if index != 0 { + return Err(std::io::Error::from_raw_os_error(libc::EINVAL)); + } + if let InterruptSourceConfig::Legacy(_mask) = config { + //self.status_mask = mask.valid_flag_mask; + return self.vmfd.register_irqfd(self.irqfd.as_raw_fd(), self.base); + } + Err(std::io::Error::from_raw_os_error(libc::EINVAL)) + } + + fn trigger(&self, index: InterruptIndex, flags: u32) -> Result<()> { + if index != 0 { + return Err(std::io::Error::from_raw_os_error(libc::EINVAL)); + } + self.status.fetch_or(flags as usize, Ordering::SeqCst); + self.irqfd.write(1) + } + + fn ack(&self, index: InterruptIndex, flags: u32) -> Result<()> { + if index != 0 { + return Err(std::io::Error::from_raw_os_error(libc::EINVAL)); + } + self.status.fetch_and(!(flags as usize), Ordering::SeqCst); + Ok(()) + } + } +} + +#[cfg(feature = "msi_irq")] +mod msi { + //TODO +} + +#[cfg(feature = "generic_msi")] +mod generic_msi { + //TODO +} diff --git a/src/interrupt/mod.rs b/src/interrupt/mod.rs index 8b70eba..3a9662b 100644 --- a/src/interrupt/mod.rs +++ b/src/interrupt/mod.rs @@ -186,3 +186,8 @@ pub trait InterruptSourceGroup { /// will get atomically cleared from the `interrupt_status` register. fn ack(&self, index: InterruptIndex, flags: u32) -> Result<()>; } + +#[cfg(feature = "kvm_irq")] +mod kvm_irq; +#[cfg(feature = "kvm_irq")] +pub use kvm_irq::KvmIrqManager;