[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Qemu-devel] RFC: Add support for KVM_CAP_SPLIT_IRQCHIP
|
From: |
Matt Gingell |
|
Subject: |
[Qemu-devel] RFC: Add support for KVM_CAP_SPLIT_IRQCHIP |
|
Date: |
Thu, 29 Oct 2015 13:25:41 -0700 |
Hi,
The following patch adds support for the new KVM split irqchip
interface discussed recently on the KVM mailing list.
[kvm] KVM: x86: Split the APIC from the rest of IRQCHIP.
http://www.mailbrowse.com/kvm/138252.html
Our testing found some issues with the implementation of split IRQ
chip in the kernel, and without changes to address those it will not
be possible to test this. We've been able to do some preliminary
testing of the QEMU against our local kernel though and I'm able to
boot Linux and Windows with KVM_CAP_SPLIT_IRQCHIP enabled.
While we work on getting the KVM piece ready to submit, I'd appreciate
any feedback or discussion on the user space portion.
Thanks,
Matt Gingell
diff --git a/hw/core/machine.c b/hw/core/machine.c
index f4db340..3c14e78 100644
--- a/hw/core/machine.c
+++ b/hw/core/machine.c
@@ -11,6 +11,7 @@
*/
#include "hw/boards.h"
+#include "qapi-visit.h"
#include "qapi/visitor.h"
#include "hw/sysbus.h"
#include "sysemu/sysemu.h"
@@ -31,12 +32,34 @@ static void machine_set_accel(Object *obj, const char
*value, Error **errp)
ms->accel = g_strdup(value);
}
-static void machine_set_kernel_irqchip(Object *obj, bool value, Error **errp)
+static void machine_set_kernel_irqchip(Object *obj, Visitor *v,
+ void *opaque, const char *name,
+ Error **errp)
{
- MachineState *ms = MACHINE(obj);
-
- ms->kernel_irqchip_allowed = value;
- ms->kernel_irqchip_required = value;
+ OnOffSplit mode;
+ MachineState *ms = MACHINE(obj);
+
+ visit_type_OnOffSplit(v, &mode, name, errp);
+ switch (mode) {
+ case ON_OFF_SPLIT_ON:
+ ms->kernel_irqchip_allowed = true;
+ ms->kernel_irqchip_required = true;
+ ms->kernel_irqchip_split = false;
+ break;
+ case ON_OFF_SPLIT_OFF:
+ ms->kernel_irqchip_allowed = false;
+ ms->kernel_irqchip_required = false;
+ ms->kernel_irqchip_split = false;
+ break;
+ case ON_OFF_SPLIT_SPLIT:
+ ms->kernel_irqchip_allowed = true;
+ ms->kernel_irqchip_required = true;
+ ms->kernel_irqchip_split = true;
+ break;
+ default:
+ error_report("Option 'kernel-irqchip' must be one of on|off|split");
+ exit(1);
+ }
}
static void machine_get_kvm_shadow_mem(Object *obj, Visitor *v,
@@ -341,12 +364,12 @@ static void machine_initfn(Object *obj)
object_property_set_description(obj, "accel",
"Accelerator list",
NULL);
- object_property_add_bool(obj, "kernel-irqchip",
- NULL,
- machine_set_kernel_irqchip,
- NULL);
+ object_property_add(obj, "kernel-irqchip", "OnOffSplit",
+ NULL,
+ machine_set_kernel_irqchip,
+ NULL, NULL, NULL);
object_property_set_description(obj, "kernel-irqchip",
- "Use KVM in-kernel irqchip",
+ "Configure KVM in-kernel irqchip",
NULL);
object_property_add(obj, "kvm-shadow-mem", "int",
machine_get_kvm_shadow_mem,
@@ -477,6 +500,11 @@ bool machine_kernel_irqchip_required(MachineState *machine)
return machine->kernel_irqchip_required;
}
+bool machine_kernel_irqchip_split(MachineState *machine)
+{
+ return machine->kernel_irqchip_split;
+}
+
int machine_kvm_shadow_mem(MachineState *machine)
{
return machine->kvm_shadow_mem;
diff --git a/hw/i386/pc.c b/hw/i386/pc.c
index efbd41a..13db224 100644
--- a/hw/i386/pc.c
+++ b/hw/i386/pc.c
@@ -66,6 +66,7 @@
#include "hw/mem/pc-dimm.h"
#include "qapi/visitor.h"
#include "qapi-visit.h"
+#include "qom/cpu.h"
/* debug PC/ISA interrupts */
//#define DEBUG_IRQ
@@ -1530,10 +1531,11 @@ void pc_basic_device_init(ISABus *isa_bus, qemu_irq
*gsi,
qemu_register_boot_set(pc_boot_set, *rtc_state);
if (!xen_enabled()) {
- if (kvm_irqchip_in_kernel()) {
+ if (kvm_pit_in_kernel()) {
pit = kvm_pit_init(isa_bus, 0x40);
} else {
pit = pit_init(isa_bus, 0x40, pit_isa_irq, pit_alt_irq);
+ DPRINTF("Created software PIT.\n");
}
if (hpet) {
/* connect PIT to output control line of the HPET */
@@ -1605,10 +1607,11 @@ void ioapic_init_gsi(GSIState *gsi_state, const char
*parent_name)
SysBusDevice *d;
unsigned int i;
- if (kvm_irqchip_in_kernel()) {
+ if (kvm_ioapic_in_kernel()) {
dev = qdev_create(NULL, "kvm-ioapic");
} else {
dev = qdev_create(NULL, "ioapic");
+ DPRINTF("Created software ioapic.\n");
}
if (parent_name) {
object_property_add_child(object_resolve_path(parent_name, NULL),
diff --git a/hw/i386/pc_piix.c b/hw/i386/pc_piix.c
index 4514cd1..95b40d5 100644
--- a/hw/i386/pc_piix.c
+++ b/hw/i386/pc_piix.c
@@ -187,7 +187,7 @@ static void pc_init1(MachineState *machine,
}
gsi_state = g_malloc0(sizeof(*gsi_state));
- if (kvm_irqchip_in_kernel()) {
+ if (kvm_irqchip_in_kernel() && !kvm_irqchip_is_split()) {
kvm_pc_setup_irq_routing(pci_enabled);
gsi = qemu_allocate_irqs(kvm_pc_gsi_handler, gsi_state,
GSI_NUM_PINS);
@@ -211,7 +211,7 @@ static void pc_init1(MachineState *machine,
}
isa_bus_irqs(isa_bus, gsi);
- if (kvm_irqchip_in_kernel()) {
+ if (kvm_pic_in_kernel()) {
i8259 = kvm_i8259_init(isa_bus);
} else if (xen_enabled()) {
i8259 = xen_interrupt_controller_init();
diff --git a/hw/intc/ioapic.c b/hw/intc/ioapic.c
index de2dd4b..1dd96b6 100644
--- a/hw/intc/ioapic.c
+++ b/hw/intc/ioapic.c
@@ -25,6 +25,7 @@
#include "hw/i386/pc.h"
#include "hw/i386/ioapic.h"
#include "hw/i386/ioapic_internal.h"
+#include "include/hw/pci/msi.h"
//#define DEBUG_IOAPIC
@@ -35,6 +36,10 @@
#define DPRINTF(fmt, ...)
#endif
+#define APIC_DELIVERY_MODE_SHIFT = 8;
+#define APIC_POLARITY_SHIFT = 14;
+#define APIC_TRIG_MODE_SHIFT = 15;
+
static IOAPICCommonState *ioapics[MAX_IOAPICS];
/* global variable from ioapic_common.c */
@@ -54,6 +59,8 @@ static void ioapic_service(IOAPICCommonState *s)
for (i = 0; i < IOAPIC_NUM_PINS; i++) {
mask = 1 << i;
if (s->irr & mask) {
+ int coalesce = 0;
+
entry = s->ioredtbl[i];
if (!(entry & IOAPIC_LVT_MASKED)) {
trig_mode = ((entry >> IOAPIC_LVT_TRIGGER_MODE_SHIFT) & 1);
@@ -64,6 +71,7 @@ static void ioapic_service(IOAPICCommonState *s)
if (trig_mode == IOAPIC_TRIGGER_EDGE) {
s->irr &= ~mask;
} else {
+ coalesce = s->ioredtbl[i] & IOAPIC_LVT_REMOTE_IRR;
s->ioredtbl[i] |= IOAPIC_LVT_REMOTE_IRR;
}
if (delivery_mode == IOAPIC_DM_EXTINT) {
@@ -71,8 +79,19 @@ static void ioapic_service(IOAPICCommonState *s)
} else {
vector = entry & IOAPIC_VECTOR_MASK;
}
- apic_deliver_irq(dest, dest_mode, delivery_mode,
- vector, trig_mode);
+ if (kvm_irqchip_is_split()) {
+ if (trig_mode == IOAPIC_TRIGGER_EDGE) {
+ kvm_set_irq(kvm_state, i, 1);
+ kvm_set_irq(kvm_state, i, 0);
+ } else {
+ if (!coalesce) {
+ kvm_set_irq(kvm_state, i, 1);
+ }
+ }
+ } else {
+ apic_deliver_irq(dest, dest_mode, delivery_mode, vector,
+ trig_mode);
+ }
}
}
}
@@ -116,6 +135,42 @@ static void ioapic_set_irq(void *opaque, int vector, int
level)
}
}
+static void ioapic_update_kvm_routes(IOAPICCommonState *s)
+{
+ int i;
+
+ if (kvm_irqchip_is_split()) {
+ for (i = 0; i < IOAPIC_NUM_PINS; i++) {
+ uint64_t entry = s->ioredtbl[i];
+ uint8_t trig_mode;
+ uint8_t delivery_mode;
+ uint8_t dest;
+ uint8_t dest_mode;
+ uint64_t pin_polarity;
+ MSIMessage msg;
+
+ trig_mode = ((entry >> IOAPIC_LVT_TRIGGER_MODE_SHIFT) & 1);
+ dest = entry >> IOAPIC_LVT_DEST_SHIFT;
+ dest_mode = (entry >> IOAPIC_LVT_DEST_MODE_SHIFT) & 1;
+ pin_polarity = (entry >> IOAPIC_LVT_TRIGGER_MODE_SHIFT) & 1;
+ delivery_mode =
+ (entry >> IOAPIC_LVT_DELIV_MODE_SHIFT) & IOAPIC_DM_MASK;
+
+ msg.address = APIC_DEFAULT_ADDRESS;
+ msg.address |= dest_mode << 2;
+ msg.address |= dest << 12;
+
+ msg.data = entry & IOAPIC_VECTOR_MASK;
+ msg.data |= delivery_mode << APIC_DELIVERY_MODE_SHIFT;
+ msg.data |= pin_polarity << APIC_POLARITY_SHIFT;
+ msg.data |= trig_mode << APIC_TRIGGER_MODE_SHIFT;
+
+ kvm_irqchip_update_msi_route(kvm_state, i, msg);
+ }
+ kvm_irqchip_commit_routes(kvm_state);
+ }
+}
+
void ioapic_eoi_broadcast(int vector)
{
IOAPICCommonState *s;
@@ -229,6 +284,10 @@ ioapic_mem_write(void *opaque, hwaddr addr, uint64_t val,
}
break;
}
+
+ if (kvm_irqchip_is_split()) {
+ ioapic_update_kvm_routes(s);
+ }
}
static const MemoryRegionOps ioapic_io_ops = {
diff --git a/include/hw/boards.h b/include/hw/boards.h
index 3e9a92c..f647993 100644
--- a/include/hw/boards.h
+++ b/include/hw/boards.h
@@ -36,6 +36,7 @@ bool machine_usb(MachineState *machine);
bool machine_iommu(MachineState *machine);
bool machine_kernel_irqchip_allowed(MachineState *machine);
bool machine_kernel_irqchip_required(MachineState *machine);
+bool machine_kernel_irqchip_split(MachineState *machine);
int machine_kvm_shadow_mem(MachineState *machine);
int machine_phandle_start(MachineState *machine);
bool machine_dump_guest_core(MachineState *machine);
@@ -107,6 +108,7 @@ struct MachineState {
char *accel;
bool kernel_irqchip_allowed;
bool kernel_irqchip_required;
+ bool kernel_irqchip_split;
int kvm_shadow_mem;
char *dtb;
char *dumpdtb;
diff --git a/include/hw/i386/pc.h b/include/hw/i386/pc.h
index c13e91d..ca0137a 100644
--- a/include/hw/i386/pc.h
+++ b/include/hw/i386/pc.h
@@ -126,6 +126,17 @@ void hmp_info_irq(Monitor *mon, const QDict *qdict);
/* ioapic.c */
+#define kvm_pit_in_kernel() \
+ (kvm_irqchip_in_kernel() && !kvm_irqchip_is_split())
+#define kvm_pic_in_kernel() \
+ (kvm_irqchip_in_kernel() && !kvm_irqchip_is_split())
+#define kvm_ioapic_in_kernel() \
+ (kvm_irqchip_in_kernel() && !kvm_irqchip_is_split())
+#define kvm_apic_in_kernel() \
+ (kvm_irqchip_in_kernel())
+
+/* Global System Interrupts */
+
void kvm_ioapic_dump_state(Monitor *mon, const QDict *qdict);
void ioapic_dump_state(Monitor *mon, const QDict *qdict);
diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h
index 2a58b4d..0ceebd6 100644
--- a/include/sysemu/kvm.h
+++ b/include/sysemu/kvm.h
@@ -43,6 +43,7 @@
extern bool kvm_allowed;
extern bool kvm_kernel_irqchip;
+extern bool kvm_split_irqchip;
extern bool kvm_async_interrupts_allowed;
extern bool kvm_halt_in_kernel_allowed;
extern bool kvm_eventfds_allowed;
@@ -69,6 +70,16 @@ extern bool kvm_readonly_mem_allowed;
#define kvm_irqchip_in_kernel() (kvm_kernel_irqchip)
/**
+ * kvm_irqchip_is_split:
+ *
+ * Returns: true if the user asked us to split the irqchip
+ * implementation between user and kernel space. The details are
+ * architecture and machine specific. On PC, it means that the PIC,
+ * IOAPIC, and PIT are in user space while the LAPIC is in the kernel.
+ */
+#define kvm_irqchip_is_split() (kvm_split_irqchip)
+
+/**
* kvm_async_interrupts_enabled:
*
* Returns: true if we can deliver interrupts to KVM
@@ -287,6 +298,8 @@ MemTxAttrs kvm_arch_post_run(CPUState *cpu, struct kvm_run
*run);
int kvm_arch_handle_exit(CPUState *cpu, struct kvm_run *run);
+int kvm_arch_handle_ioapic_eoi(CPUState *cpu, struct kvm_run *run);
+
int kvm_arch_process_async_events(CPUState *cpu);
int kvm_arch_get_registers(CPUState *cpu);
diff --git a/kvm-all.c b/kvm-all.c
index 0be4615..69333b8 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -100,6 +100,7 @@ struct KVMState
KVMState *kvm_state;
bool kvm_kernel_irqchip;
+bool kvm_split_irqchip;
bool kvm_async_interrupts_allowed;
bool kvm_halt_in_kernel_allowed;
bool kvm_eventfds_allowed;
@@ -1389,19 +1390,30 @@ static void kvm_irqchip_create(MachineState *machine,
KVMState *s)
fprintf(stderr, "Enable kernel irqchip failed: %s\n",
strerror(-ret));
exit(1);
}
- } else {
- return;
}
- /* First probe and see if there's a arch-specific hook to create the
- * in-kernel irqchip for us */
- ret = kvm_arch_irqchip_create(s);
- if (ret == 0) {
- ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP);
- }
- if (ret < 0) {
- fprintf(stderr, "Create kernel irqchip failed: %s\n", strerror(-ret));
- exit(1);
+ if (machine_kernel_irqchip_split(machine)) {
+ ret = kvm_vm_enable_cap(s, KVM_CAP_SPLIT_IRQCHIP, 0, 24);
+ if (ret) {
+ fprintf(stderr, "Could not enable split irqchip mode: %s\n",
+ strerror(-ret));
+ exit(1);
+ } else {
+ DPRINTF("Enabled KVM_CAP_SPLIT_IRQCHIP\n");
+ kvm_split_irqchip = true;
+ }
+ } else {
+ /* First probe and see if there's a arch-specific hook to create the
+ * in-kernel irqchip for us */
+ ret = kvm_arch_irqchip_create(s);
+ if (ret == 0) {
+ ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP);
+ }
+ if (ret < 0) {
+ fprintf(stderr, "Create kernel irqchip failed: %s\n",
+ strerror(-ret));
+ exit(1);
+ }
}
kvm_kernel_irqchip = true;
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 683f713..26daafb 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -8,7 +8,7 @@
*/
#include <linux/types.h>
-
+#include <linux/compiler.h>
#include <linux/ioctl.h>
#include <asm/kvm.h>
@@ -183,6 +183,7 @@ struct kvm_s390_skeys {
#define KVM_EXIT_EPR 23
#define KVM_EXIT_SYSTEM_EVENT 24
#define KVM_EXIT_S390_STSI 25
+#define KVM_EXIT_IOAPIC_EOI 26
/* For KVM_EXIT_INTERNAL_ERROR */
/* Emulate instruction failed. */
@@ -330,6 +331,10 @@ struct kvm_run {
__u8 sel1;
__u16 sel2;
} s390_stsi;
+ /* KVM_EXIT_IOAPIC_EOI */
+ struct {
+ __u8 vector;
+ } eoi;
/* Fix the size of the union. */
char padding[256];
};
@@ -415,7 +420,7 @@ struct kvm_dirty_log {
__u32 slot;
__u32 padding1;
union {
- void *dirty_bitmap; /* one bit per page */
+ void __user *dirty_bitmap; /* one bit per page */
__u64 padding2;
};
};
@@ -819,6 +824,7 @@ struct kvm_ppc_smmu_info {
#define KVM_CAP_DISABLE_QUIRKS 116
#define KVM_CAP_X86_SMM 117
#define KVM_CAP_MULTI_ADDRESS_SPACE 118
+#define KVM_CAP_SPLIT_IRQCHIP 119
#ifdef KVM_CAP_IRQ_ROUTING
diff --git a/qapi/common.json b/qapi/common.json
index bad56bf..06cc724 100644
--- a/qapi/common.json
+++ b/qapi/common.json
@@ -114,3 +114,19 @@
##
{ 'enum': 'OnOffAuto',
'data': [ 'auto', 'on', 'off' ] }
+
+##
+# @OnOffSplit
+#
+# An enumeration of three values: on, off, and split
+#
+# @on: Enabled
+#
+# @off: Disabled
+#
+# @split: Mixed
+#
+# Since: 2.5 (???)
+##
+{ 'enum': 'OnOffSplit',
+ 'data': [ 'on', 'off', 'split' ] }
diff --git a/qemu-options.hx b/qemu-options.hx
index 328404c..b1301ad 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -33,6 +33,7 @@ DEF("machine", HAS_ARG, QEMU_OPTION_machine, \
" property accel=accel1[:accel2[:...]] selects
accelerator\n"
" supported accelerators are kvm, xen, tcg (default: tcg)\n"
" kernel_irqchip=on|off controls accelerated irqchip
support\n"
+ " kernel_irqchip=on|off|split controls accelerated irqchip
support (default=off)\n"
" vmport=on|off|auto controls emulation of vmport (default:
auto)\n"
" kvm_shadow_mem=size of KVM shadow MMU\n"
" dump-guest-core=on|off include guest memory in a core
dump (default=on)\n"
@@ -54,10 +55,15 @@ This is used to enable an accelerator. Depending on the
target architecture,
kvm, xen, or tcg can be available. By default, tcg is used. If there is more
than one accelerator specified, the next one is used if the previous one fails
to initialize.
+<<<<<<<
@item kernel_irqchip=on|off
Enables in-kernel irqchip support for the chosen accelerator when available.
@item gfx_passthru=on|off
Enables IGD GFX passthrough support for the chosen machine when available.
+=======
address@hidden kernel_irqchip=on|off|split
+Controls in-kernel irqchip support for the chosen accelerator when available.
+>>>>>>>
@item vmport=on|off|auto
Enables emulation of VMWare IO port, for vmmouse etc. auto says to select the
value based on accel. For accel=xen the default is off otherwise the default
diff --git a/target-i386/cpu.c b/target-i386/cpu.c
index bd411b9..f608c8e 100644
--- a/target-i386/cpu.c
+++ b/target-i386/cpu.c
@@ -22,6 +22,7 @@
#include <inttypes.h>
#include "cpu.h"
+#include "hw/i386/pc.h"
#include "sysemu/kvm.h"
#include "sysemu/cpus.h"
#include "kvm_i386.h"
@@ -2727,7 +2728,7 @@ static void x86_cpu_apic_create(X86CPU *cpu, Error **errp)
APICCommonState *apic;
const char *apic_type = "apic";
- if (kvm_irqchip_in_kernel()) {
+ if (kvm_apic_in_kernel()) {
apic_type = "kvm-apic";
} else if (xen_enabled()) {
apic_type = "xen-apic";
diff --git a/target-i386/kvm.c b/target-i386/kvm.c
index 7b0ba17..91dedad 100644
--- a/target-i386/kvm.c
+++ b/target-i386/kvm.c
@@ -35,6 +35,7 @@
#include "exec/ioport.h"
#include "standard-headers/asm-x86/hyperv.h"
#include "hw/pci/pci.h"
+#include "hw/pci/msi.h"
#include "migration/migration.h"
#include "exec/memattrs.h"
@@ -2348,7 +2349,7 @@ void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run)
}
}
- if (!kvm_irqchip_in_kernel()) {
+ if (!kvm_pic_in_kernel()) {
qemu_mutex_lock_iothread();
}
@@ -2366,7 +2367,8 @@ void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run)
}
}
- if (!kvm_irqchip_in_kernel()) {
+ if (!kvm_pic_in_kernel()) {
+
/* Try to inject an interrupt if the guest can accept it */
if (run->ready_for_interrupt_injection &&
(cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
@@ -2765,6 +2767,10 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run
*run)
ret = kvm_handle_debug(cpu, &run->debug.arch);
qemu_mutex_unlock_iothread();
break;
+ case KVM_EXIT_IOAPIC_EOI:
+ ioapic_eoi_broadcast(run->eoi.vector);
+ ret = 0;
+ break;
default:
fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason);
ret = -1;
@@ -2799,6 +2805,20 @@ void kvm_arch_init_irq_routing(KVMState *s)
*/
kvm_msi_via_irqfd_allowed = true;
kvm_gsi_routing_allowed = true;
+
+ if (kvm_irqchip_is_split()) {
+ int i;
+
+ /* If the ioapic is in QEMU and the lapics are in KVM, reserve
+ MSI routes for signaling interrupts to the local apics. */
+ for (i = 0; i < IOAPIC_NUM_PINS; i++) {
+ struct MSIMessage msg = { 0x0, 0x0 };
+ if (kvm_irqchip_add_msi_route(s, msg) < 0) {
+ fprintf(stderr, "Could not enable split IRQ mode.");
+ exit(-1);
+ }
+ }
+ }
}
/* Classic KVM device assignment interface. Will remain x86 only. */
- [Qemu-devel] RFC: Add support for KVM_CAP_SPLIT_IRQCHIP,
Matt Gingell <=