When hotplugging vCPUs to the Windows vms, we observed strange instance
crash on Intel(R) Xeon(R) CPU E3-1230 v6:
panic hyper-v: arg1='0x3e', arg2='0x46d359bbdff', arg3='0x56d359bbdff',
arg4='0x0', arg5='0x0'
Presumably, Windows thinks that hotplugged CPU is not "equivalent enough"
to the previous ones. The problem lies within msr 3a. During the startup,
Windows assigns some value to this register. During the hotplug it
expects similar value on the new vCPU in msr 3a. But by default it
is zero.
This patch introduces new CPU option: QEMU will copy msr 3a value from
the first vCPU during the hotplug. This problem may not be limited to
SGX feature, so the whole register is copied.
By default the option is set to auto and hyper-v is used as Windows
indicator to enable this new feature.
Resolves: #2669
Signed-off-by: Andrey Zhadchenko <andrey.zhadchenko@virtuozzo.com>
---
target/i386/cpu.c | 2 ++
target/i386/cpu.h | 3 +++
target/i386/kvm/kvm.c | 43 +++++++++++++++++++++++++++++++++++++++++++
3 files changed, 48 insertions(+)
diff --git a/target/i386/cpu.c b/target/i386/cpu.c
index 40aefb38f6..5c02f0962d 100644
--- a/target/i386/cpu.c
+++ b/target/i386/cpu.c
@@ -9389,6 +9389,8 @@ static const Property x86_cpu_properties[] = {
DEFINE_PROP_BOOL("x-intel-pt-auto-level", X86CPU, intel_pt_auto_level,
true),
DEFINE_PROP_BOOL("x-l1-cache-per-thread", X86CPU, l1_cache_per_core,
true),
+ DEFINE_PROP_ON_OFF_AUTO("kvm-win-hack-sgx-cpu-hotplug", X86CPU,
+ kvm_win_hack_sgx_cpu_hotplug, ON_OFF_AUTO_AUTO),
};
#ifndef CONFIG_USER_ONLY
diff --git a/target/i386/cpu.h b/target/i386/cpu.h
index 545851cbde..0505d3d1cd 100644
--- a/target/i386/cpu.h
+++ b/target/i386/cpu.h
@@ -2301,6 +2301,9 @@ struct ArchCPU {
/* Forcefully disable KVM PV features not exposed in guest CPUIDs */
bool kvm_pv_enforce_cpuid;
+ /* Copy msr 3a on cpu hotplug */
+ OnOffAuto kvm_win_hack_sgx_cpu_hotplug;
+
/* Number of physical address bits supported */
uint32_t phys_bits;
diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c
index 56a6b9b638..c1e7d15e2e 100644
--- a/target/i386/kvm/kvm.c
+++ b/target/i386/kvm/kvm.c
@@ -5266,6 +5266,42 @@ static int kvm_get_nested_state(X86CPU *cpu)
return ret;
}
+static int kvm_win_hack_hotplug_with_sgx(CPUState *cs)
+{
+ DeviceState *dev = DEVICE(cs);
+ X86CPU *cpu = X86_CPU(cs);
+ int ret;
+
+ /*
+ * If CPU supports Intel SGX, Windows guests expect readmsr 0x3a after
+ * hotplug to have some bits set, just like on other vCPUs. Unfortunately
+ * by default it is zero and other vCPUs registers are filled by Windows
+ * itself during startup.
+ * Just copy the value from another vCPU.
+ */
+
+ if (cpu->kvm_win_hack_sgx_cpu_hotplug == ON_OFF_AUTO_OFF ||
+ (cpu->kvm_win_hack_sgx_cpu_hotplug == ON_OFF_AUTO_AUTO &&
+ !hyperv_enabled(cpu))) {
+ return 0;
+ }
+
+ if (cpu->env.msr_ia32_feature_control) {
+ return 0;
+ }
+
+ if (IS_INTEL_CPU(&cpu->env) && dev->hotplugged && first_cpu) {
+ ret = kvm_get_one_msr(X86_CPU(first_cpu),
+ MSR_IA32_FEATURE_CONTROL,
+ &cpu->env.msr_ia32_feature_control);
+ if (ret != 1) {
+ return ret;
+ }
+ }
+
+ return 0;
+}
+
int kvm_arch_put_registers(CPUState *cpu, int level, Error **errp)
{
X86CPU *x86_cpu = X86_CPU(cpu);
@@ -5273,6 +5309,13 @@ int kvm_arch_put_registers(CPUState *cpu, int level,
Error **errp)
assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
+ if (level == KVM_PUT_FULL_STATE) {
+ ret = kvm_win_hack_hotplug_with_sgx(cpu);
+ if (ret < 0) {
+ return ret;
+ }
+ }
+
/*
* Put MSR_IA32_FEATURE_CONTROL first, this ensures the VM gets out of VMX
* root operation upon vCPU reset. kvm_put_msr_feature_control() should
also