Re: Windows 10 passthrough lag/micro-freeze issues during disk I/O

[Peter Geis]

Have you tried io threading, dedicating a core to the host, and
pinning the io threads to that core?
The same core should be dedicated to qemu's overhead.

On Fri, Mar 20, 2020 at 3:59 PM Th3R3al <address@hidden> wrote:
>
> Hello Jérôme,
>
> just want to give a +1. But have the exact issue you described with
> network i/o. If i try to transfer a big file from a samba share (at
> around 40MB/s~) the VM is freezing. Nearly identical configuration to yours.
>
> Greetings
>
> Th3R3al
>
> Am 20.03.20 um 20:46 schrieb Jérôme Poulin:
> > Hi everyone,
> >
> > I have a VM right now that is setup as my gaming machine with video
> > pass-through, it works perfectly except when there's I/O on the host.
> >
> > When there is I/O, the VM will have micro-freeze events where
> > everything locks up for milliseconds up to a second at a time at
> > frequent intervals, the host is on NVMe, CPUs are pinned with a
> > dedicated core for emulator.
> >
> > Whatever I put, virtio, virtio-scsi, cache=none, cache=writeback,
> > threads or native, as soon as there is I/O, I get the same problem, to
> > the point where it causes disconnections on the virtual USB bus.
> >
> > VM is setup with hugepages, CPU pinning, I tried making qemu threads
> > realtime with no success (even blue screens). I tried with dedicated
> > cores (isolcpus), i440/q35, now I'm out of ideas. I do run a custom
> > kernel to be able to use pcie_acs_override=downstream,multifunction in
> > GRUB because I can't use passthrough otherwise.
> >
> > This problem has been following me from my old setup to my new setup
> > Old setup:
> > ASUS B150 PLUS USB3
> > Core i7 6700K
> > 16 GB DDR3
> > SATA raw LVM volume for Windows
> >
> > New setup
> > ASRock Z390 Taichi
> > Core i7 9700K
> > 48 GB DDR4
> > NVMe raw LVM volume for Windows
> >
> > Any ideas would be appreciated.
> >
> > Here is the libvirt XML definition (I excluded the buses):
> > <domain type='kvm'>
> >   <name>win10-jerome</name>
> >   <uuid>fc19f3c1-b8c6-4184-9b43-159157eff5b5</uuid>
> >   <memory unit='KiB'>15360000</memory>
> >   <currentMemory unit='KiB'>15360000</currentMemory>
> >   <memoryBacking>
> >     <hugepages/>
> >     <nosharepages/>
> >     <locked/>
> >   </memoryBacking>
> >   <vcpu placement='static'>4</vcpu>
> >   <cputune>
> >     <vcpupin vcpu='0' cpuset='0'/>
> >     <vcpupin vcpu='1' cpuset='1'/>
> >     <vcpupin vcpu='2' cpuset='2'/>
> >     <vcpupin vcpu='3' cpuset='3'/>
> >     <emulatorpin cpuset='7'/>
> >   </cputune>
> >   <resource>
> >     <partition>/machine</partition>
> >   </resource>
> >   <os>
> >     <type arch='x86_64' machine='pc-q35-4.0'>hvm</type>
> >     <boot dev='hd'/>
> >     <bootmenu enable='yes'/>
> >   </os>
> >   <features>
> >     <acpi/>
> >     <apic/>
> >     <hyperv>
> >       <relaxed state='on'/>
> >       <vapic state='on'/>
> >       <spinlocks state='on' retries='8191'/>
> >       <vpindex state='on'/>
> >       <runtime state='on'/>
> >       <synic state='on'/>
> >       <stimer state='on'/>
> >       <reset state='on'/>
> >       <frequencies state='on'/>
> >     </hyperv>
> >     <ioapic driver='kvm'/>
> >   </features>
> >   <cpu mode='host-passthrough' check='none'>
> >     <topology sockets='1' cores='4' threads='1'/>
> >     <feature policy='require' name='invtsc'/>
> >   </cpu>
> >   <clock offset='utc'>
> >     <timer name='hypervclock' present='yes'/>
> >     <timer name='hpet' present='yes'/>
> >   </clock>
> >   <devices>
> >     <emulator>/usr/bin/kvm-spice</emulator>
> >     <disk type='block' device='disk'>
> >       <driver name='qemu' type='raw' cache='none' io='native'/>
> >       <source dev='/dev/vgP4Root/win10jerome'/>
> >       <backingStore/>
> >       <target dev='sda' bus='sata'/>
> >       <address type='drive' controller='0' bus='0' target='0' unit='0'/>
> >     </disk>
> >     <disk type='block' device='disk'>
> >       <driver name='qemu' type='raw' cache='none' io='native'
> > discard='ignore' detect_zeroes='off'/>
> >       <source dev='/dev/vgP4Root/win10data'/>
> >       <target dev='sdb' bus='scsi'/>
> >       <address type='drive' controller='0' bus='0' target='0' unit='1'/>
> >     </disk>
> >     <disk type='block' device='disk'>
> >       <driver name='qemu' type='raw' cache='none' io='native'
> > discard='ignore' detect_zeroes='off'/>
> >       <source dev='/dev/vgP4Root/win10swap'/>
> >       <target dev='sdd' bus='scsi'/>
> >       <address type='drive' controller='0' bus='0' target='0' unit='3'/>
> >     </disk>
> >     <interface type='direct'>
> >       <mac address='52:54:00:31:1e:f9'/>
> >       <source dev='eth0' mode='bridge'/>
> >       <model type='virtio'/>
> >       <address type='pci' domain='0x0000' bus='0x04' slot='0x00'
> > function='0x0'/>
> >     </interface>
> >     <input type='mouse' bus='ps2'/>
> >     <input type='keyboard' bus='ps2'/>
> >     <graphics type='spice' autoport='yes' listen='127.0.0.1'>
> >       <listen type='address' address='127.0.0.1'/>
> >     </graphics>
> >     <video>
> >       <model type='qxl' ram='65536' vram='16384' vgamem='16384'
> > heads='1' primary='yes'/>
> >       <address type='pci' domain='0x00    <hostdev mode='subsystem'
> > type='pci' managed='yes'>
> >     </video>
> >     <hostdev mode='subsystem' type='pci' managed='yes'>
> >       <source>
> >         <address domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
> >       </source>
> >       <rom bar='on'/>
> >       <address type='pci' domain='0x0000' bus='0x02' slot='0x00'
> > function='0x0' multifunction='on'/>
> >     </hostdev>
> >     <hostdev mode='subsystem' type='pci' managed='yes'>
> >       <source>
> >         <address domain='0x0000' bus='0x02' slot='0x00' function='0x1'/>
> >       </source>
> >       <rom bar='on'/>
> >       <address type='pci' domain='0x0000' bus='0x02' slot='0x00'
> > function='0x1'/>
> >     </hostdev>
> > 00' bus='0x00' slot='0x01' function='0x0'/>
> >     <rng model='virtio'>
> >       <backend model='random'>/dev/urandom</backend>
> >       <address type='pci' domain='0x0000' bus='0x07' slot='0x00'
> > function='0x0'/>
> >     </rng>
> >   </devices>
> > </domain>
> >
> > Generated command line:
> > qemu-system-x86_64 -enable-kvm -name
> > guest=win10-jerome,debug-threads=on -S -object
> > secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-22-win10-jerome/master-key.aes
> > -machine
> > pc-q35-4.0,accel=kvm,usb=off,dump-guest-core=off,mem-merge=off,kernel_irqchip=on
> > -cpu
> > host,invtsc=on,hv_time,hv_relaxed,hv_vapic,hv_spinlocks=0x1fff,hv_vpindex,hv_runtime,hv_synic,hv_stimer,hv_reset,hv_frequencies
> > -m 15000 -mem-prealloc -mem-path
> > /dev/hugepages/libvirt/qemu/22-win10-jerome -overcommit mem-lock=on
> > -smp 4,sockets=1,cores=4,threads=1 -uuid
> > fc19f3c1-b8c6-4184-9b43-159157eff5b5 -no-user-config -nodefaults
> > -chardev socket,id=charmonitor,fd=25,server,nowait -mon
> > chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-shutdown
> > -global ICH9-LPC.disable_s3=0 -global ICH9-LPC.disable_s4=0 -boot
> > menu=on,strict=on -device
> > pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x2
> > -device
> > pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1
> > -device
> > pcie-root-port,port=0x12,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2
> > -device
> > pcie-root-port,port=0x13,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3
> > -device
> > pcie-root-port,port=0x14,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4
> > -device
> > pcie-root-port,port=0x15,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5
> > -device
> > pcie-root-port,port=0x16,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6
> > -device
> > pcie-root-port,port=0x17,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x7
> > -device i82801b11-bridge,id=pci.9,bus=pcie.0,addr=0x1e -device
> > pci-bridge,chassis_nr=10,id=pci.10,bus=pci.9,addr=0x0 -device
> > ich9-usb-ehci1,id=usb,bus=pcie.0,addr=0x1d.0x7 -device
> > ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pcie.0,multifunction=on,addr=0x1d
> > -device
> > ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pcie.0,addr=0x1d.0x1
> > -device
> > ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pcie.0,addr=0x1d.0x2
> > -device virtio-scsi-pci,id=scsi0,bus=pci.5,addr=0x0 -device
> > virtio-serial-pci,id=virtio-serial0,bus=pci.1,addr=0x0 -drive
> > file=/dev/vgP4Root/win10jerome,format=raw,if=none,id=drive-sata0-0-0,cache=none,aio=native
> > -device
> > ide-hd,bus=ide.0,drive=drive-sata0-0-0,id=sata0-0-0,bootindex=1,write-cache=on
> > -drive
> > file=/dev/vgP4Root/win10data,format=raw,if=none,id=drive-scsi0-0-0-1,cache=none,discard=ignore,detect-zeroes=off,aio=native
> > -device
> > scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=1,device_id=drive-scsi0-0-0-1,drive=drive-scsi0-0-0-1,id=scsi0-0-0-1,write-cache=on
> > -drive
> > file=/dev/vgP4Root/win10swap,format=raw,if=none,id=drive-scsi0-0-0-3,cache=none,discard=ignore,detect-zeroes=off,aio=native
> > -device
> > scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=3,device_id=drive-scsi0-0-0-3,drive=drive-scsi0-0-0-3,id=scsi0-0-0-3,write-cache=on
> > -netdev tap,fd=27,id=hostnet0,vhost=on,vhostfd=28 -device
> > virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:31:1e:f9,bus=pci.4,addr=0x0
> > -spice
> > port=5901,addr=127.0.0.1,disable-ticketing,seamless-migration=on
> > -device
> > qxl-vga,id=video0,ram_size=67108864,vram_size=16777216,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pcie.0,addr=0x1
> > -chardev spicevmc,id=charredir0,name=usbredir -device
> > usb-redir,chardev=charredir0,id=redir0,bus=usb.0,port=2 -chardev
> > spicevmc,id=charredir1,name=usbredir -device
> > usb-redir,chardev=charredir1,id=redir1,bus=usb.0,port=3 -device
> > vfio-pci,host=02:00.0,id=hostdev0,bus=pci.2,multifunction=on,addr=0x0,rombar=1
> > -device
> > vfio-pci,host=02:00.1,id=hostdev1,bus=pci.2,addr=0x0.0x1,rombar=1
> > -device usb-host,hostbus=1,hostaddr=7,id=hostdev2,bus=usb.0,port=5
> > -device usb-host,hostbus=1,hostaddr=56,id=hostdev3,bus=usb.0,port=1
> > -device usb-host,hostbus=1,hostaddr=25,id=hostdev4,bus=usb.0,port=4
> > -device virtio-balloon-pci,id=balloon0,bus=pci.3,addr=0x0 -object
> > rng-random,id=objrng0,filename=/dev/urandom -device
> > virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.7,addr=0x0 -sandbox
> > on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny
> > -msg timestamp=on
>