virtio-net-failover intermittent test hangs eating CPU on s390 host

[Peter Maydell]

Found a couple of virtio-net-failover test processes sat on the
s390 CI runner with the virtio-net-failover process eating CPU.
Backtrace (I captured from both, but the backtraces are the same
in both cases):


Process tree:
virtio-net-fail(3435488)---qemu-system-i38(3435776)
===========================================================
PROCESS: 3435488
gitlab-+ 3435488 3415953 24 Mar30 ?        04:01:46
/home/gitlab-runner/builds/-LCfcJ2T/0/qemu-project/qemu/build/tests/qtest/virtio-net-failover
--tap -k
[New LWP 3435489]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/s390x-linux-gnu/libthread_db.so.1".
__libc_send (fd=fd@entry=3, buf=buf@entry=0x2aa08e5f5c0,
len=len@entry=29, flags=flags@entry=0) at
../sysdeps/unix/sysv/linux/send.c:30
30      ../sysdeps/unix/sysv/linux/send.c: No such file or directory.

Thread 2 (Thread 0x3ffb25ff900 (LWP 3435489)):
#0  syscall () at ../sysdeps/unix/sysv/linux/s390/s390-64/syscall.S:37
#1  0x000002aa086d9cf4 in qemu_futex_wait (val=<optimized out>,
f=<optimized out>) at
/home/gitlab-runner/builds/-LCfcJ2T/0/qemu-project/qemu/include/qemu/futex.h:29
#2  qemu_event_wait (ev=ev@entry=0x2aa0874b890 <rcu_call_ready_event>)
at ../util/qemu-thread-posix.c:464
#3  0x000002aa08705e82 in call_rcu_thread (opaque=opaque@entry=0x0) at
../util/rcu.c:261
#4  0x000002aa086d8d5a in qemu_thread_start (args=<optimized out>) at
../util/qemu-thread-posix.c:541
#5  0x000003ffb2887e66 in start_thread (arg=0x3ffb25ff900) at
pthread_create.c:477
#6  0x000003ffb277cbe6 in thread_start () at
../sysdeps/unix/sysv/linux/s390/s390-64/clone.S:65

Thread 1 (Thread 0x3ffb2cf2770 (LWP 3435488)):
#0  __libc_send (fd=fd@entry=3, buf=buf@entry=0x2aa08e5f5c0,
len=len@entry=29, flags=flags@entry=0) at
../sysdeps/unix/sysv/linux/send.c:30
#1  0x000002aa086d5878 in qemu_send_full (s=s@entry=3,
buf=0x2aa08e5f5c0, count=count@entry=29) at ../util/osdep.c:509
#2  0x000002aa086aab8a in socket_send (size=<optimized out>,
buf=<optimized out>, fd=3) at ../tests/qtest/libqmp.c:172
#3  _qmp_fd_vsend_fds (fd=<optimized out>, fds=<optimized out>,
fds@entry=0x0, fds_num=fds_num@entry=0, fmt=<optimized out>,
ap=ap@entry=0x3ffd0679f00) at ../tests/qtest/libqmp.c:172
#4  0x000002aa086aaf72 in qmp_fd_vsend (fd=<optimized out>,
fmt=<optimized out>, ap=ap@entry=0x3ffd0679f00) at
../tests/qtest/libqmp.c:190
#5  0x000002aa086a886c in qtest_qmp_vsend (ap=0x3ffd0679f00,
fmt=<optimized out>, s=0x2aa08e63d70) at ../tests/qtest/libqtest.c:788
#6  qtest_vqmp (ap=0x3ffd0679f00, fmt=<optimized out>,
s=0x2aa08e63d70) at ../tests/qtest/libqtest.c:762
#7  qtest_qmp (s=0x2aa08e63d70, fmt=<optimized out>) at
../tests/qtest/libqtest.c:788
#8  0x000002aa086911d0 in migrate_status (qts=<optimized out>) at
../tests/qtest/virtio-net-failover.c:596
#9  0x000002aa0869cee0 in test_migrate_off_abort (opaque=<optimized
out>) at ../tests/qtest/virtio-net-failover.c:1425
#10 0x000003ffb2a7e608 in ?? () from /lib/s390x-linux-gnu/libglib-2.0.so.0
#11 0x000003ffb2a7e392 in ?? () from /lib/s390x-linux-gnu/libglib-2.0.so.0
#12 0x000003ffb2a7e392 in ?? () from /lib/s390x-linux-gnu/libglib-2.0.so.0
#13 0x000003ffb2a7e392 in ?? () from /lib/s390x-linux-gnu/libglib-2.0.so.0
#14 0x000003ffb2a7e392 in ?? () from /lib/s390x-linux-gnu/libglib-2.0.so.0
#15 0x000003ffb2a7eada in g_test_run_suite () from
/lib/s390x-linux-gnu/libglib-2.0.so.0
#16 0x000003ffb2a7eb10 in g_test_run () from
/lib/s390x-linux-gnu/libglib-2.0.so.0
#17 0x000002aa086905e2 in main (argc=<optimized out>, argv=<optimized
out>) at ../tests/qtest/virtio-net-failover.c:1897
[Inferior 1 (process 3435488) detached]

===========================================================
PROCESS: 3435776
gitlab-+ 3435776 3435488 18 Mar30 ?        03:04:00 ./qemu-system-i386
-qtest unix:/tmp/qtest-3435488.sock -qtest-log /dev/null -chardev
socket,path=/tmp/qtest-3435488.qmp,id=char0 -mon
chardev=char0,mode=control -display none -M q35 -nodefaults -device
pcie-root-port,id=root0,addr=0x1,bus=pcie.0,chassis=1 -device
pcie-root-port,id=root1,addr=0x2,bus=pcie.0,chassis=2 -netdev
user,id=hs0 -netdev user,id=hs1 -accel qtest
[New LWP 3435778]
[New LWP 3435779]
[New LWP 3435780]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/s390x-linux-gnu/libthread_db.so.1".
0x000003ff8e871c8c in __ppoll (fds=0x2aa37996d80, nfds=5,
timeout=<optimized out>, timeout@entry=0x3ffcf0fa428,
sigmask=sigmask@entry=0x0) at ../sysdeps/unix/sysv/linux/ppoll.c:44
44      ../sysdeps/unix/sysv/linux/ppoll.c: No such file or directory.

Thread 4 (Thread 0x3ff7e9a0900 (LWP 3435780)):
#0  futex_wait_cancelable (private=0, expected=0,
futex_word=0x2aa3789d928) at ../sysdeps/nptl/futex-internal.h:183
#1  __pthread_cond_wait_common (abstime=0x0, clockid=0,
mutex=0x2aa36313260 <qemu_global_mutex>, cond=0x2aa3789d900) at
pthread_cond_wait.c:508
#2  __pthread_cond_wait (cond=cond@entry=0x2aa3789d900,
mutex=mutex@entry=0x2aa36313260 <qemu_global_mutex>) at
pthread_cond_wait.c:647
#3  0x000002aa35a3d4be in qemu_cond_wait_impl (cond=0x2aa3789d900,
mutex=0x2aa36313260 <qemu_global_mutex>, file=0x2aa35b84c4c
"../softmmu/cpus.c", line=<optimized out>) at
../util/qemu-thread-posix.c:225
#4  0x000002aa3566df2e in qemu_wait_io_event
(cpu=cpu@entry=0x2aa37897350) at ../softmmu/cpus.c:424
#5  0x000002aa356df704 in dummy_cpu_thread_fn
(arg=arg@entry=0x2aa37897350) at ../accel/dummy-cpus.c:60
#6  0x000002aa35a3cada in qemu_thread_start (args=<optimized out>) at
../util/qemu-thread-posix.c:541
#7  0x000003ff8e987e66 in start_thread (arg=0x3ff7e9a0900) at
pthread_create.c:477
#8  0x000003ff8e87cbe6 in thread_start () at
../sysdeps/unix/sysv/linux/s390/s390-64/clone.S:65

Thread 3 (Thread 0x3ff7f1a1900 (LWP 3435779)):
#0  0x000003ff8e871b32 in __GI___poll (fds=0x3ff700032a0, nfds=3,
timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:29
#1  0x000003ff91054386 in  () at /lib/s390x-linux-gnu/libglib-2.0.so.0
#2  0x000003ff91054790 in g_main_loop_run () at
/lib/s390x-linux-gnu/libglib-2.0.so.0
#3  0x000002aa358ecffe in iothread_run
(opaque=opaque@entry=0x2aa376c2f00) at ../iothread.c:70
#4  0x000002aa35a3cada in qemu_thread_start (args=<optimized out>) at
../util/qemu-thread-posix.c:541
#5  0x000003ff8e987e66 in start_thread (arg=0x3ff7f1a1900) at
pthread_create.c:477
#6  0x000003ff8e87cbe6 in thread_start () at
../sysdeps/unix/sysv/linux/s390/s390-64/clone.S:65

Thread 2 (Thread 0x3ff7faa3900 (LWP 3435778)):
#0  syscall () at ../sysdeps/unix/sysv/linux/s390/s390-64/syscall.S:37
#1  0x000002aa35a3da74 in qemu_futex_wait (val=<optimized out>,
f=<optimized out>) at
/home/gitlab-runner/builds/-LCfcJ2T/0/qemu-project/qemu/include/qemu/futex.h:29
#2  qemu_event_wait (ev=ev@entry=0x2aa36343548 <rcu_call_ready_event>)
at ../util/qemu-thread-posix.c:464
#3  0x000002aa35a4786a in call_rcu_thread (opaque=opaque@entry=0x0) at
../util/rcu.c:261
#4  0x000002aa35a3cada in qemu_thread_start (args=<optimized out>) at
../util/qemu-thread-posix.c:541
#5  0x000003ff8e987e66 in start_thread (arg=0x3ff7faa3900) at
pthread_create.c:477
#6  0x000003ff8e87cbe6 in thread_start () at
../sysdeps/unix/sysv/linux/s390/s390-64/clone.S:65

Thread 1 (Thread 0x3ff921fe4a0 (LWP 3435776)):
#0  0x000003ff8e871c8c in __ppoll (fds=0x2aa37996d80, nfds=5,
timeout=<optimized out>, timeout@entry=0x3ffcf0fa428,
sigmask=sigmask@entry=0x0) at ../sysdeps/unix/sysv/linux/ppoll.c:44
#1  0x000002aa35a53aa2 in ppoll (__ss=0x0, __timeout=0x3ffcf0fa428,
__nfds=<optimized out>, __fds=<optimized out>) at
/usr/include/s390x-linux-gnu/bits/poll2.h:77
#2  qemu_poll_ns (fds=<optimized out>, nfds=<optimized out>,
timeout=timeout@entry=1000000000) at ../util/qemu-timer.c:351
#3  0x000002aa35a50ab8 in os_host_main_loop_wait (timeout=1000000000)
at ../util/main-loop.c:308
#4  main_loop_wait (nonblocking=nonblocking@entry=0) at ../util/main-loop.c:592
#5  0x000002aa35674b04 in qemu_main_loop () at ../softmmu/runstate.c:731
#6  0x000002aa3548df9a in qemu_default_main () at ../softmmu/main.c:37
#7  0x000003ff8e7a440a in __libc_start_main (main=0x2aa3548b560
<main>, argc=<optimized out>, argv=0x3ffcf0fa878, init=<optimized
out>, fini=<optimized out>, rtld_fini=0x3ff92210e50 <_dl_fini>,
stack_end=0x3ffcf0fa7c0) at libc-start.c:308
#8  0x000002aa3548dec4 in _start () at ../softmmu/main.c:48
[Inferior 1 (process 3435776) detached]

thanks
-- PMM