Re: [PATCH V2 17/21] vhost-net: control virtqueue support

[Jason Wang]

On Sun, Sep 5, 2021 at 4:40 AM Michael S. Tsirkin <mst@redhat.com> wrote:
>
> On Fri, Sep 03, 2021 at 05:10:27PM +0800, Jason Wang wrote:
> > We assume there's no cvq in the past, this is not true when we need
> > control virtqueue support for vhost-user backends. So this patch
> > implements the control virtqueue support for vhost-net. As datapath,
> > the control virtqueue is also required to be coupled with the
> > NetClientState. The vhost_net_start/stop() are tweaked to accept the
> > number of datapath queue pairs plus the the number of control
> > virtqueue for us to start and stop the vhost device.
> >
> > Signed-off-by: Jason Wang <jasowang@redhat.com>
>
>
> Fails build:
>
> FAILED: libcommon.fa.p/hw_net_vhost_net-stub.c.o
> cc -Ilibcommon.fa.p -I. -Iqapi -Itrace -Iui -Iui/shader 
> -I/usr/include/spice-1 -I/usr/include/spice-server -I/usr/include/cacard 
> -I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include -I/usr/include/nss3 
> -I/usr/include/nspr4 -I/usr/include/libmount -I/usr/include/blkid 
> -I/usr/include/pixman-1 -I/usr/include/p11-kit-1 -I/usr/include/SDL2 
> -I/usr/include/libpng16 -I/usr/include/virgl -I/usr/include/libusb-1.0 
> -I/usr/include/slirp -I/usr/include/gtk-3.0 -I/usr/include/pango-1.0 
> -I/usr/include/harfbuzz -I/usr/include/freetype2 -I/usr/include/fribidi 
> -I/usr/include/libxml2 -I/usr/include/cairo -I/usr/include/gdk-pixbuf-2.0 
> -I/usr/include/gio-unix-2.0 -I/usr/include/atk-1.0 
> -I/usr/include/at-spi2-atk/2.0 -I/usr/include/dbus-1.0 
> -I/usr/lib64/dbus-1.0/include -I/usr/include/at-spi-2.0 
> -I/usr/include/vte-2.91 -I/usr/include/capstone -fdiagnostics-color=auto 
> -pipe -Wall -Winvalid-pch -Werror -std=gnu11 -O2 -g -isystem 
> /scm/qemu/linux-headers -isystem linux-headers -iquote . -iquote /scm/qemu 
> -iquote /scm/qemu/include -iquote /scm/qemu/disas/libvixl -iquote 
> /scm/qemu/tcg/i386 -pthread -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -m64 -mcx16 
> -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -Wstrict-prototypes 
> -Wredundant-decls -Wundef -Wwrite-strings -Wmissing-prototypes 
> -fno-strict-aliasing -fno-common -fwrapv -Wold-style-declaration 
> -Wold-style-definition -Wtype-limits -Wformat-security -Wformat-y2k 
> -Winit-self -Wignored-qualifiers -Wempty-body -Wnested-externs -Wendif-labels 
> -Wexpansion-to-defined -Wimplicit-fallthrough=2 -Wno-missing-include-dirs 
> -Wno-shift-negative-value -Wno-psabi -fstack-protector-strong -fPIC 
> -D_DEFAULT_SOURCE -D_XOPEN_SOURCE=600 -DNCURSES_WIDECHAR 
> -DSTRUCT_IOVEC_DEFINED -D_REENTRANT -Wno-undef -MD -MQ 
> libcommon.fa.p/hw_net_vhost_net-stub.c.o -MF 
> libcommon.fa.p/hw_net_vhost_net-stub.c.o.d -o 
> libcommon.fa.p/hw_net_vhost_net-stub.c.o -c ../hw/net/vhost_net-stub.c
> ../hw/net/vhost_net-stub.c:34:5: error: conflicting types for 
> ‘vhost_net_start’
>    34 | int vhost_net_start(VirtIODevice *dev,
>       |     ^~~~~~~~~~~~~~~
> In file included from ../hw/net/vhost_net-stub.c:19:
> /scm/qemu/include/net/vhost_net.h:24:5: note: previous declaration of 
> ‘vhost_net_start’ was here
>    24 | int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
>       |     ^~~~~~~~~~~~~~~
> ../hw/net/vhost_net-stub.c:40:6: error: conflicting types for ‘vhost_net_stop’
>    40 | void vhost_net_stop(VirtIODevice *dev,
>       |      ^~~~~~~~~~~~~~
> In file included from ../hw/net/vhost_net-stub.c:19:
> /scm/qemu/include/net/vhost_net.h:26:6: note: previous declaration of 
> ‘vhost_net_stop’ was here
>    26 | void vhost_net_stop(VirtIODevice *dev, NetClientState *ncs,
>       |      ^~~~~~~~~~~~~~
> ninja: build stopped: subcommand failed.
> make[1]: *** [Makefile:156: run-ninja] Error 1

Will fix this.

Thanks

>
>
>
> > ---
> >  hw/net/vhost_net.c      | 43 ++++++++++++++++++++++++++++++-----------
> >  hw/net/virtio-net.c     |  4 ++--
> >  include/net/vhost_net.h |  6 ++++--
> >  3 files changed, 38 insertions(+), 15 deletions(-)
> >
> > diff --git a/hw/net/vhost_net.c b/hw/net/vhost_net.c
> > index 386ec2eaa2..7e0b60b4d9 100644
> > --- a/hw/net/vhost_net.c
> > +++ b/hw/net/vhost_net.c
> > @@ -315,11 +315,14 @@ static void vhost_net_stop_one(struct vhost_net *net,
> >  }
> >
> >  int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
> > -                    int total_queues)
> > +                    int data_qps, int cvq)
> >  {
> >      BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(dev)));
> >      VirtioBusState *vbus = VIRTIO_BUS(qbus);
> >      VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
> > +    int total_notifiers = data_qps * 2 + cvq;
> > +    VirtIONet *n = VIRTIO_NET(dev);
> > +    int nvhosts = data_qps + cvq;
> >      struct vhost_net *net;
> >      int r, e, i;
> >      NetClientState *peer;
> > @@ -329,9 +332,14 @@ int vhost_net_start(VirtIODevice *dev, NetClientState 
> > *ncs,
> >          return -ENOSYS;
> >      }
> >
> > -    for (i = 0; i < total_queues; i++) {
> > +    for (i = 0; i < nvhosts; i++) {
> > +
> > +        if (i < data_qps) {
> > +            peer = qemu_get_peer(ncs, i);
> > +        } else { /* Control Virtqueue */
> > +            peer = qemu_get_peer(ncs, n->max_queues);
> > +        }
> >
> > -        peer = qemu_get_peer(ncs, i);
> >          net = get_vhost_net(peer);
> >          vhost_net_set_vq_index(net, i * 2);
> >
> > @@ -344,14 +352,18 @@ int vhost_net_start(VirtIODevice *dev, NetClientState 
> > *ncs,
> >          }
> >       }
> >
> > -    r = k->set_guest_notifiers(qbus->parent, total_queues * 2, true);
> > +    r = k->set_guest_notifiers(qbus->parent, total_notifiers, true);
> >      if (r < 0) {
> >          error_report("Error binding guest notifier: %d", -r);
> >          goto err;
> >      }
> >
> > -    for (i = 0; i < total_queues; i++) {
> > -        peer = qemu_get_peer(ncs, i);
> > +    for (i = 0; i < nvhosts; i++) {
> > +        if (i < data_qps) {
> > +            peer = qemu_get_peer(ncs, i);
> > +        } else {
> > +            peer = qemu_get_peer(ncs, n->max_queues);
> > +        }
> >          r = vhost_net_start_one(get_vhost_net(peer), dev);
> >
> >          if (r < 0) {
> > @@ -375,7 +387,7 @@ err_start:
> >          peer = qemu_get_peer(ncs , i);
> >          vhost_net_stop_one(get_vhost_net(peer), dev);
> >      }
> > -    e = k->set_guest_notifiers(qbus->parent, total_queues * 2, false);
> > +    e = k->set_guest_notifiers(qbus->parent, total_notifiers, false);
> >      if (e < 0) {
> >          fprintf(stderr, "vhost guest notifier cleanup failed: %d\n", e);
> >          fflush(stderr);
> > @@ -385,18 +397,27 @@ err:
> >  }
> >
> >  void vhost_net_stop(VirtIODevice *dev, NetClientState *ncs,
> > -                    int total_queues)
> > +                    int data_qps, int cvq)
> >  {
> >      BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(dev)));
> >      VirtioBusState *vbus = VIRTIO_BUS(qbus);
> >      VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
> > +    VirtIONet *n = VIRTIO_NET(dev);
> > +    NetClientState *peer;
> > +    int total_notifiers = data_qps * 2 + cvq;
> > +    int nvhosts = data_qps + cvq;
> >      int i, r;
> >
> > -    for (i = 0; i < total_queues; i++) {
> > -        vhost_net_stop_one(get_vhost_net(ncs[i].peer), dev);
> > +    for (i = 0; i < nvhosts; i++) {
> > +        if (i < data_qps) {
> > +            peer = qemu_get_peer(ncs, i);
> > +        } else {
> > +            peer = qemu_get_peer(ncs, n->max_queues);
> > +        }
> > +        vhost_net_stop_one(get_vhost_net(peer), dev);
> >      }
> >
> > -    r = k->set_guest_notifiers(qbus->parent, total_queues * 2, false);
> > +    r = k->set_guest_notifiers(qbus->parent, total_notifiers, false);
> >      if (r < 0) {
> >          fprintf(stderr, "vhost guest notifier cleanup failed: %d\n", r);
> >          fflush(stderr);
> > diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c
> > index 16d20cdee5..8fccbaa44c 100644
> > --- a/hw/net/virtio-net.c
> > +++ b/hw/net/virtio-net.c
> > @@ -285,14 +285,14 @@ static void virtio_net_vhost_status(VirtIONet *n, 
> > uint8_t status)
> >          }
> >
> >          n->vhost_started = 1;
> > -        r = vhost_net_start(vdev, n->nic->ncs, queues);
> > +        r = vhost_net_start(vdev, n->nic->ncs, queues, 0);
> >          if (r < 0) {
> >              error_report("unable to start vhost net: %d: "
> >                           "falling back on userspace virtio", -r);
> >              n->vhost_started = 0;
> >          }
> >      } else {
> > -        vhost_net_stop(vdev, n->nic->ncs, queues);
> > +        vhost_net_stop(vdev, n->nic->ncs, queues, 0);
> >          n->vhost_started = 0;
> >      }
> >  }
> > diff --git a/include/net/vhost_net.h b/include/net/vhost_net.h
> > index fba40cf695..e656e38af9 100644
> > --- a/include/net/vhost_net.h
> > +++ b/include/net/vhost_net.h
> > @@ -21,8 +21,10 @@ typedef struct VhostNetOptions {
> >  uint64_t vhost_net_get_max_queues(VHostNetState *net);
> >  struct vhost_net *vhost_net_init(VhostNetOptions *options);
> >
> > -int vhost_net_start(VirtIODevice *dev, NetClientState *ncs, int 
> > total_queues);
> > -void vhost_net_stop(VirtIODevice *dev, NetClientState *ncs, int 
> > total_queues);
> > +int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
> > +                    int data_qps, int cvq);
> > +void vhost_net_stop(VirtIODevice *dev, NetClientState *ncs,
> > +                    int data_qps, int cvq);
> >
> >  void vhost_net_cleanup(VHostNetState *net);
> >
> > --
> > 2.25.1
>