Re: [PATCH v13 10/19] multi-process: Associate fd of a PCIDevice with its object

[Jag Raman]

> On Dec 16, 2020, at 12:34 PM, Marc-André Lureau <marcandre.lureau@gmail.com> 
> wrote:
> 
> Hi
> 
> On Wed, Dec 16, 2020 at 9:06 PM Jag Raman <jag.raman@oracle.com> wrote:
> 
> 
> > On Dec 16, 2020, at 8:43 AM, Marc-André Lureau <marcandre.lureau@gmail.com> 
> > wrote:
> > 
> > 
> > 
> > On Mon, Dec 14, 2020 at 9:15 AM Jagannathan Raman <jag.raman@oracle.com> 
> > wrote:
> > Associate the file descriptor for a PCIDevice in remote process with
> > DeviceState object.
> > 
> > Signed-off-by: Elena Ufimtseva <elena.ufimtseva@oracle.com>
> > Signed-off-by: John G Johnson <john.g.johnson@oracle.com>
> > Signed-off-by: Jagannathan Raman <jag.raman@oracle.com>
> > Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
> > ---
> >  hw/remote/remote-obj.c | 206 
> > +++++++++++++++++++++++++++++++++++++++++++++++++
> >  MAINTAINERS            |   1 +
> >  hw/remote/meson.build  |   1 +
> >  3 files changed, 208 insertions(+)
> >  create mode 100644 hw/remote/remote-obj.c
> > 
> > diff --git a/hw/remote/remote-obj.c b/hw/remote/remote-obj.c
> > new file mode 100644
> > index 0000000..b5573b7
> > --- /dev/null
> > +++ b/hw/remote/remote-obj.c
> > @@ -0,0 +1,206 @@
> > +/*
> > + * Copyright © 2020 Oracle and/or its affiliates.
> > + *
> > + * This work is licensed under the terms of the GNU GPL-v2, version 2 or 
> > later.
> > + *
> > + * See the COPYING file in the top-level directory.
> > + *
> > + */
> > +
> > +#include "qemu/osdep.h"
> > +#include "qemu-common.h"
> > +
> > +#include "qemu/error-report.h"
> > +#include "qemu/notify.h"
> > +#include "qom/object_interfaces.h"
> > +#include "hw/qdev-core.h"
> > +#include "io/channel.h"
> > +#include "hw/qdev-core.h"
> > +#include "hw/remote/machine.h"
> > +#include "io/channel-util.h"
> > +#include "qapi/error.h"
> > +#include "sysemu/sysemu.h"
> > +#include "hw/pci/pci.h"
> > +#include "qemu/sockets.h"
> > +
> > +#define TYPE_REMOTE_OBJECT "x-remote-object"
> > +#define REMOTE_OBJECT(obj) \
> > +    OBJECT_CHECK(RemoteObject, (obj), TYPE_REMOTE_OBJECT)
> > +#define REMOTE_OBJECT_GET_CLASS(obj) \
> > +    OBJECT_GET_CLASS(RemoteObjectClass, (obj), TYPE_REMOTE_OBJECT)
> > +#define REMOTE_OBJECT_CLASS(klass) \
> > +    OBJECT_CLASS_CHECK(RemoteObjectClass, (klass), TYPE_REMOTE_OBJECT)
> > +
> > +typedef struct {
> > +    ObjectClass parent_class;
> > +
> > +    unsigned int nr_devs;
> > +    unsigned int max_devs;
> > +} RemoteObjectClass;
> > +
> > +typedef struct {
> > +    /* private */
> > +    Object parent;
> > +
> > +    Notifier machine_done;
> > +
> > +    int32_t fd;
> > +    char *devid;
> > +
> > +    QIOChannel *ioc;
> > +
> > +    DeviceState *dev;
> > +    DeviceListener listener;
> > +} RemoteObject;
> > +
> > +static void remote_object_set_fd(Object *obj, Visitor *v, const char *name,
> > +                                 void *opaque, Error **errp)
> > +{
> > +    RemoteObject *o = REMOTE_OBJECT(obj);
> > +    int32_t value;
> > +
> > +    if (!visit_type_int32(v, name, &value, errp)) {
> > +        return;
> > +    }
> > +
> > +    if (!fd_is_socket(value)) {
> > +        o->fd = -1;
> > +        error_setg(errp, "File descriptor %d is not a socket", value);
> > +        return;
> > +    }
> > +
> > +    o->fd = value;
> > +}
> > +
> > +static void remote_object_set_devid(Object *obj, const char *str, Error 
> > **errp)
> > +{
> > +    RemoteObject *o = REMOTE_OBJECT(obj);
> > +
> > +    g_free(o->devid);
> > +
> > +    o->devid = g_strdup(str);
> > +}
> > +
> > +static void remote_object_release(DeviceListener *listener, DeviceState 
> > *dev)
> > +{
> > +    RemoteObject *o = container_of(listener, RemoteObject, listener);
> > +
> > +    if (o->dev == dev) {
> > +        object_unref(OBJECT(o));
> > +    }
> > +}
> > +
> > +static void remote_object_machine_done(Notifier *notifier, void *data)
> > +{
> > +    RemoteObject *o = container_of(notifier, RemoteObject, machine_done);
> > +    DeviceState *dev = NULL;
> > +    QIOChannel *ioc = NULL;
> > +    Coroutine *co = NULL;
> > +    RemoteCommDev *comdev = NULL;
> > +    Error *err = NULL;
> > +
> > +    dev = qdev_find_recursive(sysbus_get_default(), o->devid);
> > +    if (!dev || !object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE)) {
> > +        error_report("%s is not a PCI device", o->devid);
> > +        return;
> > +    }
> > +
> > +    ioc = qio_channel_new_fd(o->fd, &err);
> > +    if (!ioc) {
> > +        error_report_err(err);
> > +        return;
> > +    }
> > +    qio_channel_set_blocking(ioc, false, NULL);
> > +
> > +    o->dev = dev;
> > +
> > +    o->listener.unrealize = remote_object_release;
> > +    device_listener_register(&o->listener);
> > +
> > +    /* co-routine should free this. */
> > +    comdev = g_new0(RemoteCommDev, 1);
> > +    *comdev = (RemoteCommDev) {
> > +        .ioc = ioc,
> > +        .dev = PCI_DEVICE(dev),
> > +    };
> > +
> > +    co = qemu_coroutine_create(mpqemu_remote_msg_loop_co, comdev);
> > +    qemu_coroutine_enter(co);
> > +}
> > +
> > +static void remote_object_init(Object *obj)
> > +{
> > +    RemoteObjectClass *k = REMOTE_OBJECT_GET_CLASS(obj);
> > +    RemoteObject *o = REMOTE_OBJECT(obj);
> > +
> > +    if (k->nr_devs >= k->max_devs) {
> > +        error_report("Reached maximum number of devices: %u", k->max_devs);
> > +        return;
> > +    }
> > +
> > +    o->ioc = NULL;
> > +    o->fd = -1;
> > +    o->devid = NULL;
> > +
> > +    k->nr_devs++;
> > +
> > +    o->machine_done.notify = remote_object_machine_done;
> > +    qemu_add_machine_init_done_notifier(&o->machine_done);
> > +}
> > +
> > +static void remote_object_finalize(Object *obj)
> > +{
> > +    RemoteObjectClass *k = REMOTE_OBJECT_GET_CLASS(obj);
> > +    RemoteObject *o = REMOTE_OBJECT(obj);
> > +
> > +    device_listener_unregister(&o->listener);
> > +
> > +    if (o->ioc) {
> > +        qio_channel_shutdown(o->ioc, QIO_CHANNEL_SHUTDOWN_BOTH, NULL);
> > +        qio_channel_close(o->ioc, NULL);
> > +    }
> > +
> > +    object_unref(OBJECT(o->ioc));
> > +
> > +    k->nr_devs--;
> > +    g_free(o->devid);
> > +}
> > +
> > +static void remote_object_class_init(ObjectClass *klass, void *data)
> > +{
> > +    RemoteObjectClass *k = REMOTE_OBJECT_CLASS(klass);
> > +
> > +    /*
> > +     * Limit number of supported devices to 1. This is done to avoid 
> > devices
> > +     * from one VM accessing the RAM of another VM. This is done until we
> > +     * start using separate address spaces for individual devices.
> > +     */
> > +    k->max_devs = 1;
> > +    k->nr_devs = 0;
> > +
> > +    object_class_property_add(klass, "fd", "int32_t", NULL,
> > +                              remote_object_set_fd, NULL, NULL);
> > 
> > Let's be consistent with pci-proxy-dev, and make it a string property that 
> > is parsed with monitor_fd_param()
> 
> Thank you for the feedback, Marc-Andre!
> 
> Would it be alright if we added this to util/qemu-sockets.c since this logic 
> is used more than once, or is it
> an overkill? I saw that socket_get_fd() in qemu-sockets.c is doing something 
> similar, but using
> monitor_get_fd() instead.
> 
> 
> That sounds like a good idea. socket_get_fd() could become an exported 
> function.
> 
> You should move the num != 1 check to socket_listen(), that should be fine.

Thank you for confirming!

—
Jag

> 
> thanks
>  
> --
> Jag
> 
> > 
> > +    object_class_property_add_str(klass, "devid", NULL,
> > +                                  remote_object_set_devid);
> > +}
> > +
> > +static const TypeInfo remote_object_info = {
> > +    .name = TYPE_REMOTE_OBJECT,
> > +    .parent = TYPE_OBJECT,
> > +    .instance_size = sizeof(RemoteObject),
> > +    .instance_init = remote_object_init,
> > +    .instance_finalize = remote_object_finalize,
> > +    .class_size = sizeof(RemoteObjectClass),
> > +    .class_init = remote_object_class_init,
> > +    .interfaces = (InterfaceInfo[]) {
> > +        { TYPE_USER_CREATABLE },
> > +        { }
> > +    }
> > +};
> > +
> > +static void register_types(void)
> > +{
> > +    type_register_static(&remote_object_info);
> > +}
> > +
> > +type_init(register_types);
> > diff --git a/MAINTAINERS b/MAINTAINERS
> > index 4f2cd91..f049059 100644
> > --- a/MAINTAINERS
> > +++ b/MAINTAINERS
> > @@ -3160,6 +3160,7 @@ F: include/hw/remote/machine.h
> >  F: hw/remote/mpqemu-link.c
> >  F: include/hw/remote/mpqemu-link.h
> >  F: hw/remote/message.c
> > +F: hw/remote/remote-obj.c
> > 
> >  Build and test automation
> >  -------------------------
> > diff --git a/hw/remote/meson.build b/hw/remote/meson.build
> > index 9f5c57f..71d0a56 100644
> > --- a/hw/remote/meson.build
> > +++ b/hw/remote/meson.build
> > @@ -3,5 +3,6 @@ remote_ss = ss.source_set()
> >  remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c'))
> >  remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c'))
> >  remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c'))
> > +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c'))
> > 
> >  softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
> > -- 
> > 1.8.3.1
> > 
> > 
> > 
> > -- 
> > Marc-André Lureau
> 
> 
> 
> -- 
> Marc-André Lureau