[Qemu-devel] [PATCH 04/13] RFC: qapi: learn to split the schema by 'top-unit'

[Marc-André Lureau]

A module can declare belonging to a 'top-unit', with the corresponding
pragma value (the default 'top-unit' is None).

The generators have a chance to break the generated output by
units (top-units are visited first).  Generated types, visitors,
events and commands are split by 'top-unit'. Generated introspection
and documentation remain monolithic.

In the following commits, a 'target' unit is introduced. The generated
'target' files will be allowed to used poisoned defines for
conditional compilation, since they will be built with the target
objects.

TODO: if this approach is acceptable, split the patch, write tests & doc.
Note: I don't really have better ideas, but open for suggestions...

Signed-off-by: Marc-André Lureau <address@hidden>
---
 scripts/qapi/commands.py |  22 +++++---
 scripts/qapi/common.py   | 105 +++++++++++++++++++++++++++++++--------
 scripts/qapi/events.py   |  37 ++++++++------
 scripts/qapi/types.py    |   8 +--
 scripts/qapi/visit.py    |   8 +--
 5 files changed, 130 insertions(+), 50 deletions(-)

diff --git a/scripts/qapi/commands.py b/scripts/qapi/commands.py
index 94313b8aef..e4dc92e70b 100644
--- a/scripts/qapi/commands.py
+++ b/scripts/qapi/commands.py
@@ -237,14 +237,20 @@ class 
QAPISchemaGenCommandVisitor(QAPISchemaModularCVisitor):
         QAPISchemaModularCVisitor.__init__(
             self, prefix, 'qapi-commands',
             ' * Schema-defined QAPI/QMP commands', __doc__)
+
+    def visit_unit_begin(self, unit):
+        super(self.__class__, self).visit_unit_begin(unit)
         self._regy = QAPIGenCCode()
         self._visited_ret_types = {}
 
-    def _begin_module(self, name):
+    def _begin_module(self, name, main_module):
         self._visited_ret_types[self._genc] = set()
-        commands = self._module_basename('qapi-commands', name)
-        types = self._module_basename('qapi-types', name)
-        visit = self._module_basename('qapi-visit', name)
+        commands = self._module_basename('qapi-commands', name,
+                                         self._unit, main_module)
+        types = self._module_basename('qapi-types', name,
+                                      self._unit, main_module)
+        visit = self._module_basename('qapi-visit', name,
+                                      self._unit, main_module)
         self._genc.add(mcgen('''
 #include "qemu/osdep.h"
 #include "qemu-common.h"
@@ -267,13 +273,13 @@ class 
QAPISchemaGenCommandVisitor(QAPISchemaModularCVisitor):
 ''',
                              types=types))
 
-    def visit_end(self):
-        (genc, genh) = self._module[self._main_module]
+    def visit_unit_end(self):
+        (genc, genh) = self.get_module_gen(self._main_module)
         genh.add(mcgen('''
 void %(c_prefix)sqmp_register_commands(QmpCommandList *cmds);
 ''',
-                       c_prefix=c_name(self._prefix, protect=False)))
-        genc.add(gen_registry(self._regy.get_content(), self._prefix))
+                       c_prefix=c_name(self._prefix_unit(), protect=False)))
+        genc.add(gen_registry(self._regy.get_content(), self._prefix_unit()))
 
     def visit_command(self, name, info, ifcond, arg_type, ret_type, gen,
                       success_response, boxed, allow_oob, allow_preconfig):
diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py
index 8c2d97369e..1ef3a4f70b 100644
--- a/scripts/qapi/common.py
+++ b/scripts/qapi/common.py
@@ -269,11 +269,12 @@ class QAPISchemaParser(object):
         self.exprs = []
         self.docs = []
         self.accept()
+        self._top_unit = None
         cur_doc = None
 
         while self.tok is not None:
             info = {'file': self.fname, 'line': self.line,
-                    'parent': self.incl_info}
+                    'parent': self.incl_info, 'top-unit': self._top_unit}
             if self.tok == '#':
                 self.reject_expr_doc(cur_doc)
                 cur_doc = self.get_doc(info)
@@ -296,6 +297,9 @@ class QAPISchemaParser(object):
                 exprs_include = self._include(include, info, incl_fname,
                                               previously_included)
                 if exprs_include:
+                    incl_info = self.exprs[-1]['info']
+                    if exprs_include._top_unit:
+                        incl_info['has-pragma-top-unit'] = 
exprs_include._top_unit
                     self.exprs.extend(exprs_include.exprs)
                     self.docs.extend(exprs_include.docs)
             elif "pragma" in expr:
@@ -357,6 +361,11 @@ class QAPISchemaParser(object):
                 raise QAPISemError(info,
                                    "Pragma 'doc-required' must be boolean")
             doc_required = value
+        elif name == 'top-unit':
+            if not isinstance(value, str):
+                raise QAPISemError(info,
+                                   "Pragma 'top-unit' must be a string")
+            self._top_unit = value
         elif name == 'returns-whitelist':
             if (not isinstance(value, list)
                     or any([not isinstance(elt, str) for elt in value])):
@@ -1081,6 +1090,11 @@ class QAPISchemaEntity(object):
     def c_name(self):
         return c_name(self.name)
 
+    def get_top_unit(self):
+        if self.info:
+            return self.info['top-unit']
+        return None
+
     def check(self, schema):
         if isinstance(self._ifcond, QAPISchemaType):
             # inherit the condition from a type
@@ -1101,6 +1115,12 @@ class QAPISchemaVisitor(object):
     def visit_begin(self, schema):
         pass
 
+    def visit_unit_begin(self, unit):
+        pass
+
+    def visit_unit_end(self):
+        pass
+
     def visit_end(self):
         pass
 
@@ -1610,7 +1630,7 @@ class QAPISchema(object):
         parser = QAPISchemaParser(f)
         exprs = check_exprs(parser.exprs)
         self.docs = parser.docs
-        self._entity_list = []
+        self._entity_list = {} # dict of unit name -> list of entity
         self._entity_dict = {}
         self._predefining = True
         self._def_predefineds()
@@ -1622,7 +1642,8 @@ class QAPISchema(object):
         # Only the predefined types are allowed to not have info
         assert ent.info or self._predefining
         assert ent.name is None or ent.name not in self._entity_dict
-        self._entity_list.append(ent)
+        entity_list = self._entity_list.setdefault(ent.get_top_unit(), [])
+        entity_list.append(ent)
         if ent.name is not None:
             self._entity_dict[ent.name] = ent
         if ent.info:
@@ -1861,18 +1882,23 @@ class QAPISchema(object):
                 assert False
 
     def check(self):
-        for ent in self._entity_list:
-            ent.check(self)
+        for unit in self._entity_list:
+            for ent in self._entity_list[unit]:
+                ent.check(self)
 
     def visit(self, visitor):
         visitor.visit_begin(self)
-        module = None
-        for entity in self._entity_list:
-            if visitor.visit_needed(entity):
+        for unit in self._entity_list:
+            module = None
+            visitor.visit_unit_begin(unit)
+            for entity in self._entity_list[unit]:
+                if not visitor.visit_needed(entity):
+                    continue
                 if entity.module != module:
                     module = entity.module
                     visitor.visit_module(module)
                 entity.visit(visitor)
+            visitor.visit_unit_end()
         visitor.visit_end()
 
 
@@ -2313,6 +2339,19 @@ class QAPISchemaMonolithicCVisitor(QAPISchemaVisitor):
         self._genh.write(output_dir, self._prefix + self._what + '.h')
 
 
+class QAPIGenCModule(object):
+
+    def __init__(self, blurb, pydoc, unit, main_module=False):
+        self.genc = QAPIGenC(blurb, pydoc)
+        self.genh = QAPIGenH(blurb, pydoc)
+        self.unit = unit
+        self.main_module = main_module
+
+    def write(self, output_dir, basename):
+        self.genc.write(output_dir, basename + '.c')
+        self.genh.write(output_dir, basename + '.h')
+
+
 class QAPISchemaModularCVisitor(QAPISchemaVisitor):
 
     def __init__(self, prefix, what, blurb, pydoc):
@@ -2321,48 +2360,70 @@ class QAPISchemaModularCVisitor(QAPISchemaVisitor):
         self._blurb = blurb
         self._pydoc = pydoc
         self._module = {}
+        self._unit = None
         self._main_module = None
 
-    def _module_basename(self, what, name):
+    def _module_basename(self, what, name, unit=None, main_module=False):
         if name is None:
             return re.sub(r'-', '-builtin-', what)
         basename = os.path.join(os.path.dirname(name),
                                 self._prefix + what)
-        if name == self._main_module:
+        if unit:
+            basename = unit + '-' + basename
+        if main_module:
             return basename
         return basename + '-' + os.path.splitext(os.path.basename(name))[0]
 
+    def _prefix_unit(self):
+        if self._unit:
+            return self._prefix + self._unit + '-'
+        return self._prefix
+
+    def visit_unit_begin(self, unit):
+        self._unit = unit
+        self._main_module = None
+
     def _add_module(self, name, blurb):
-        if self._main_module is None and name is not None:
+        main_module = False
+        if (name is not None and
+            ((self._unit is None and self._main_module is None) or
+             (self._unit == os.path.splitext(os.path.basename(name))[0]))):
             self._main_module = name
-        genc = QAPIGenC(blurb, self._pydoc)
-        genh = QAPIGenH(blurb, self._pydoc)
-        self._module[name] = (genc, genh)
+            main_module = True
+        self._module[name] = QAPIGenCModule(blurb, self._pydoc,
+                                            self._unit, main_module)
         self._set_module(name)
+        return main_module
+
+    def get_module_gen(self, name):
+        mod = self._module[name]
+        return mod.genc, mod.genh
 
     def _set_module(self, name):
-        self._genc, self._genh = self._module[name]
+        self._genc, self._genh = self.get_module_gen(name)
 
     def write(self, output_dir, opt_builtins=False):
         for name in self._module:
             if name is None and not opt_builtins:
                 continue
-            basename = self._module_basename(self._what, name)
-            (genc, genh) = self._module[name]
-            genc.write(output_dir, basename + '.c')
-            genh.write(output_dir, basename + '.h')
+            module = self._module[name]
+            basename = self._module_basename(self._what, name,
+                                             module.unit, module.main_module)
+            module.write(output_dir, basename)
 
-    def _begin_module(self, name):
+    def _begin_module(self, name, main_module):
         pass
 
     def visit_module(self, name):
         if name in self._module:
             self._set_module(name)
             return
-        self._add_module(name, self._blurb)
-        self._begin_module(name)
+        main_module = self._add_module(name, self._blurb)
+        self._begin_module(name, main_module)
 
     def visit_include(self, name, info):
+        if 'has-pragma-top-unit' in info:
+            return
         basename = self._module_basename(self._what, name)
         self._genh.preamble_add(mcgen('''
 #include "%(basename)s.h"
diff --git a/scripts/qapi/events.py b/scripts/qapi/events.py
index 37ee5de682..b73e10e16d 100644
--- a/scripts/qapi/events.py
+++ b/scripts/qapi/events.py
@@ -58,7 +58,12 @@ def gen_param_var(typ):
     return ret
 
 
-def gen_event_send(name, arg_type, boxed, event_enum_name):
+def gen_event_send(unit, name, arg_type, boxed, event_enum_name):
+    if not unit:
+        unit = ''
+    else:
+        unit += '_'
+
     # FIXME: Our declaration of local variables (and of 'errp' in the
     # parameter list) can collide with exploded members of the event's
     # data type passed in as parameters.  If this collision ever hits in
@@ -86,7 +91,7 @@ def gen_event_send(name, arg_type, boxed, event_enum_name):
 
     ret += mcgen('''
 
-    emit = qmp_event_get_func_emit();
+    emit = %(unit)sqmp_event_get_func_emit();
     if (!emit) {
         return;
     }
@@ -94,7 +99,7 @@ def gen_event_send(name, arg_type, boxed, event_enum_name):
     qmp = qmp_event_build_dict("%(name)s");
 
 ''',
-                 name=name)
+                 name=name, unit=unit)
 
     if arg_type and not arg_type.is_empty():
         ret += mcgen('''
@@ -143,12 +148,17 @@ class 
QAPISchemaGenEventVisitor(QAPISchemaModularCVisitor):
         QAPISchemaModularCVisitor.__init__(
             self, prefix, 'qapi-events',
             ' * Schema-defined QAPI/QMP events', __doc__)
-        self._event_enum_name = c_name(prefix + 'QAPIEvent', protect=False)
+
+    def visit_unit_begin(self, unit):
+        super(self.__class__, self).visit_unit_begin(unit)
+        self._event_enum_name = c_name(self._prefix_unit() + 'QAPIEvent', 
protect=False)
         self._event_enum_members = []
 
-    def _begin_module(self, name):
-        types = self._module_basename('qapi-types', name)
-        visit = self._module_basename('qapi-visit', name)
+    def _begin_module(self, name, main_module):
+        types = self._module_basename('qapi-types', name,
+                                      self._unit, main_module)
+        visit = self._module_basename('qapi-visit', name,
+                                      self._unit, main_module)
         self._genc.add(mcgen('''
 #include "qemu/osdep.h"
 #include "qemu-common.h"
@@ -160,7 +170,7 @@ class QAPISchemaGenEventVisitor(QAPISchemaModularCVisitor):
 #include "qapi/qmp-event.h"
 
 ''',
-                             visit=visit, prefix=self._prefix))
+                             visit=visit, prefix=self._prefix_unit()))
         self._genh.add(mcgen('''
 #include "qapi/util.h"
 #include "%(types)s.h"
@@ -168,17 +178,16 @@ class 
QAPISchemaGenEventVisitor(QAPISchemaModularCVisitor):
 ''',
                              types=types))
 
-    def visit_end(self):
-        (genc, genh) = self._module[self._main_module]
+    def visit_unit_end(self):
+        (genc, genh) = self.get_module_gen(self._main_module)
         genh.add(gen_enum(self._event_enum_name, self._event_enum_members))
-        genc.add(gen_enum_lookup(self._event_enum_name,
-                                 self._event_enum_members))
+        genc.add(gen_enum_lookup(self._event_enum_name, 
self._event_enum_members))
 
     def visit_event(self, name, info, ifcond, arg_type, boxed):
         with ifcontext(ifcond, self._genh, self._genc):
             self._genh.add(gen_event_send_decl(name, arg_type, boxed))
-            self._genc.add(gen_event_send(name, arg_type, boxed,
-                                          self._event_enum_name))
+            self._genc.add(gen_event_send(self._unit, name,
+                                          arg_type, boxed, 
self._event_enum_name))
         self._event_enum_members.append(QAPISchemaMember(name, ifcond))
 
 
diff --git a/scripts/qapi/types.py b/scripts/qapi/types.py
index 62d4cf9f95..db85cf9b9b 100644
--- a/scripts/qapi/types.py
+++ b/scripts/qapi/types.py
@@ -194,9 +194,11 @@ class QAPISchemaGenTypeVisitor(QAPISchemaModularCVisitor):
 #include "qapi/util.h"
 '''))
 
-    def _begin_module(self, name):
-        types = self._module_basename('qapi-types', name)
-        visit = self._module_basename('qapi-visit', name)
+    def _begin_module(self, name, main_module):
+        types = self._module_basename('qapi-types', name,
+                                      self._unit, main_module)
+        visit = self._module_basename('qapi-visit', name,
+                                      self._unit, main_module)
         self._genc.preamble_add(mcgen('''
 #include "qemu/osdep.h"
 #include "qapi/dealloc-visitor.h"
diff --git a/scripts/qapi/visit.py b/scripts/qapi/visit.py
index 82eab72b21..258ebdb690 100644
--- a/scripts/qapi/visit.py
+++ b/scripts/qapi/visit.py
@@ -298,9 +298,11 @@ class QAPISchemaGenVisitVisitor(QAPISchemaModularCVisitor):
 ''',
                                       prefix=prefix))
 
-    def _begin_module(self, name):
-        types = self._module_basename('qapi-types', name)
-        visit = self._module_basename('qapi-visit', name)
+    def _begin_module(self, name, main_module):
+        types = self._module_basename('qapi-types', name,
+                                      self._unit, main_module)
+        visit = self._module_basename('qapi-visit', name,
+                                      self._unit, main_module)
         self._genc.preamble_add(mcgen('''
 #include "qemu/osdep.h"
 #include "qemu-common.h"
-- 
2.20.0