[Qemu-devel] [PATCH v3 11/14] qapi: add qapi2texi script

[Marc-André Lureau]

As the name suggests, the qapi2texi script converts JSON QAPI
description into a texi file suitable for different target
formats (info/man/txt/pdf/html...).

It parses the following kind of blocks:

Free-form:

  ##
  # = Section
  # == Subsection
  #
  # Some text foo with *emphasis*
  # 1. with a list
  # 2. like that
  #
  # And some code:
  # | $ echo foo
  # | -> do this
  # | <- get that
  #
  ##

Symbol:

  ##
  # @symbol:
  #
  # Symbol body ditto ergo sum. Foo bar
  # baz ding.
  #
  # @arg: foo
  # @arg: #optional foo
  #
  # Returns: returns bla bla
  #          Or bla blah
  #
  # Since: version
  # Notes: notes, comments can have
  #        - itemized list
  #        - like this
  #
  # Example:
  #
  # -> { "execute": "quit" }
  # <- { "return": {} }
  #
  ##

That's roughly following the following BNF grammar:

api_comment = "##\n" comment "##\n"
comment = freeform_comment | symbol_comment
freeform_comment = { "#" text "\n" }
symbol_comment = "#" "@" name ":\n" { freeform | member | meta }
member = "#" '@' name ':' [ text ] freeform_comment
meta = "#" ( "Returns:", "Since:", "Note:", "Notes:", "Example:", "Examples:" ) 
[ text ] freeform_comment
text = free-text markdown-like, "#optional" for members

Thanks to the following json expressions, the documentation is enhanced
with extra information about the type of arguments and return value
expected.

Signed-off-by: Marc-André Lureau <address@hidden>
---
 scripts/qapi.py        | 175 ++++++++++++++++++++++++++-
 scripts/qapi2texi.py   | 316 +++++++++++++++++++++++++++++++++++++++++++++++++
 docs/qapi-code-gen.txt |  44 +++++--
 3 files changed, 524 insertions(+), 11 deletions(-)
 create mode 100755 scripts/qapi2texi.py

diff --git a/scripts/qapi.py b/scripts/qapi.py
index 21bc32f..ed52ee4 100644
--- a/scripts/qapi.py
+++ b/scripts/qapi.py
@@ -122,6 +122,103 @@ class QAPIExprError(Exception):
             "%s:%d: %s" % (self.info['file'], self.info['line'], self.msg)
 
 
+class QAPIDoc(object):
+    def __init__(self, parser):
+        self.parser = parser
+        self.symbol = None
+        self.body = []
+        # args is {'arg': 'doc', ...}
+        self.args = OrderedDict()
+        # meta is [(Since/Notes/Examples/Returns:, 'doc'), ...]
+        self.meta = []
+        # the current section to populate, array of [dict, key, comment...]
+        self.section = None
+        self.expr_elem = None
+
+    def get_body(self):
+        return "\n".join(self.body)
+
+    def has_meta(self, name):
+        """Returns True if the doc has a meta section 'name'"""
+        return next((True for i in self.meta if i[0] == name), False)
+
+    def append(self, line):
+        """Adds a # comment line, to be parsed and added in a section"""
+        line = line[1:]
+        if len(line) == 0:
+            self._append_section(line)
+            return
+
+        if line[0] != ' ':
+            raise QAPISchemaError(self.parser, "missing space after #")
+
+        line = line[1:]
+        # take the first word out
+        name = line.split(' ', 1)[0]
+        if name.startswith("@") and name.endswith(":"):
+            line = line[len(name):]
+            name = name[1:-1]
+            if self.symbol is None:
+                # the first is the symbol this APIDoc object documents
+                if len(self.body):
+                    raise QAPISchemaError(self.parser, "symbol must come 
first")
+                self.symbol = name
+            else:
+                # else an arg
+                self._start_args_section(name)
+        elif self.symbol and name in (
+                "Returns:", "Since:",
+                # those are often singular or plural
+                "Note:", "Notes:",
+                "Example:", "Examples:"):
+            # new "meta" section
+            line = line[len(name):]
+            self._start_meta_section(name[:-1])
+
+        self._append_section(line)
+
+    def _start_args_section(self, name):
+        self.end_section()
+        if self.args.has_key(name):
+            raise QAPISchemaError(self.parser, "'%s' arg duplicated" % name)
+        self.section = [self.args, name]
+
+    def _start_meta_section(self, name):
+        self.end_section()
+        if name in ("Returns", "Since") and self.has_meta(name):
+            raise QAPISchemaError(self.parser, "'%s' section duplicated" % 
name)
+        self.section = [self.meta, name]
+
+    def _append_section(self, line):
+        """Add a comment to the current section, or the comment body"""
+        if self.section:
+            name = self.section[1]
+            if not name.startswith("Example"):
+                # an empty line ends the section, except with Example
+                if len(self.section) > 2 and len(line) == 0:
+                    self.end_section()
+                    return
+                # Example is verbatim
+                line = line.strip()
+            if len(line) > 0:
+                self.section.append(line)
+        else:
+            self.body.append(line.strip())
+
+    def end_section(self):
+        if self.section is not None:
+            target = self.section[0]
+            name = self.section[1]
+            if len(self.section) < 3:
+                raise QAPISchemaError(self.parser, "Empty doc section")
+            doc = "\n".join(self.section[2:])
+            if isinstance(target, dict):
+                target[name] = doc
+            else:
+                target.append((name, doc))
+            self.section = None
+
+
 class QAPISchemaParser(object):
 
     def __init__(self, fp, previously_included=[], incl_info=None):
@@ -137,9 +234,15 @@ class QAPISchemaParser(object):
         self.line = 1
         self.line_pos = 0
         self.exprs = []
+        self.docs = []
         self.accept()
 
         while self.tok is not None:
+            if self.tok == '#' and self.val.startswith('##'):
+                doc = self.get_doc()
+                self.docs.append(doc)
+                continue
+
             expr_info = {'file': fname, 'line': self.line,
                          'parent': self.incl_info}
             expr = self.get_expr(False)
@@ -160,6 +263,7 @@ class QAPISchemaParser(object):
                         raise QAPIExprError(expr_info, "Inclusion loop for %s"
                                             % include)
                     inf = inf['parent']
+
                 # skip multiple include of the same file
                 if incl_abs_fname in previously_included:
                     continue
@@ -171,12 +275,40 @@ class QAPISchemaParser(object):
                 exprs_include = QAPISchemaParser(fobj, previously_included,
                                                  expr_info)
                 self.exprs.extend(exprs_include.exprs)
+                self.docs.extend(exprs_include.docs)
             else:
                 expr_elem = {'expr': expr,
                              'info': expr_info}
+                if len(self.docs) > 0:
+                    self.docs[-1].expr_elem = expr_elem
                 self.exprs.append(expr_elem)
 
-    def accept(self):
+    def get_doc(self):
+        if self.val != '##':
+            raise QAPISchemaError(self, "Doc comment not starting with '##'")
+
+        doc = QAPIDoc(self)
+        self.accept(False)
+        while self.tok == '#':
+            if self.val.startswith('##'):
+                # ## ends doc
+                if self.val != '##':
+                    raise QAPISchemaError(self, "non-empty '##' line %s"
+                                          % self.val)
+                self.accept()
+                doc.end_section()
+                return doc
+            else:
+                doc.append(self.val)
+            self.accept(False)
+
+        if self.val != '##':
+            raise QAPISchemaError(self, "Doc comment not finishing with '##'")
+
+        doc.end_section()
+        return doc
+
+    def accept(self, skip_comment=True):
         while True:
             self.tok = self.src[self.cursor]
             self.pos = self.cursor
@@ -184,7 +316,13 @@ class QAPISchemaParser(object):
             self.val = None
 
             if self.tok == '#':
+                if self.src[self.cursor] == '#':
+                    # ## starts a doc comment
+                    skip_comment = False
                 self.cursor = self.src.find('\n', self.cursor)
+                self.val = self.src[self.pos:self.cursor]
+                if not skip_comment:
+                    return
             elif self.tok in "{}:,[]":
                 return
             elif self.tok == "'":
@@ -779,6 +917,41 @@ def check_exprs(exprs):
 
     return exprs
 
+def check_docs(docs):
+    for doc in docs:
+        expr_elem = doc.expr_elem
+        if not expr_elem:
+            continue
+
+        expr = expr_elem['expr']
+        for i in ('enum', 'union', 'alternate', 'struct', 'command', 'event'):
+            if i in expr:
+                meta = i
+                break
+
+        info = expr_elem['info']
+        name = expr[meta]
+        if doc.symbol != name:
+            raise QAPIExprError(info,
+                                "Documentation symbol mismatch '%s' != '%s'"
+                                % (doc.symbol, name))
+        if not 'command' in expr and doc.has_meta('Returns'):
+            raise QAPIExprError(info, "Invalid return documentation")
+
+        doc_args = set(doc.args.keys())
+        if meta == 'union':
+            data = expr.get('base', [])
+        else:
+            data = expr.get('data', [])
+        if isinstance(data, dict):
+            data = data.keys()
+        args = set([k.strip('*') for k in data])
+        if meta == 'alternate' or \
+           (meta == 'union' and not expr.get('discriminator')):
+            args.add('type')
+        if not doc_args.issubset(args):
+            raise QAPIExprError(info, "Members documentation is not a subset 
of"
+                                " API %r > %r" % (list(doc_args), list(args)))
 
 #
 # Schema compiler frontend
diff --git a/scripts/qapi2texi.py b/scripts/qapi2texi.py
new file mode 100755
index 0000000..7e2440c
--- /dev/null
+++ b/scripts/qapi2texi.py
@@ -0,0 +1,316 @@
+#!/usr/bin/env python
+# QAPI texi generator
+#
+# This work is licensed under the terms of the GNU LGPL, version 2+.
+# See the COPYING file in the top-level directory.
+"""This script produces the documentation of a qapi schema in texinfo format"""
+import re
+import sys
+
+from qapi import *
+
+COMMAND_FMT = """
address@hidden {type} {{{ret}}} {name} @
+{{{args}}}
+
+{body}
+
address@hidden deftypefn
+
+""".format
+
+ENUM_FMT = """
address@hidden Enum {name}
+
+{body}
+
address@hidden deftp
+
+""".format
+
+STRUCT_FMT = """
address@hidden {type} {name} @
+{{{attrs}}}
+
+{body}
+
address@hidden deftp
+
+""".format
+
+EXAMPLE_FMT = """@example
+{code}
address@hidden example
+""".format
+
+
+def subst_strong(doc):
+    """Replaces *foo* by @strong{foo}"""
+    return re.sub(r'\*([^_\n]+)\*', r'@emph{\1}', doc)
+
+
+def subst_emph(doc):
+    """Replaces _foo_ by @emph{foo}"""
+    return re.sub(r'\s_([^_\n]+)_\s', r' @emph{\1} ', doc)
+
+
+def subst_vars(doc):
+    """Replaces @var by @var{var}"""
+    return re.sub(r'@([\w-]+)', r'@var{\1}', doc)
+
+
+def subst_braces(doc):
+    """Replaces {} with @{ @}"""
+    return doc.replace("{", "@{").replace("}", "@}")
+
+
+def texi_example(doc):
+    """Format @example"""
+    doc = subst_braces(doc).strip('\n')
+    return EXAMPLE_FMT(code=doc)
+
+
+def texi_comment(doc):
+    """
+    Format a comment
+
+    Lines starting with:
+    - |: generates an @example
+    - =: generates @section
+    - ==: generates @subsection
+    - 1. or 1): generates an @enumerate @item
+    - o/*/-: generates an @itemize list
+    """
+    lines = []
+    doc = subst_braces(doc)
+    doc = subst_vars(doc)
+    doc = subst_emph(doc)
+    doc = subst_strong(doc)
+    inlist = ""
+    lastempty = False
+    for line in doc.split('\n'):
+        empty = line == ""
+
+        if line.startswith("| "):
+            line = EXAMPLE_FMT(code=line[1:])
+        elif line.startswith("= "):
+            line = "@section " + line[1:]
+        elif line.startswith("== "):
+            line = "@subsection " + line[2:]
+        elif re.match("^([0-9]*[.)]) ", line):
+            if not inlist:
+                lines.append("@enumerate")
+                inlist = "enumerate"
+            line = line[line.find(" ")+1:]
+            lines.append("@item")
+        elif re.match("^[o*-] ", line):
+            if not inlist:
+                lines.append("@itemize %s" % {'o': "@bullet",
+                                              '*': "@minus",
+                                              '-': ""}[line[0]])
+                inlist = "itemize"
+            lines.append("@item")
+            line = line[2:]
+        elif lastempty and inlist:
+            lines.append("@end %s\n" % inlist)
+            inlist = ""
+
+        lastempty = empty
+        lines.append(line)
+
+    if inlist:
+        lines.append("@end %s\n" % inlist)
+    return "\n".join(lines)
+
+
+def texi_args(expr):
+    """
+    Format the functions/structure/events.. arguments/members
+    """
+    data = expr["data"] if "data" in expr else {}
+    if isinstance(data, str):
+        args = data
+    else:
+        arg_list = []
+        for name, typ in data.iteritems():
+            # optional arg
+            if name.startswith("*"):
+                name = name[1:]
+                arg_list.append("['%s': @var{%s}]" % (name, typ))
+            # regular arg
+            else:
+                arg_list.append("'%s': @var{%s}" % (name, typ))
+        args = ", ".join(arg_list)
+    return args
+
+def section_order(section):
+    return {"Returns": 0,
+            "Note": 1,
+            "Notes": 1,
+            "Since": 2,
+            "Example": 3,
+            "Examples": 3}[section]
+
+def texi_body(doc, arg="@var"):
+    """
+    Format the body of a symbol documentation:
+    - a table of arguments
+    - followed by "Returns/Notes/Since/Example" sections
+    """
+    body = "@table %s\n" % arg
+    for arg, desc in doc.args.iteritems():
+        if desc.startswith("#optional"):
+            desc = desc[10:]
+            arg += "*"
+        elif desc.endswith("#optional"):
+            desc = desc[:-10]
+            arg += "*"
+        body += "@item %s\n%s\n" % (arg, texi_comment(desc))
+    body += "@end table\n"
+    body += texi_comment(doc.get_body())
+
+    meta = sorted(doc.meta, key=lambda i: section_order(i[0]))
+    for m in meta:
+        key, doc = m
+        func = texi_comment
+        if key.startswith("Example"):
+            func = texi_example
+
+        body += "address@hidden address@hidden quotation" % \
+                (key, func(doc))
+    return body
+
+
+def texi_alternate(expr, doc):
+    """
+    Format an alternate to texi
+    """
+    args = texi_args(expr)
+    body = texi_body(doc)
+    return STRUCT_FMT(type="Alternate",
+                      name=doc.symbol,
+                      attrs="[ " + args + " ]",
+                      body=body)
+
+
+def texi_union(expr, doc):
+    """
+    Format an union to texi
+    """
+    args = texi_args(expr)
+    body = texi_body(doc)
+    return STRUCT_FMT(type="Union",
+                      name=doc.symbol,
+                      attrs="[ " + args + " ]",
+                      body=body)
+
+
+def texi_enum(_, doc):
+    """
+    Format an enum to texi
+    """
+    body = texi_body(doc, "@samp")
+    return ENUM_FMT(name=doc.symbol,
+                    body=body)
+
+
+def texi_struct(expr, doc):
+    """
+    Format a struct to texi
+    """
+    args = texi_args(expr)
+    body = texi_body(doc)
+    return STRUCT_FMT(type="Struct",
+                      name=doc.symbol,
+                      attrs="@{ " + args + " @}",
+                      body=body)
+
+
+def texi_command(expr, doc):
+    """
+    Format a command to texi
+    """
+    args = texi_args(expr)
+    ret = expr["returns"] if "returns" in expr else ""
+    body = texi_body(doc)
+    return COMMAND_FMT(type="Command",
+                       name=doc.symbol,
+                       ret=ret,
+                       args="(" + args + ")",
+                       body=body)
+
+
+def texi_event(expr, doc):
+    """
+    Format an event to texi
+    """
+    args = texi_args(expr)
+    body = texi_body(doc)
+    return COMMAND_FMT(type="Event",
+                       name=doc.symbol,
+                       ret="",
+                       args="(" + args + ")",
+                       body=body)
+
+
+def texi(docs):
+    """
+    Convert QAPI schema expressions to texi documentation
+    """
+    res = []
+    for doc in docs:
+        try:
+            expr_elem = doc.expr_elem
+            if expr_elem is None:
+                res.append(texi_body(doc))
+                continue
+
+            expr = expr_elem['expr']
+            (kind, _) = expr.items()[0]
+
+            fmt = {"command": texi_command,
+                   "struct": texi_struct,
+                   "enum": texi_enum,
+                   "union": texi_union,
+                   "alternate": texi_alternate,
+                   "event": texi_event}
+            try:
+                fmt = fmt[kind]
+            except KeyError:
+                raise ValueError("Unknown expression kind '%s'" % kind)
+            res.append(fmt(expr, doc))
+        except:
+            print >>sys.stderr, "error at @%s" % qapi
+            raise
+
+    return '\n'.join(res)
+
+
+def parse_schema(fname):
+    """
+    Parse the given schema file and return the exprs
+    """
+    try:
+        schema = QAPISchemaParser(open(fname, "r"))
+        check_exprs(schema.exprs)
+        check_docs(schema.docs)
+        return schema.docs
+    except (QAPISchemaError, QAPIExprError), err:
+        print >>sys.stderr, err
+        exit(1)
+
+
+def main(argv):
+    """
+    Takes schema argument, prints result to stdout
+    """
+    if len(argv) != 2:
+        print >>sys.stderr, "%s: need exactly 1 argument: SCHEMA" % argv[0]
+        sys.exit(1)
+
+    docs = parse_schema(argv[1])
+    print texi(docs)
+
+
+if __name__ == "__main__":
+    main(sys.argv)
diff --git a/docs/qapi-code-gen.txt b/docs/qapi-code-gen.txt
index 2841c51..d82e251 100644
--- a/docs/qapi-code-gen.txt
+++ b/docs/qapi-code-gen.txt
@@ -45,16 +45,13 @@ QAPI parser does not).  At present, there is no place where 
a QAPI
 schema requires the use of JSON numbers or null.
 
 Comments are allowed; anything between an unquoted # and the following
-newline is ignored.  Although there is not yet a documentation
-generator, a form of stylized comments has developed for consistently
-documenting details about an expression and when it was added to the
-schema.  The documentation is delimited between two lines of ##, then
-the first line names the expression, an optional overview is provided,
-then individual documentation about each member of 'data' is provided,
-and finally, a 'Since: x.y.z' tag lists the release that introduced
-the expression.  Optional members are tagged with the phrase
-'#optional', often with their default value; and extensions added
-after the expression was first released are also given a '(since
+newline is ignored.  The documentation is delimited between two lines
+of ##, then the first line names the expression, an optional overview
+is provided, then individual documentation about each member of 'data'
+is provided, and finally, a 'Since: x.y.z' tag lists the release that
+introduced the expression.  Optional members are tagged with the
+phrase '#optional', often with their default value; and extensions
+added after the expression was first released are also given a '(since
 x.y.z)' comment.  For example:
 
     ##
@@ -73,12 +70,39 @@ x.y.z)' comment.  For example:
     #           (Since 2.0)
     #
     # Since: 0.14.0
+    #
+    # Notes: You can also make a list:
+    #        - with items
+    #        - like this
+    #
+    # Example:
+    #
+    # -> { "execute": ... }
+    # <- { "return": ... }
+    #
     ##
     { 'struct': 'BlockStats',
       'data': {'*device': 'str', 'stats': 'BlockDeviceStats',
                '*parent': 'BlockStats',
                '*backing': 'BlockStats'} }
 
+It's also possible to create documentation sections, such as:
+
+    ##
+    # = Section
+    # == Subsection
+    #
+    # Some text foo with *emphasis*
+    # 1. with a list
+    # 2. like that
+    #
+    # And some code:
+    # | $ echo foo
+    # | -> do this
+    # | <- get that
+    #
+    ##
+
 The schema sets up a series of types, as well as commands and events
 that will use those types.  Forward references are allowed: the parser
 scans in two passes, where the first pass learns all type names, and
-- 
2.10.0