Re: [PATCH v2 3/8] docs/sphinx: Add new hxtool Sphinx extension

[Alex Bennée]

Peter Maydell <address@hidden> writes:

> Some of our documentation includes sections which are created
> by assembling fragments of texinfo from a .hx source file into
> a .texi file, which is then included from qemu-doc.texi or
> qemu-img.texi.
>
> For Sphinx, rather than creating a file to include, the most natural
> way to handle this is to have a small custom Sphinx extension which
> reads the .hx file and process it.  So instead of:
>  * makefile produces foo.texi from foo.hx
>  * qemu-doc.texi says '@include foo.texi'
> we have:
>  * qemu-doc.rst says 'hxtool-doc:: foo.hx'
>  * the Sphinx extension for hxtool has code that runs to handle that
>    Sphinx directive which reads the .hx file and emits the appropriate
>    documentation contents
>
> This is pretty much the same way the kerneldoc extension works right
> now. It also has the advantage that it should work for third-party
> services like readthedocs that expect to build the docs directly with
> sphinx rather than by invoking our makefiles.
>
> In this commit we implement the hxtool extension.
>
> Note that syntax errors in the rST fragments will be correctly
> reported to the user with the filename and line number within the
> hx file.
>
> Signed-off-by: Peter Maydell <address@hidden>

Reviewed-by: Alex Bennée <address@hidden>
Tested-by: Alex Bennée <address@hidden>

> ---
>  docs/conf.py          |   3 +-
>  docs/sphinx/hxtool.py | 210 ++++++++++++++++++++++++++++++++++++++++++
>  2 files changed, 212 insertions(+), 1 deletion(-)
>  create mode 100644 docs/sphinx/hxtool.py
>
> diff --git a/docs/conf.py b/docs/conf.py
> index 259c6049da7..ee7faa6b4e7 100644
> --- a/docs/conf.py
> +++ b/docs/conf.py
> @@ -54,7 +54,7 @@ needs_sphinx = '1.3'
>  # Add any Sphinx extension module names here, as strings. They can be
>  # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
>  # ones.
> -extensions = ['kerneldoc', 'qmp_lexer']
> +extensions = ['kerneldoc', 'qmp_lexer', 'hxtool']
>  
>  # Add any paths that contain templates here, relative to this directory.
>  templates_path = ['_templates']
> @@ -221,3 +221,4 @@ texinfo_documents = [
>  # find everything.
>  kerneldoc_bin = os.path.join(qemu_docdir, '../scripts/kernel-doc')
>  kerneldoc_srctree = os.path.join(qemu_docdir, '..')
> +hxtool_srctree = os.path.join(qemu_docdir, '..')
> diff --git a/docs/sphinx/hxtool.py b/docs/sphinx/hxtool.py
> new file mode 100644
> index 00000000000..5d6736f3002
> --- /dev/null
> +++ b/docs/sphinx/hxtool.py
> @@ -0,0 +1,210 @@
> +# coding=utf-8
> +#
> +# QEMU hxtool .hx file parsing extension
> +#
> +# Copyright (c) 2020 Linaro
> +#
> +# This work is licensed under the terms of the GNU GPLv2 or later.
> +# See the COPYING file in the top-level directory.
> +"""hxtool is a Sphinx extension that implements the hxtool-doc directive"""
> +
> +# The purpose of this extension is to read fragments of rST
> +# from .hx files, and insert them all into the current document.
> +# The rST fragments are delimited by SRST/ERST lines.
> +# The conf.py file must set the hxtool_srctree config value to
> +# the root of the QEMU source tree.
> +# Each hxtool-doc:: directive takes one argument which is the
> +# path of the .hx file to process, relative to the source tree.
> +
> +import os
> +import re
> +from enum import Enum
> +
> +from docutils import nodes
> +from docutils.statemachine import ViewList
> +from docutils.parsers.rst import directives, Directive
> +from sphinx.errors import ExtensionError
> +from sphinx.util.nodes import nested_parse_with_titles
> +import sphinx
> +
> +# Sphinx up to 1.6 uses AutodocReporter; 1.7 and later
> +# use switch_source_input. Check borrowed from kerneldoc.py.
> +Use_SSI = sphinx.__version__[:3] >= '1.7'
> +if Use_SSI:
> +    from sphinx.util.docutils import switch_source_input
> +else:
> +    from sphinx.ext.autodoc import AutodocReporter
> +
> +__version__ = '1.0'
> +
> +# We parse hx files with a state machine which may be in one of three
> +# states: reading the C code fragment, inside a texi fragment,
> +# or inside a rST fragment.
> +class HxState(Enum):
> +    CTEXT = 1
> +    TEXI = 2
> +    RST = 3
> +
> +def serror(file, lnum, errtext):
> +    """Raise an exception giving a user-friendly syntax error message"""
> +    raise ExtensionError('%s line %d: syntax error: %s' % (file, lnum, 
> errtext))
> +
> +def parse_directive(line):
> +    """Return first word of line, if any"""
> +    return re.split('\W', line)[0]
> +
> +def parse_defheading(file, lnum, line):
> +    """Handle a DEFHEADING directive"""
> +    # The input should be "DEFHEADING(some string)", though note that
> +    # the 'some string' could be the empty string. If the string is
> +    # empty we ignore the directive -- these are used only to add
> +    # blank lines in the plain-text content of the --help output.
> +    #
> +    # Return the heading text
> +    match = re.match(r'DEFHEADING\((.*)\)', line)
> +    if match is None:
> +        serror(file, lnum, "Invalid DEFHEADING line")
> +    return match.group(1)
> +
> +def parse_archheading(file, lnum, line):
> +    """Handle an ARCHHEADING directive"""
> +    # The input should be "ARCHHEADING(some string, other arg)",
> +    # though note that the 'some string' could be the empty string.
> +    # As with DEFHEADING, empty string ARCHHEADINGs will be ignored.
> +    #
> +    # Return the heading text
> +    match = re.match(r'ARCHHEADING\((.*),.*\)', line)
> +    if match is None:
> +        serror(file, lnum, "Invalid ARCHHEADING line")
> +    return match.group(1)
> +
> +class HxtoolDocDirective(Directive):
> +    """Extract rST fragments from the specified .hx file"""
> +    required_argument = 1
> +    optional_arguments = 1
> +    option_spec = {
> +        'hxfile': directives.unchanged_required
> +    }
> +    has_content = False
> +
> +    def run(self):
> +        env = self.state.document.settings.env
> +        hxfile = env.config.hxtool_srctree + '/' + self.arguments[0]
> +
> +        # Tell sphinx of the dependency
> +        env.note_dependency(os.path.abspath(hxfile))
> +
> +        state = HxState.CTEXT
> +        # We build up lines of rST in this ViewList, which we will
> +        # later put into a 'section' node.
> +        rstlist = ViewList()
> +        current_node = None
> +        node_list = []
> +
> +        with open(hxfile) as f:
> +            lines = (l.rstrip() for l in f)
> +            for lnum, line in enumerate(lines, 1):
> +                directive = parse_directive(line)
> +
> +                if directive == 'HXCOMM':
> +                    pass
> +                elif directive == 'STEXI':
> +                    if state == HxState.RST:
> +                        serror(hxfile, lnum, 'expected ERST, found STEXI')
> +                    elif state == HxState.TEXI:
> +                        serror(hxfile, lnum, 'expected ETEXI, found STEXI')
> +                    else:
> +                        state = HxState.TEXI
> +                elif directive == 'ETEXI':
> +                    if state == HxState.RST:
> +                        serror(hxfile, lnum, 'expected ERST, found ETEXI')
> +                    elif state == HxState.CTEXT:
> +                        serror(hxfile, lnum, 'expected STEXI, found ETEXI')
> +                    else:
> +                        state = HxState.CTEXT
> +                elif directive == 'SRST':
> +                    if state == HxState.RST:
> +                        serror(hxfile, lnum, 'expected ERST, found SRST')
> +                    elif state == HxState.TEXI:
> +                        serror(hxfile, lnum, 'expected ETEXI, found SRST')
> +                    else:
> +                        state = HxState.RST
> +                elif directive == 'ERST':
> +                    if state == HxState.TEXI:
> +                        serror(hxfile, lnum, 'expected ETEXI, found ERST')
> +                    elif state == HxState.CTEXT:
> +                        serror(hxfile, lnum, 'expected SRST, found ERST')
> +                    else:
> +                        state = HxState.CTEXT
> +                elif directive == 'DEFHEADING' or directive == 'ARCHHEADING':
> +                    if directive == 'DEFHEADING':
> +                        heading = parse_defheading(hxfile, lnum, line)
> +                    else:
> +                        heading = parse_archheading(hxfile, lnum, line)
> +                    if heading == "":
> +                        continue
> +                    # Put the accumulated rST into the previous node,
> +                    # and then start a fresh section with this heading.
> +                    if len(rstlist) > 0:
> +                        if current_node is None:
> +                            # We had some rST fragments before the first
> +                            # DEFHEADING. We don't have a section to put
> +                            # these in, so rather than magicing up a section,
> +                            # make it a syntax error.
> +                            serror(hxfile, lnum,
> +                                   'first DEFHEADING must precede all rST 
> text')
> +                        self.do_parse(rstlist, current_node)
> +                        rstlist = ViewList()
> +                    if current_node is not None:
> +                        node_list.append(current_node)
> +                    section_id = 'hxtool-%d' % env.new_serialno('hxtool')
> +                    current_node = nodes.section(ids=[section_id])
> +                    current_node += nodes.title(heading, heading)
> +                else:
> +                    # Not a directive: put in output if we are in rST 
> fragment
> +                    if state == HxState.RST:
> +                        # Sphinx counts its lines from 0
> +                        rstlist.append(line, hxfile, lnum - 1)
> +
> +        if current_node is None:
> +            # We don't have multiple sections, so just parse the rst
> +            # fragments into a dummy node so we can return the children.
> +            current_node = nodes.section()
> +            self.do_parse(rstlist, current_node)
> +            return current_node.children
> +        else:
> +            # Put the remaining accumulated rST into the last section, and
> +            # return all the sections.
> +            if len(rstlist) > 0:
> +                self.do_parse(rstlist, current_node)
> +            node_list.append(current_node)
> +            return node_list
> +
> +    # This is from kerneldoc.py -- it works around an API change in
> +    # Sphinx between 1.6 and 1.7. Unlike kerneldoc.py, we use
> +    # sphinx.util.nodes.nested_parse_with_titles() rather than the
> +    # plain self.state.nested_parse(), and so we can drop the saving
> +    # of title_styles and section_level that kerneldoc.py does,
> +    # because nested_parse_with_titles() does that for us.
> +    def do_parse(self, result, node):
> +        if Use_SSI:
> +            with switch_source_input(self.state, result):
> +                nested_parse_with_titles(self.state, result, node)
> +        else:
> +            save = self.state.memo.reporter
> +            self.state.memo.reporter = AutodocReporter(result, 
> self.state.memo.reporter)
> +            try:
> +                nested_parse_with_titles(self.state, result, node)
> +            finally:
> +                self.state.memo.reporter = save
> +
> +def setup(app):
> +    """ Register hxtool-doc directive with Sphinx"""
> +    app.add_config_value('hxtool_srctree', None, 'env')
> +    app.add_directive('hxtool-doc', HxtoolDocDirective)
> +
> +    return dict(
> +        version = __version__,
> +        parallel_read_safe = True,
> +        parallel_write_safe = True
> +    )


-- 
Alex Bennée