diff options
Diffstat (limited to 'Documentation/sphinx')
| -rw-r--r-- | Documentation/sphinx/automarkup.py | 102 | ||||
| -rw-r--r-- | Documentation/sphinx/cdomain.py | 5 | ||||
| -rw-r--r-- | Documentation/sphinx/kerneldoc.py | 61 | ||||
| -rw-r--r-- | Documentation/sphinx/kernellog.py | 28 | ||||
| -rw-r--r-- | Documentation/sphinx/kfigure.py | 40 | ||||
| -rw-r--r-- | Documentation/sphinx/load_config.py | 27 | ||||
| -rwxr-xr-x | Documentation/sphinx/maintainers_include.py | 197 | ||||
| -rw-r--r-- | Documentation/sphinx/parallel-wrapper.sh | 33 | ||||
| -rw-r--r-- | Documentation/sphinx/requirements.txt | 4 |
9 files changed, 459 insertions, 38 deletions
diff --git a/Documentation/sphinx/automarkup.py b/Documentation/sphinx/automarkup.py new file mode 100644 index 000000000000..5b6119ff69f4 --- /dev/null +++ b/Documentation/sphinx/automarkup.py @@ -0,0 +1,102 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright 2019 Jonathan Corbet <[email protected]> +# +# Apply kernel-specific tweaks after the initial document processing +# has been done. +# +from docutils import nodes +from sphinx import addnodes +from sphinx.environment import NoUri +import re + +# +# Regex nastiness. Of course. +# Try to identify "function()" that's not already marked up some +# other way. Sphinx doesn't like a lot of stuff right after a +# :c:func: block (i.e. ":c:func:`mmap()`s" flakes out), so the last +# bit tries to restrict matches to things that won't create trouble. +# +RE_function = re.compile(r'([\w_][\w\d_]+\(\))') + +# +# Many places in the docs refer to common system calls. It is +# pointless to try to cross-reference them and, as has been known +# to happen, somebody defining a function by these names can lead +# to the creation of incorrect and confusing cross references. So +# just don't even try with these names. +# +Skipfuncs = [ 'open', 'close', 'read', 'write', 'fcntl', 'mmap', + 'select', 'poll', 'fork', 'execve', 'clone', 'ioctl', + 'socket' ] + +# +# Find all occurrences of function() and try to replace them with +# appropriate cross references. +# +def markup_funcs(docname, app, node): + cdom = app.env.domains['c'] + t = node.astext() + done = 0 + repl = [ ] + for m in RE_function.finditer(t): + # + # Include any text prior to function() as a normal text node. + # + if m.start() > done: + repl.append(nodes.Text(t[done:m.start()])) + # + # Go through the dance of getting an xref out of the C domain + # + target = m.group(1)[:-2] + target_text = nodes.Text(target + '()') + xref = None + if target not in Skipfuncs: + lit_text = nodes.literal(classes=['xref', 'c', 'c-func']) + lit_text += target_text + pxref = addnodes.pending_xref('', refdomain = 'c', + reftype = 'function', + reftarget = target, modname = None, + classname = None) + # + # XXX The Latex builder will throw NoUri exceptions here, + # work around that by ignoring them. + # + try: + xref = cdom.resolve_xref(app.env, docname, app.builder, + 'function', target, pxref, lit_text) + except NoUri: + xref = None + # + # Toss the xref into the list if we got it; otherwise just put + # the function text. + # + if xref: + repl.append(xref) + else: + repl.append(target_text) + done = m.end() + if done < len(t): + repl.append(nodes.Text(t[done:])) + return repl + +def auto_markup(app, doctree, name): + # + # This loop could eventually be improved on. Someday maybe we + # want a proper tree traversal with a lot of awareness of which + # kinds of nodes to prune. But this works well for now. + # + # The nodes.literal test catches ``literal text``, its purpose is to + # avoid adding cross-references to functions that have been explicitly + # marked with cc:func:. + # + for para in doctree.traverse(nodes.paragraph): + for node in para.traverse(nodes.Text): + if not isinstance(node.parent, nodes.literal): + node.parent.replace(node, markup_funcs(name, app, node)) + +def setup(app): + app.connect('doctree-resolved', auto_markup) + return { + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/Documentation/sphinx/cdomain.py b/Documentation/sphinx/cdomain.py index cf13ff3a656c..cbac8e608dc4 100644 --- a/Documentation/sphinx/cdomain.py +++ b/Documentation/sphinx/cdomain.py @@ -48,7 +48,10 @@ major, minor, patch = sphinx.version_info[:3] def setup(app): - app.override_domain(CDomain) + if (major == 1 and minor < 8): + app.override_domain(CDomain) + else: + app.add_domain(CDomain, override=True) return dict( version = __version__, diff --git a/Documentation/sphinx/kerneldoc.py b/Documentation/sphinx/kerneldoc.py index 9d0a7f08f93b..4bcbd6ae01cd 100644 --- a/Documentation/sphinx/kerneldoc.py +++ b/Documentation/sphinx/kerneldoc.py @@ -37,7 +37,19 @@ import glob from docutils import nodes, statemachine from docutils.statemachine import ViewList from docutils.parsers.rst import directives, Directive -from sphinx.ext.autodoc import AutodocReporter + +# +# AutodocReporter is only good up to Sphinx 1.7 +# +import sphinx + +Use_SSI = sphinx.__version__[:3] >= '1.7' +if Use_SSI: + from sphinx.util.docutils import switch_source_input +else: + from sphinx.ext.autodoc import AutodocReporter + +import kernellog __version__ = '1.0' @@ -47,9 +59,10 @@ class KernelDocDirective(Directive): optional_arguments = 4 option_spec = { 'doc': directives.unchanged_required, - 'functions': directives.unchanged, 'export': directives.unchanged, 'internal': directives.unchanged, + 'identifiers': directives.unchanged, + 'functions': directives.unchanged, } has_content = False @@ -65,6 +78,10 @@ class KernelDocDirective(Directive): tab_width = self.options.get('tab-width', self.state.document.settings.tab_width) + # 'function' is an alias of 'identifiers' + if 'functions' in self.options: + self.options['identifiers'] = self.options.get('functions') + # FIXME: make this nicer and more robust against errors if 'export' in self.options: cmd += ['-export'] @@ -74,11 +91,11 @@ class KernelDocDirective(Directive): export_file_patterns = str(self.options.get('internal')).split() elif 'doc' in self.options: cmd += ['-function', str(self.options.get('doc'))] - elif 'functions' in self.options: - functions = self.options.get('functions').split() - if functions: - for f in functions: - cmd += ['-function', f] + elif 'identifiers' in self.options: + identifiers = self.options.get('identifiers').split() + if identifiers: + for i in identifiers: + cmd += ['-function', i] else: cmd += ['-no-doc-sections'] @@ -90,7 +107,8 @@ class KernelDocDirective(Directive): cmd += [filename] try: - env.app.verbose('calling kernel-doc \'%s\'' % (" ".join(cmd))) + kernellog.verbose(env.app, + 'calling kernel-doc \'%s\'' % (" ".join(cmd))) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() @@ -100,7 +118,8 @@ class KernelDocDirective(Directive): if p.returncode != 0: sys.stderr.write(err) - env.app.warn('kernel-doc \'%s\' failed with return code %d' % (" ".join(cmd), p.returncode)) + kernellog.warn(env.app, + 'kernel-doc \'%s\' failed with return code %d' % (" ".join(cmd), p.returncode)) return [nodes.error(None, nodes.paragraph(text = "kernel-doc missing"))] elif env.config.kerneldoc_verbosity > 0: sys.stderr.write(err) @@ -121,20 +140,28 @@ class KernelDocDirective(Directive): lineoffset += 1 node = nodes.section() - buf = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter + self.do_parse(result, node) + + return node.children + + except Exception as e: # pylint: disable=W0703 + kernellog.warn(env.app, 'kernel-doc \'%s\' processing failed with: %s' % + (" ".join(cmd), str(e))) + return [nodes.error(None, nodes.paragraph(text = "kernel-doc missing"))] + + def do_parse(self, result, node): + if Use_SSI: + with switch_source_input(self.state, result): + self.state.nested_parse(result, 0, node, match_titles=1) + else: + save = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter self.state.memo.reporter = AutodocReporter(result, self.state.memo.reporter) self.state.memo.title_styles, self.state.memo.section_level = [], 0 try: self.state.nested_parse(result, 0, node, match_titles=1) finally: - self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = buf + self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = save - return node.children - - except Exception as e: # pylint: disable=W0703 - env.app.warn('kernel-doc \'%s\' processing failed with: %s' % - (" ".join(cmd), str(e))) - return [nodes.error(None, nodes.paragraph(text = "kernel-doc missing"))] def setup(app): app.add_config_value('kerneldoc_bin', None, 'env') diff --git a/Documentation/sphinx/kernellog.py b/Documentation/sphinx/kernellog.py new file mode 100644 index 000000000000..af924f51a7dc --- /dev/null +++ b/Documentation/sphinx/kernellog.py @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Sphinx has deprecated its older logging interface, but the replacement +# only goes back to 1.6. So here's a wrapper layer to keep around for +# as long as we support 1.4. +# +import sphinx + +if sphinx.__version__[:3] >= '1.6': + UseLogging = True + from sphinx.util import logging + logger = logging.getLogger('kerneldoc') +else: + UseLogging = False + +def warn(app, message): + if UseLogging: + logger.warning(message) + else: + app.warn(message) + +def verbose(app, message): + if UseLogging: + logger.verbose(message) + else: + app.verbose(message) + + diff --git a/Documentation/sphinx/kfigure.py b/Documentation/sphinx/kfigure.py index b97228d2cc0e..fbfe6693bb60 100644 --- a/Documentation/sphinx/kfigure.py +++ b/Documentation/sphinx/kfigure.py @@ -60,6 +60,8 @@ import sphinx from sphinx.util.nodes import clean_astext from six import iteritems +import kernellog + PY3 = sys.version_info[0] == 3 if PY3: @@ -171,20 +173,20 @@ def setupTools(app): This function is called once, when the builder is initiated. """ global dot_cmd, convert_cmd # pylint: disable=W0603 - app.verbose("kfigure: check installed tools ...") + kernellog.verbose(app, "kfigure: check installed tools ...") dot_cmd = which('dot') convert_cmd = which('convert') if dot_cmd: - app.verbose("use dot(1) from: " + dot_cmd) + kernellog.verbose(app, "use dot(1) from: " + dot_cmd) else: - app.warn("dot(1) not found, for better output quality install " - "graphviz from http://www.graphviz.org") + kernellog.warn(app, "dot(1) not found, for better output quality install " + "graphviz from http://www.graphviz.org") if convert_cmd: - app.verbose("use convert(1) from: " + convert_cmd) + kernellog.verbose(app, "use convert(1) from: " + convert_cmd) else: - app.warn( + kernellog.warn(app, "convert(1) not found, for SVG to PDF conversion install " "ImageMagick (https://www.imagemagick.org)") @@ -220,12 +222,13 @@ def convert_image(img_node, translator, src_fname=None): # in kernel builds, use 'make SPHINXOPTS=-v' to see verbose messages - app.verbose('assert best format for: ' + img_node['uri']) + kernellog.verbose(app, 'assert best format for: ' + img_node['uri']) if in_ext == '.dot': if not dot_cmd: - app.verbose("dot from graphviz not available / include DOT raw.") + kernellog.verbose(app, + "dot from graphviz not available / include DOT raw.") img_node.replace_self(file2literal(src_fname)) elif translator.builder.format == 'latex': @@ -252,7 +255,8 @@ def convert_image(img_node, translator, src_fname=None): if translator.builder.format == 'latex': if convert_cmd is None: - app.verbose("no SVG to PDF conversion available / include SVG raw.") + kernellog.verbose(app, + "no SVG to PDF conversion available / include SVG raw.") img_node.replace_self(file2literal(src_fname)) else: dst_fname = path.join(translator.builder.outdir, fname + '.pdf') @@ -265,18 +269,19 @@ def convert_image(img_node, translator, src_fname=None): _name = dst_fname[len(translator.builder.outdir) + 1:] if isNewer(dst_fname, src_fname): - app.verbose("convert: {out}/%s already exists and is newer" % _name) + kernellog.verbose(app, + "convert: {out}/%s already exists and is newer" % _name) else: ok = False mkdir(path.dirname(dst_fname)) if in_ext == '.dot': - app.verbose('convert DOT to: {out}/' + _name) + kernellog.verbose(app, 'convert DOT to: {out}/' + _name) ok = dot2format(app, src_fname, dst_fname) elif in_ext == '.svg': - app.verbose('convert SVG to: {out}/' + _name) + kernellog.verbose(app, 'convert SVG to: {out}/' + _name) ok = svg2pdf(app, src_fname, dst_fname) if not ok: @@ -305,7 +310,8 @@ def dot2format(app, dot_fname, out_fname): with open(out_fname, "w") as out: exit_code = subprocess.call(cmd, stdout = out) if exit_code != 0: - app.warn("Error #%d when calling: %s" % (exit_code, " ".join(cmd))) + kernellog.warn(app, + "Error #%d when calling: %s" % (exit_code, " ".join(cmd))) return bool(exit_code == 0) def svg2pdf(app, svg_fname, pdf_fname): @@ -322,7 +328,7 @@ def svg2pdf(app, svg_fname, pdf_fname): # use stdout and stderr from parent exit_code = subprocess.call(cmd) if exit_code != 0: - app.warn("Error #%d when calling: %s" % (exit_code, " ".join(cmd))) + kernellog.warn(app, "Error #%d when calling: %s" % (exit_code, " ".join(cmd))) return bool(exit_code == 0) @@ -415,15 +421,15 @@ def visit_kernel_render(self, node): app = self.builder.app srclang = node.get('srclang') - app.verbose('visit kernel-render node lang: "%s"' % (srclang)) + kernellog.verbose(app, 'visit kernel-render node lang: "%s"' % (srclang)) tmp_ext = RENDER_MARKUP_EXT.get(srclang, None) if tmp_ext is None: - app.warn('kernel-render: "%s" unknown / include raw.' % (srclang)) + kernellog.warn(app, 'kernel-render: "%s" unknown / include raw.' % (srclang)) return if not dot_cmd and tmp_ext == '.dot': - app.verbose("dot from graphviz not available / include raw.") + kernellog.verbose(app, "dot from graphviz not available / include raw.") return literal_block = node[0] diff --git a/Documentation/sphinx/load_config.py b/Documentation/sphinx/load_config.py index 301a21aa4f63..eeb394b39e2c 100644 --- a/Documentation/sphinx/load_config.py +++ b/Documentation/sphinx/load_config.py @@ -21,6 +21,29 @@ def loadConfig(namespace): and os.path.normpath(namespace["__file__"]) != os.path.normpath(config_file) ): config_file = os.path.abspath(config_file) + # Let's avoid one conf.py file just due to latex_documents + start = config_file.find('Documentation/') + if start >= 0: + start = config_file.find('/', start + 1) + + end = config_file.rfind('/') + if start >= 0 and end > 0: + dir = config_file[start + 1:end] + + print("source directory: %s" % dir) + new_latex_docs = [] + latex_documents = namespace['latex_documents'] + + for l in latex_documents: + if l[0].find(dir + '/') == 0: + has = True + fn = l[0][len(dir) + 1:] + new_latex_docs.append((fn, l[1], l[2], l[3], l[4])) + break + + namespace['latex_documents'] = new_latex_docs + + # If there is an extra conf.py file, load it if os.path.isfile(config_file): sys.stdout.write("load additional sphinx-config: %s\n" % config_file) config = namespace.copy() @@ -29,4 +52,6 @@ def loadConfig(namespace): del config['__file__'] namespace.update(config) else: - sys.stderr.write("WARNING: additional sphinx-config not found: %s\n" % config_file) + config = namespace.copy() + config['tags'].add("subproject") + namespace.update(config) diff --git a/Documentation/sphinx/maintainers_include.py b/Documentation/sphinx/maintainers_include.py new file mode 100755 index 000000000000..dc8fed48d3c2 --- /dev/null +++ b/Documentation/sphinx/maintainers_include.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: GPL-2.0 +# -*- coding: utf-8; mode: python -*- +# pylint: disable=R0903, C0330, R0914, R0912, E0401 + +u""" + maintainers-include + ~~~~~~~~~~~~~~~~~~~ + + Implementation of the ``maintainers-include`` reST-directive. + + :copyright: Copyright (C) 2019 Kees Cook <[email protected]> + :license: GPL Version 2, June 1991 see linux/COPYING for details. + + The ``maintainers-include`` reST-directive performs extensive parsing + specific to the Linux kernel's standard "MAINTAINERS" file, in an + effort to avoid needing to heavily mark up the original plain text. +""" + +import sys +import re +import os.path + +from docutils import statemachine +from docutils.utils.error_reporting import ErrorString +from docutils.parsers.rst import Directive +from docutils.parsers.rst.directives.misc import Include + +__version__ = '1.0' + +def setup(app): + app.add_directive("maintainers-include", MaintainersInclude) + return dict( + version = __version__, + parallel_read_safe = True, + parallel_write_safe = True + ) + +class MaintainersInclude(Include): + u"""MaintainersInclude (``maintainers-include``) directive""" + required_arguments = 0 + + def parse_maintainers(self, path): + """Parse all the MAINTAINERS lines into ReST for human-readability""" + + result = list() + result.append(".. _maintainers:") + result.append("") + + # Poor man's state machine. + descriptions = False + maintainers = False + subsystems = False + + # Field letter to field name mapping. + field_letter = None + fields = dict() + + prev = None + field_prev = "" + field_content = "" + + for line in open(path): + if sys.version_info.major == 2: + line = unicode(line, 'utf-8') + # Have we reached the end of the preformatted Descriptions text? + if descriptions and line.startswith('Maintainers'): + descriptions = False + # Ensure a blank line following the last "|"-prefixed line. + result.append("") + + # Start subsystem processing? This is to skip processing the text + # between the Maintainers heading and the first subsystem name. + if maintainers and not subsystems: + if re.search('^[A-Z0-9]', line): + subsystems = True + + # Drop needless input whitespace. + line = line.rstrip() + + # Linkify all non-wildcard refs to ReST files in Documentation/. + pat = '(Documentation/([^\s\?\*]*)\.rst)' + m = re.search(pat, line) + if m: + # maintainers.rst is in a subdirectory, so include "../". + line = re.sub(pat, ':doc:`%s <../%s>`' % (m.group(2), m.group(2)), line) + + # Check state machine for output rendering behavior. + output = None + if descriptions: + # Escape the escapes in preformatted text. + output = "| %s" % (line.replace("\\", "\\\\")) + # Look for and record field letter to field name mappings: + # R: Designated *reviewer*: FullName <address@domain> + m = re.search("\s(\S):\s", line) + if m: + field_letter = m.group(1) + if field_letter and not field_letter in fields: + m = re.search("\*([^\*]+)\*", line) + if m: + fields[field_letter] = m.group(1) + elif subsystems: + # Skip empty lines: subsystem parser adds them as needed. + if len(line) == 0: + continue + # Subsystem fields are batched into "field_content" + if line[1] != ':': + # Render a subsystem entry as: + # SUBSYSTEM NAME + # ~~~~~~~~~~~~~~ + + # Flush pending field content. + output = field_content + "\n\n" + field_content = "" + + # Collapse whitespace in subsystem name. + heading = re.sub("\s+", " ", line) + output = output + "%s\n%s" % (heading, "~" * len(heading)) + field_prev = "" + else: + # Render a subsystem field as: + # :Field: entry + # entry... + field, details = line.split(':', 1) + details = details.strip() + + # Mark paths (and regexes) as literal text for improved + # readability and to escape any escapes. + if field in ['F', 'N', 'X', 'K']: + # But only if not already marked :) + if not ':doc:' in details: + details = '``%s``' % (details) + + # Comma separate email field continuations. + if field == field_prev and field_prev in ['M', 'R', 'L']: + field_content = field_content + "," + + # Do not repeat field names, so that field entries + # will be collapsed together. + if field != field_prev: + output = field_content + "\n" + field_content = ":%s:" % (fields.get(field, field)) + field_content = field_content + "\n\t%s" % (details) + field_prev = field + else: + output = line + + # Re-split on any added newlines in any above parsing. + if output != None: + for separated in output.split('\n'): + result.append(separated) + + # Update the state machine when we find heading separators. + if line.startswith('----------'): + if prev.startswith('Descriptions'): + descriptions = True + if prev.startswith('Maintainers'): + maintainers = True + + # Retain previous line for state machine transitions. + prev = line + + # Flush pending field contents. + if field_content != "": + for separated in field_content.split('\n'): + result.append(separated) + + output = "\n".join(result) + # For debugging the pre-rendered results... + #print(output, file=open("/tmp/MAINTAINERS.rst", "w")) + + self.state_machine.insert_input( + statemachine.string2lines(output), path) + + def run(self): + """Include the MAINTAINERS file as part of this reST file.""" + if not self.state.document.settings.file_insertion_enabled: + raise self.warning('"%s" directive disabled.' % self.name) + + # Walk up source path directories to find Documentation/../ + path = self.state_machine.document.attributes['source'] + path = os.path.realpath(path) + tail = path + while tail != "Documentation" and tail != "": + (path, tail) = os.path.split(path) + + # Append "MAINTAINERS" + path = os.path.join(path, "MAINTAINERS") + + try: + self.state.document.settings.record_dependencies.add(path) + lines = self.parse_maintainers(path) + except IOError as error: + raise self.severe('Problems with "%s" directive path:\n%s.' % + (self.name, ErrorString(error))) + + return [] diff --git a/Documentation/sphinx/parallel-wrapper.sh b/Documentation/sphinx/parallel-wrapper.sh new file mode 100644 index 000000000000..7daf5133bdd3 --- /dev/null +++ b/Documentation/sphinx/parallel-wrapper.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0+ +# +# Figure out if we should follow a specific parallelism from the make +# environment (as exported by scripts/jobserver-exec), or fall back to +# the "auto" parallelism when "-jN" is not specified at the top-level +# "make" invocation. + +sphinx="$1" +shift || true + +parallel="$PARALLELISM" +if [ -z "$parallel" ] ; then + # If no parallelism is specified at the top-level make, then + # fall back to the expected "-jauto" mode that the "htmldocs" + # target has had. + auto=$(perl -e 'open IN,"'"$sphinx"' --version 2>&1 |"; + while (<IN>) { + if (m/([\d\.]+)/) { + print "auto" if ($1 >= "1.7") + } + } + close IN') + if [ -n "$auto" ] ; then + parallel="$auto" + fi +fi +# Only if some parallelism has been determined do we add the -jN option. +if [ -n "$parallel" ] ; then + parallel="-j$parallel" +fi + +exec "$sphinx" "$parallel" "$@" diff --git a/Documentation/sphinx/requirements.txt b/Documentation/sphinx/requirements.txt index 742be3e12619..14e29a0ae480 100644 --- a/Documentation/sphinx/requirements.txt +++ b/Documentation/sphinx/requirements.txt @@ -1,3 +1,3 @@ -docutils==0.12 -Sphinx==1.4.9 +docutils +Sphinx==1.7.9 sphinx_rtd_theme |