Add semantic markup support.

Author: felixfontein

changelogs/fragments/281-semantic-markup.yml

@@ -0,0 +1,2 @@
+major_changes:
+  - Support new semantic markup in documentation (https://github.com/ansible-community/antsibull/pull/281).

pyproject.toml

@@ -40,7 +40,7 @@ antsibull-changelog = ">= 0.14.0"
 asyncio-pool = "*"
 docutils = "*"
 importlib-metadata = {version = "*", python =  "<3.8"}
-jinja2 = "*"
+jinja2 = ">= 3.0"
 # major/minor was introduced here
 packaging = ">= 20.0"
 perky = "*"

src/antsibull/data/docsite/plugin.rst.j2

@@ -213,6 +213,12 @@ See Also
 {%   elif item.module is defined %}
    @{ ('M(' + item['module'] + ')') | rst_ify }@
       The official documentation on the **@{ item['module'] }@** module.
+{%   elif item.plugin is defined and item.plugin_type is defined and item.description %}
+   @{ ('P(' + item['plugin'] + '#' + item['plugin_type'] + ')') | rst_ify }@ @{ item['plugin_type'] }@ plugin
+       @{ item['description'] | rst_ify }@
+{%   elif item.plugin is defined and item.plugin_type is defined %}
+   @{ ('P(' + item['plugin'] + '#' + item['plugin_type'] + ')') | rst_ify }@ @{ item['plugin_type'] }@ plugin
+      The official documentation on the **@{ item['plugin'] }@** @{ item['plugin_type'] }@ plugin.
 {%   elif item.name is defined and item.link is defined and item.description %}
    `@{ item['name'] }@ <@{ item['link'] }@>`_
        @{ item['description'] | rst_ify }@

src/antsibull/data/docsite/role.rst.j2

@@ -161,6 +161,12 @@ See Also
 {%       elif item.module is defined %}
    @{ ('M(' + item['module'] + ')') | rst_ify }@
       The official documentation on the **@{ item['module'] }@** module.
+{%       elif item.plugin is defined and item.plugin_type is defined and item.description %}
+   @{ ('P(' + item['plugin'] + '#' + item['plugin_type'] + ')') | rst_ify }@ @{ item['plugin_type'] }@ plugin
+       @{ item['description'] | rst_ify }@
+{%       elif item.plugin is defined and item.plugin_type is defined %}
+   @{ ('P(' + item['plugin'] + '#' + item['plugin_type'] + ')') | rst_ify }@ @{ item['plugin_type'] }@ plugin
+      The official documentation on the **@{ item['plugin'] }@** @{ item['plugin_type'] }@ plugin.
 {%       elif item.name is defined and item.link is defined and item.description %}
    `@{ item['name'] }@ <@{ item['link'] }@>`_
        @{ item['description'] | rst_ify }@

src/antsibull/jinja2/filters.py

@@ -5,14 +5,17 @@
 """
 
 import re
+from functools import partial
 from html import escape as html_escape
 from urllib.parse import quote
 
 import typing as t
 
-from jinja2.runtime import Undefined
+from jinja2.runtime import Context, Undefined
+from jinja2.utils import pass_context
 
 from ..logging import log
+from ..semantic_helper import parse_option, parse_return_value, augment_plugin_name_type
 
 
 mlog = log.fields(mod=__name__)
@@ -22,32 +25,152 @@
 _ITALIC = re.compile(r"\bI\(([^)]+)\)")
 _BOLD = re.compile(r"\bB\(([^)]+)\)")
 _MODULE = re.compile(r"\bM\(([^).]+)\.([^).]+)\.([^)]+)\)")
+_PLUGIN = re.compile(r"\bP\(([^).]+)\.([^).]+)\.([^)]+)#([a-z]+)\)")
 _URL = re.compile(r"\bU\(([^)]+)\)")
 _LINK = re.compile(r"\bL\(([^)]+), *([^)]+)\)")
 _REF = re.compile(r"\bR\(([^)]+), *([^)]+)\)")
 _CONST = re.compile(r"\bC\(([^)]+)\)")
+_SEM_PARAMETER_STRING = r"\(((?:[^\\)]+|\\.)+)\)"
+_SEM_OPTION_NAME = re.compile(r"\bO" + _SEM_PARAMETER_STRING)
+_SEM_OPTION_VALUE = re.compile(r"\bV" + _SEM_PARAMETER_STRING)
+_SEM_ENV_VARIABLE = re.compile(r"\bE" + _SEM_PARAMETER_STRING)
+_SEM_RET_VALUE = re.compile(r"\bRV" + _SEM_PARAMETER_STRING)
 _RULER = re.compile(r"\bHORIZONTALLINE\b")
+_UNESCAPE = re.compile(r"\\(.)")
 
 _EMAIL_ADDRESS = re.compile(r"(?:<{mail}>|\({mail}\)|{mail})".format(mail=r"[\w.+-]+@[\w.-]+\.\w+"))
 
 
-def html_ify(text):
+def extract_plugin_data(context: Context) -> t.Tuple[t.Optional[str], t.Optional[str]]:
+    plugin_fqcn = context.get('plugin_name')
+    plugin_type = context.get('plugin_type')
+    if plugin_fqcn is None or plugin_type is None:
+        return None, None
+    # if plugin_type == 'role':
+    #     entry_point = context.get('entry_point', 'main')
+    #     # FIXME: use entry_point
+    return plugin_fqcn, plugin_type
+
+
+def _unescape_sem_value(text: str) -> str:
+    return _UNESCAPE.sub(r'\1', text)
+
+
+def _check_plugin(plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str],
+                  matcher: 're.Match') -> None:
+    if plugin_fqcn is None or plugin_type is None:
+        raise Exception(f'The markup {matcher.group(0)} cannot be used outside a plugin or role')
+
+
+def _create_error(text: str, error: str) -> str:  # pylint:disable=unused-argument
+    return '...'  # FIXME
+
+
+def _option_name_html(plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str],
+                      matcher: 're.Match') -> str:
+    _check_plugin(plugin_fqcn, plugin_type, matcher)
+    text = _unescape_sem_value(matcher.group(1))
+    try:
+        plugin_fqcn, plugin_type, option_link, option, value = parse_option(
+            text, plugin_fqcn, plugin_type, require_plugin=False)
+    except ValueError as exc:
+        return _create_error(text, str(exc))
+    if value is None:
+        cls = 'ansible-option'
+        text = f'{option}'
+        strong_start = '<strong>'
+        strong_end = '</strong>'
+    else:
+        cls = 'ansible-option-value'
+        text = f'{option}={value}'
+        strong_start = ''
+        strong_end = ''
+    if plugin_fqcn and plugin_type and plugin_fqcn.count('.') >= 2:
+        # TODO: handle role arguments (entrypoint!)
+        namespace, name, plugin = plugin_fqcn.split('.', 2)
+        url = f'../../{namespace}/{name}/{plugin}_{plugin_type}.html'
+        fragment = f'parameter-{option_link.replace(".", "/")}'
+        link_start = (
+            f'<a class="reference internal" href="{url}#{fragment}">'
+            '<span class="std std-ref"><span class="pre">'
+        )
+        link_end = '</span></span></a>'
+    else:
+        link_start = ''
+        link_end = ''
+    return (
+        f'<code class="{cls} literal notranslate">'
+        f'{strong_start}{link_start}{text}{link_end}{strong_end}</code>'
+    )
+
+
+def _return_value_html(plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str],
+                       matcher: 're.Match') -> str:
+    _check_plugin(plugin_fqcn, plugin_type, matcher)
+    text = _unescape_sem_value(matcher.group(1))
+    try:
+        plugin_fqcn, plugin_type, rv_link, rv, value = parse_return_value(
+            text, plugin_fqcn, plugin_type, require_plugin=False)
+    except ValueError as exc:
+        return _create_error(text, str(exc))
+    cls = 'ansible-return-value'
+    if value is None:
+        text = f'{rv}'
+    else:
+        text = f'{rv}={value}'
+    if plugin_fqcn and plugin_type and plugin_fqcn.count('.') >= 2:
+        namespace, name, plugin = plugin_fqcn.split('.', 2)
+        url = f'../../{namespace}/{name}/{plugin}_{plugin_type}.html'
+        fragment = f'return-{rv_link.replace(".", "/")}'
+        link_start = (
+            f'<a class="reference internal" href="{url}#{fragment}">'
+            '<span class="std std-ref"><span class="pre">'
+        )
+        link_end = '</span></span></a>'
+    else:
+        link_start = ''
+        link_end = ''
+    return f'<code class="{cls} literal notranslate">{link_start}{text}{link_end}</code>'
+
+
+def _value_html(matcher: 're.Match') -> str:
+    text = _unescape_sem_value(matcher.group(1))
+    return f'<code class="ansible-value literal notranslate">{text}</code>'
+
+
+def _env_var_html(matcher: 're.Match') -> str:
+    text = _unescape_sem_value(matcher.group(1))
+    return f'<code class="xref std std-envvar literal notranslate">{text}</code>'
+
+
+@pass_context
+def html_ify(context: Context, text: str) -> str:
     ''' convert symbols like I(this is in italics) to valid HTML '''
 
     flog = mlog.fields(func='html_ify')
     flog.fields(text=text).debug('Enter')
     _counts = {}
 
+    plugin_fqcn, plugin_type = extract_plugin_data(context)
+
     text = html_escape(text)
     text, _counts['italic'] = _ITALIC.subn(r"<em>\1</em>", text)
     text, _counts['bold'] = _BOLD.subn(r"<b>\1</b>", text)
     text, _counts['module'] = _MODULE.subn(
         r"<a href='../../\1/\2/\3_module.html' class='module'>\1.\2.\3</a>", text)
+    text, _counts['plugin'] = _PLUGIN.subn(
+        r"<a href='../../\1/\2/\3_\4.html' class='module plugin-\4'>\1.\2.\3</span>", text)
     text, _counts['url'] = _URL.subn(r"<a href='\1'>\1</a>", text)
     text, _counts['ref'] = _REF.subn(r"<span class='module'>\1</span>", text)
     text, _counts['link'] = _LINK.subn(r"<a href='\2'>\1</a>", text)
     text, _counts['const'] = _CONST.subn(
         r"<code class='docutils literal notranslate'>\1</code>", text)
+    text, _counts['option-name'] = _SEM_OPTION_NAME.subn(
+        partial(_option_name_html, plugin_fqcn, plugin_type), text)
+    text, _counts['option-value'] = _SEM_OPTION_VALUE.subn(_value_html, text)
+    text, _counts['environment-var'] = _SEM_ENV_VARIABLE.subn(_env_var_html, text)
+    text, _counts['return-value'] = _SEM_RET_VALUE.subn(
+        partial(_return_value_html, plugin_fqcn, plugin_type), text)
     text, _counts['ruler'] = _RULER.subn(r"<hr/>", text)
 
     text = text.strip()
@@ -95,6 +218,12 @@ def _rst_ify_module(m: 're.Match') -> str:
     return f"\\ :ref:`{rst_escape(fqcn)} <ansible_collections.{fqcn}_module>`\\ "
 
 
+def _rst_ify_plugin(m: 're.Match') -> str:
+    fqcn = f'{m.group(1)}.{m.group(2)}.{m.group(3)}'
+    plugin_type = m.group(4)
+    return f"\\ :ref:`{rst_escape(fqcn)} <ansible_collections.{fqcn}_{plugin_type}>`\\ "
+
+
 def _escape_url(url: str) -> str:
     # We include '<>[]{}' in safe to allow urls such as 'https://<HOST>:[PORT]/v{version}/' to
     # remain unmangled by percent encoding
@@ -118,20 +247,56 @@ def _rst_ify_const(m: 're.Match') -> str:
     return f"\\ :literal:`{rst_escape(m.group(1), escape_ending_whitespace=True)}`\\ "
 
 
-def rst_ify(text):
+def _rst_ify_option_name(plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str],
+                         m: 're.Match') -> str:
+    _check_plugin(plugin_fqcn, plugin_type, m)
+    text = _unescape_sem_value(m.group(1))
+    text = augment_plugin_name_type(text, plugin_fqcn, plugin_type)
+    return f"\\ :ansopt:`{rst_escape(text, escape_ending_whitespace=True)}`\\ "
+
+
+def _rst_ify_value(m: 're.Match') -> str:
+    text = _unescape_sem_value(m.group(1))
+    return f"\\ :ansval:`{rst_escape(text, escape_ending_whitespace=True)}`\\ "
+
+
+def _rst_ify_return_value(plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str],
+                          m: 're.Match') -> str:
+    _check_plugin(plugin_fqcn, plugin_type, m)
+    text = _unescape_sem_value(m.group(1))
+    text = augment_plugin_name_type(text, plugin_fqcn, plugin_type)
+    return f"\\ :ansretval:`{rst_escape(text, escape_ending_whitespace=True)}`\\ "
+
+
+def _rst_ify_envvar(m: 're.Match') -> str:
+    text = _unescape_sem_value(m.group(1))
+    return f"\\ :envvar:`{rst_escape(text, escape_ending_whitespace=True)}`\\ "
+
+
+@pass_context
+def rst_ify(context: Context, text: str) -> str:
     ''' convert symbols like I(this is in italics) to valid restructured text '''
 
     flog = mlog.fields(func='rst_ify')
     flog.fields(text=text).debug('Enter')
     _counts = {}
 
+    plugin_fqcn, plugin_type = extract_plugin_data(context)
+
     text, _counts['italic'] = _ITALIC.subn(_rst_ify_italic, text)
     text, _counts['bold'] = _BOLD.subn(_rst_ify_bold, text)
     text, _counts['module'] = _MODULE.subn(_rst_ify_module, text)
+    text, _counts['plugin'] = _PLUGIN.subn(_rst_ify_plugin, text)
     text, _counts['link'] = _LINK.subn(_rst_ify_link, text)
     text, _counts['url'] = _URL.subn(_rst_ify_url, text)
     text, _counts['ref'] = _REF.subn(_rst_ify_ref, text)
     text, _counts['const'] = _CONST.subn(_rst_ify_const, text)
+    text, _counts['option-name'] = _SEM_OPTION_NAME.subn(
+        partial(_rst_ify_option_name, plugin_fqcn, plugin_type), text)
+    text, _counts['option-value'] = _SEM_OPTION_VALUE.subn(_rst_ify_value, text)
+    text, _counts['environment-var'] = _SEM_ENV_VARIABLE.subn(_rst_ify_envvar, text)
+    text, _counts['return-value'] = _SEM_RET_VALUE.subn(
+        partial(_rst_ify_return_value, plugin_fqcn, plugin_type), text)
     text, _counts['ruler'] = _RULER.subn('\n\n.. raw:: html\n\n  <hr>\n\n', text)
 
     flog.fields(counts=_counts).info('Number of macros converted to rst equivalents')

src/antsibull/lint_extra_docs.py

@@ -13,6 +13,10 @@
 import docutils.utils
 import rstcheck
 
+from docutils.parsers.rst import roles as docutils_roles
+
+from sphinx_antsibull_ext import roles as antsibull_roles
+
 from .extra_docs import (
     find_extra_docs,
     lint_required_conditions,
@@ -58,6 +62,12 @@ def lint_optional_conditions(content: str, path: str, collection_name: str
     return [(result[0], 0, result[1]) for result in results]
 
 
+def _setup_rstcheck():
+    '''Make sure that rstcheck knows about our roles.'''
+    for name, role in antsibull_roles.ROLES.items():
+        docutils_roles.register_local_role(name, role)
+
+
 def lint_collection_extra_docs_files(path_to_collection: str
                                      ) -> t.List[t.Tuple[str, int, int, str]]:
     try:
@@ -69,6 +79,7 @@ def lint_collection_extra_docs_files(path_to_collection: str
     result = []
     all_labels = set()
     docs = find_extra_docs(path_to_collection)
+    _setup_rstcheck()
     for doc in docs:
         try:
             # Load content

src/antsibull/schemas/docs/base.py

@@ -440,6 +440,12 @@ class SeeAlsoModSchema(BaseModel):
     description: str = ""
 
 
+class SeeAlsoPluginSchema(BaseModel):
+    plugin: str
+    plugin_type: str
+    description: str = ""
+
+
 class SeeAlsoRefSchema(BaseModel):
     description: str
     ref: str
@@ -505,7 +511,10 @@ class DocSchema(BaseModel):
     filename: str = ''
     notes: t.List[str] = []
     requirements: t.List[str] = []
-    seealso: t.List[t.Union[SeeAlsoModSchema, SeeAlsoRefSchema, SeeAlsoLinkSchema]] = []
+    seealso: t.List[t.Union[SeeAlsoModSchema,
+                            SeeAlsoPluginSchema,
+                            SeeAlsoRefSchema,
+                            SeeAlsoLinkSchema]] = []
     todo: t.List[str] = []
     version_added: str = 'historical'
     version_added_collection: str = COLLECTION_NAME_F