Implement updated semantic markup draft and add option and return value linking.

Author: felixfontein

pyproject.toml

@@ -40,7 +40,7 @@ antsibull-changelog = ">= 0.10.0"
 asyncio-pool = "*"
 docutils = "*"
 importlib-metadata = {version = "*", python =  "<3.8"}
-jinja2 = "*"
+jinja2 = ">= 3.0"
 # major/minor was introduced here
 packaging = ">= 20.0"
 perky = "*"

src/antsibull/jinja2/filters.py

@@ -5,14 +5,17 @@
 """
 
 import re
+from functools import partial
 from html import escape as html_escape
 from urllib.parse import quote
 
 import typing as t
 
-from jinja2.runtime import Undefined
+from jinja2.runtime import Context, Undefined
+from jinja2.utils import pass_context
 
 from ..logging import log
+from ..semantic_helper import parse_option, parse_return_value, augment_plugin_name_type
 
 
 mlog = log.fields(mod=__name__)
@@ -27,26 +30,127 @@
 _LINK = re.compile(r"\bL\(([^)]+), *([^)]+)\)")
 _REF = re.compile(r"\bR\(([^)]+), *([^)]+)\)")
 _CONST = re.compile(r"\bC\(([^)]+)\)")
-_SEM_OPTION_NAME = re.compile(r"\bO\(([^)]+)\)")
-_SEM_OPTION_VALUE = re.compile(r"\bV\(([^)]+)\)")
-_SEM_ENV_VARIABLE = re.compile(r"\bE\(([^)]+)\)")
+_SEM_PARAMETER_STRING = r"\(((?:[^\\)]+|\\.)+)\)"
+_SEM_OPTION_NAME = re.compile(r"\bO" + _SEM_PARAMETER_STRING)
+_SEM_OPTION_VALUE = re.compile(r"\bV" + _SEM_PARAMETER_STRING)
+_SEM_ENV_VARIABLE = re.compile(r"\bE" + _SEM_PARAMETER_STRING)
+_SEM_RET_VALUE = re.compile(r"\bRV" + _SEM_PARAMETER_STRING)
 _RULER = re.compile(r"\bHORIZONTALLINE\b")
-
-
-def _option_name_html(matcher):
-    text = matcher.group(1)
-    if '=' not in text and ':' not in text:
-        return f'<code class="ansible-option literal notranslate"><strong>{text}</strong></code>'
-    return f'<code class="ansible-option-value literal notranslate">{text}</code>'
-
-
-def html_ify(text):
+_UNESCAPE = re.compile(r"\\(.)")
+
+
+def extract_plugin_data(context: Context) -> t.Tuple[t.Optional[str], t.Optional[str]]:
+    plugin_fqcn = context.get('plugin_name')
+    plugin_type = context.get('plugin_type')
+    if plugin_fqcn is None or plugin_type is None:
+        return None, None
+    # if plugin_type == 'role':
+    #     entry_point = context.get('entry_point', 'main')
+    #     # FIXME: use entry_point
+    return plugin_fqcn, plugin_type
+
+
+def _unescape_sem_value(text: str) -> str:
+    return _UNESCAPE.sub(r'\1', text)
+
+
+def _check_plugin(plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str],
+                  matcher: 're.Match') -> None:
+    if plugin_fqcn is None or plugin_type is None:
+        raise Exception(f'The markup {matcher.group(0)} cannot be used outside a plugin or role')
+
+
+def _create_error(text: str, error: str) -> str:  # pylint:disable=unused-argument
+    return '...'  # FIXME
+
+
+def _option_name_html(plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str],
+                      matcher: 're.Match') -> str:
+    _check_plugin(plugin_fqcn, plugin_type, matcher)
+    text = _unescape_sem_value(matcher.group(1))
+    try:
+        plugin_fqcn, plugin_type, option_link, option, value = parse_option(
+            text, plugin_fqcn, plugin_type, require_plugin=False)
+    except ValueError as exc:
+        return _create_error(text, str(exc))
+    if value is None:
+        cls = 'ansible-option'
+        text = f'{option}'
+        strong_start = '<strong>'
+        strong_end = '</strong>'
+    else:
+        cls = 'ansible-option-value'
+        text = f'{option}={value}'
+        strong_start = ''
+        strong_end = ''
+    if plugin_fqcn and plugin_type and plugin_fqcn.count('.') >= 2:
+        # TODO: handle role arguments (entrypoint!)
+        namespace, name, plugin = plugin_fqcn.split('.', 2)
+        url = f'../../{namespace}/{name}/{plugin}_{plugin_type}.html'
+        fragment = f'parameter-{option_link.replace(".", "/")}'
+        link_start = (
+            f'<a class="reference internal" href="{url}#{fragment}">'
+            '<span class="std std-ref"><span class="pre">'
+        )
+        link_end = '</span></span></a>'
+    else:
+        link_start = ''
+        link_end = ''
+    return (
+        f'<code class="{cls} literal notranslate">'
+        f'{strong_start}{link_start}{text}{link_end}{strong_end}</code>'
+    )
+
+
+def _return_value_html(plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str],
+                       matcher: 're.Match') -> str:
+    _check_plugin(plugin_fqcn, plugin_type, matcher)
+    text = _unescape_sem_value(matcher.group(1))
+    try:
+        plugin_fqcn, plugin_type, rv_link, rv, value = parse_return_value(
+            text, plugin_fqcn, plugin_type, require_plugin=False)
+    except ValueError as exc:
+        return _create_error(text, str(exc))
+    cls = 'ansible-return-value'
+    if value is None:
+        text = f'{rv}'
+    else:
+        text = f'{rv}={value}'
+    if plugin_fqcn and plugin_type and plugin_fqcn.count('.') >= 2:
+        namespace, name, plugin = plugin_fqcn.split('.', 2)
+        url = f'../../{namespace}/{name}/{plugin}_{plugin_type}.html'
+        fragment = f'return-{rv_link.replace(".", "/")}'
+        link_start = (
+            f'<a class="reference internal" href="{url}#{fragment}">'
+            '<span class="std std-ref"><span class="pre">'
+        )
+        link_end = '</span></span></a>'
+    else:
+        link_start = ''
+        link_end = ''
+    return f'<code class="{cls} literal notranslate">{link_start}{text}{link_end}</code>'
+
+
+def _value_html(matcher: 're.Match') -> str:
+    text = _unescape_sem_value(matcher.group(1))
+    return f'<code class="ansible-value literal notranslate">{text}</code>'
+
+
+def _env_var_html(matcher: 're.Match') -> str:
+    text = _unescape_sem_value(matcher.group(1))
+    return f'<code class="xref std std-envvar literal notranslate">{text}</code>'
+
+
+@pass_context
+def html_ify(context: Context, text: str) -> str:
     ''' convert symbols like I(this is in italics) to valid HTML '''
 
     flog = mlog.fields(func='html_ify')
     flog.fields(text=text).debug('Enter')
     _counts = {}
 
+    plugin_fqcn, plugin_type = extract_plugin_data(context)
+
     text = html_escape(text)
     text, _counts['italic'] = _ITALIC.subn(r"<em>\1</em>", text)
     text, _counts['bold'] = _BOLD.subn(r"<b>\1</b>", text)
@@ -59,11 +163,12 @@ def html_ify(text):
     text, _counts['link'] = _LINK.subn(r"<a href='\2'>\1</a>", text)
     text, _counts['const'] = _CONST.subn(
         r"<code class='docutils literal notranslate'>\1</code>", text)
-    text, _counts['option-name'] = _SEM_OPTION_NAME.subn(_option_name_html, text)
-    text, _counts['option-value'] = _SEM_OPTION_VALUE.subn(
-        r"<code class='ansible-value literal notranslate'>\1</code>", text)
-    text, _counts['environment-var'] = _SEM_ENV_VARIABLE.subn(
-        r"<code class='xref std std-envvar literal notranslate'>\1</code>", text)
+    text, _counts['option-name'] = _SEM_OPTION_NAME.subn(
+        partial(_option_name_html, plugin_fqcn, plugin_type), text)
+    text, _counts['option-value'] = _SEM_OPTION_VALUE.subn(_value_html, text)
+    text, _counts['environment-var'] = _SEM_ENV_VARIABLE.subn(_env_var_html, text)
+    text, _counts['return-value'] = _SEM_RET_VALUE.subn(
+        partial(_return_value_html, plugin_fqcn, plugin_type), text)
     text, _counts['ruler'] = _RULER.subn(r"<hr/>", text)
 
     text = text.strip()
@@ -134,35 +239,55 @@ def _rst_ify_const(m: 're.Match') -> str:
     return f"\\ :literal:`{rst_escape(m.group(1), escape_ending_whitespace=True)}`\\ "
 
 
-def _rst_ify_option_name(m):
-    return f"\\ :ansopt:`{rst_escape(m.group(1), escape_ending_whitespace=True)}`\\ "
+def _rst_ify_option_name(plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str],
+                         m: 're.Match') -> str:
+    _check_plugin(plugin_fqcn, plugin_type, m)
+    text = _unescape_sem_value(m.group(1))
+    text = augment_plugin_name_type(text, plugin_fqcn, plugin_type)
+    return f"\\ :ansopt:`{rst_escape(text, escape_ending_whitespace=True)}`\\ "
+
 
+def _rst_ify_value(m: 're.Match') -> str:
+    text = _unescape_sem_value(m.group(1))
+    return f"\\ :ansval:`{rst_escape(text, escape_ending_whitespace=True)}`\\ "
 
-def _rst_ify_value(m):
-    return f"\\ :ansval:`{rst_escape(m.group(1), escape_ending_whitespace=True)}`\\ "
 
+def _rst_ify_return_value(plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str],
+                          m: 're.Match') -> str:
+    _check_plugin(plugin_fqcn, plugin_type, m)
+    text = _unescape_sem_value(m.group(1))
+    text = augment_plugin_name_type(text, plugin_fqcn, plugin_type)
+    return f"\\ :ansretval:`{rst_escape(text, escape_ending_whitespace=True)}`\\ "
 
-def _rst_ify_envvar(m):
-    return f"\\ :envvar:`{rst_escape(m.group(1), escape_ending_whitespace=True)}`\\ "
 
+def _rst_ify_envvar(m: 're.Match') -> str:
+    text = _unescape_sem_value(m.group(1))
+    return f"\\ :envvar:`{rst_escape(text, escape_ending_whitespace=True)}`\\ "
 
-def rst_ify(text):
+
+@pass_context
+def rst_ify(context: Context, text: str) -> str:
     ''' convert symbols like I(this is in italics) to valid restructured text '''
 
     flog = mlog.fields(func='rst_ify')
     flog.fields(text=text).debug('Enter')
     _counts = {}
 
+    plugin_fqcn, plugin_type = extract_plugin_data(context)
+
     text, _counts['italic'] = _ITALIC.subn(_rst_ify_italic, text)
     text, _counts['bold'] = _BOLD.subn(_rst_ify_bold, text)
     text, _counts['module'] = _MODULE.subn(_rst_ify_module, text)
     text, _counts['link'] = _LINK.subn(_rst_ify_link, text)
     text, _counts['url'] = _URL.subn(_rst_ify_url, text)
     text, _counts['ref'] = _REF.subn(_rst_ify_ref, text)
     text, _counts['const'] = _CONST.subn(_rst_ify_const, text)
-    text, _counts['option-name'] = _SEM_OPTION_NAME.subn(_rst_ify_option_name, text)
+    text, _counts['option-name'] = _SEM_OPTION_NAME.subn(
+        partial(_rst_ify_option_name, plugin_fqcn, plugin_type), text)
     text, _counts['option-value'] = _SEM_OPTION_VALUE.subn(_rst_ify_value, text)
     text, _counts['environment-var'] = _SEM_ENV_VARIABLE.subn(_rst_ify_envvar, text)
+    text, _counts['return-value'] = _SEM_RET_VALUE.subn(
+        partial(_rst_ify_return_value, plugin_fqcn, plugin_type), text)
     text, _counts['ruler'] = _RULER.subn('\n\n.. raw:: html\n\n  <hr>\n\n', text)
 
     flog.fields(counts=_counts).info('Number of macros converted to rst equivalents')

src/antsibull/semantic_helper.py

@@ -0,0 +1,82 @@
+# Copyright: (c) 2021, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+"""
+Helpers for parsing semantic markup.
+"""
+
+import re
+
+import typing as t
+
+
+_ARRAY_STUB_RE = re.compile(r'\[([^\]]*)\]')
+_FQCN_TYPE_PREFIX_RE = re.compile(r'^([^.]+\.[^.]+\.[^#]+)#([a-z]+):(.*)$')
+_IGNORE_MARKER = 'ignore:'
+
+
+def _remove_array_stubs(text: str) -> str:
+    return _ARRAY_STUB_RE.sub('', text)
+
+
+def parse_option(text: str, plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str],
+                 require_plugin=False) -> t.Tuple[str, str, str, str, t.Optional[str]]:
+    """
+    Given the contents of O(...) / :ansopt:`...` with potential escaping removed,
+    split it into plugin FQCN, plugin type, option link name, option name, and option value.
+    """
+    value = None
+    if '=' in text:
+        text, value = text.split('=', 1)
+    m = _FQCN_TYPE_PREFIX_RE.match(text)
+    if m:
+        plugin_fqcn = m.group(1)
+        plugin_type = m.group(2)
+        text = m.group(3)
+    elif require_plugin:
+        raise ValueError('Cannot extract plugin name and type')
+    elif text.startswith(_IGNORE_MARKER):
+        plugin_fqcn = ''
+        plugin_type = ''
+        text = text[len(_IGNORE_MARKER):]
+    if ':' in text or '#' in text:
+        raise ValueError(f'Invalid option name "{text}"')
+    return plugin_fqcn, plugin_type, _remove_array_stubs(text), text, value
+
+
+def parse_return_value(text: str, plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str],
+                       require_plugin=False) -> t.Tuple[str, str, str, str, t.Optional[str]]:
+    """
+    Given the contents of RV(...) / :ansretval:`...` with potential escaping removed,
+    split it into plugin FQCN, plugin type, option link name, option name, and option value.
+    """
+    value = None
+    if '=' in text:
+        text, value = text.split('=', 1)
+    m = _FQCN_TYPE_PREFIX_RE.match(text)
+    if m:
+        plugin_fqcn = m.group(1)
+        plugin_type = m.group(2)
+        text = m.group(3)
+    elif require_plugin:
+        raise ValueError('Cannot extract plugin name and type')
+    elif text.startswith(_IGNORE_MARKER):
+        plugin_fqcn = ''
+        plugin_type = ''
+        text = text[len(_IGNORE_MARKER):]
+    if ':' in text or '#' in text:
+        raise ValueError(f'Invalid return value name "{text}"')
+    return plugin_fqcn, plugin_type, _remove_array_stubs(text), text, value
+
+
+def augment_plugin_name_type(text: str, plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str]
+                             ) -> str:
+    """
+    Given the text contents of O(...) or RV(...) and a plugin's FQCN and type, insert
+    the FQCN and type if they are not already present.
+    """
+    value = None
+    if '=' in text:
+        text, value = text.split('=', 1)
+    if ':' not in text and plugin_fqcn and plugin_type:
+        text = f'{plugin_fqcn}#{plugin_type}:{text}'
+    return text if value is None else f'{text}={value}'