Skip to content

Commit e802838

Browse files
committed
Add semantic markup support.
1 parent fe69059 commit e802838

File tree

11 files changed

+461
-6
lines changed

11 files changed

+461
-6
lines changed
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
major_changes:
2+
- Support new semantic markup in documentation (https://github.com/ansible-community/antsibull-docs/pull/4).

pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ ansible-pygments = "*"
3838
antsibull-core = ">= 1.0.0, < 2.0.0"
3939
asyncio-pool = "*"
4040
docutils = "*"
41-
jinja2 = "*"
41+
jinja2 = ">= 3.0"
4242
rstcheck = ">= 3.0.0, < 7.0.0"
4343
sphinx = "*"
4444

src/antsibull_docs/jinja2/filters.py

Lines changed: 169 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,19 @@
88

99
import json
1010
import re
11+
from functools import partial
1112
from html import escape as html_escape
1213
from urllib.parse import quote
1314

1415
import typing as t
1516

16-
from jinja2.runtime import Undefined
17+
from jinja2.runtime import Context, Undefined
18+
from jinja2.utils import pass_context
1719

1820
from antsibull_core.logging import log
1921

22+
from ..semantic_helper import parse_option, parse_return_value, augment_plugin_name_type
23+
2024

2125
mlog = log.fields(mod=__name__)
2226

@@ -25,32 +29,152 @@
2529
_ITALIC = re.compile(r"\bI\(([^)]+)\)")
2630
_BOLD = re.compile(r"\bB\(([^)]+)\)")
2731
_MODULE = re.compile(r"\bM\(([^).]+)\.([^).]+)\.([^)]+)\)")
32+
_PLUGIN = re.compile(r"\bP\(([^).]+)\.([^).]+)\.([^)]+)#([a-z]+)\)")
2833
_URL = re.compile(r"\bU\(([^)]+)\)")
2934
_LINK = re.compile(r"\bL\(([^)]+), *([^)]+)\)")
3035
_REF = re.compile(r"\bR\(([^)]+), *([^)]+)\)")
3136
_CONST = re.compile(r"\bC\(([^)]+)\)")
37+
_SEM_PARAMETER_STRING = r"\(((?:[^\\)]+|\\.)+)\)"
38+
_SEM_OPTION_NAME = re.compile(r"\bO" + _SEM_PARAMETER_STRING)
39+
_SEM_OPTION_VALUE = re.compile(r"\bV" + _SEM_PARAMETER_STRING)
40+
_SEM_ENV_VARIABLE = re.compile(r"\bE" + _SEM_PARAMETER_STRING)
41+
_SEM_RET_VALUE = re.compile(r"\bRV" + _SEM_PARAMETER_STRING)
3242
_RULER = re.compile(r"\bHORIZONTALLINE\b")
43+
_UNESCAPE = re.compile(r"\\(.)")
3344

3445
_EMAIL_ADDRESS = re.compile(r"(?:<{mail}>|\({mail}\)|{mail})".format(mail=r"[\w.+-]+@[\w.-]+\.\w+"))
3546

3647

37-
def html_ify(text):
48+
def extract_plugin_data(context: Context) -> t.Tuple[t.Optional[str], t.Optional[str]]:
49+
plugin_fqcn = context.get('plugin_name')
50+
plugin_type = context.get('plugin_type')
51+
if plugin_fqcn is None or plugin_type is None:
52+
return None, None
53+
# if plugin_type == 'role':
54+
# entry_point = context.get('entry_point', 'main')
55+
# # FIXME: use entry_point
56+
return plugin_fqcn, plugin_type
57+
58+
59+
def _unescape_sem_value(text: str) -> str:
60+
return _UNESCAPE.sub(r'\1', text)
61+
62+
63+
def _check_plugin(plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str],
64+
matcher: 're.Match') -> None:
65+
if plugin_fqcn is None or plugin_type is None:
66+
raise Exception(f'The markup {matcher.group(0)} cannot be used outside a plugin or role')
67+
68+
69+
def _create_error(text: str, error: str) -> str: # pylint:disable=unused-argument
70+
return '...' # FIXME
71+
72+
73+
def _option_name_html(plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str],
74+
matcher: 're.Match') -> str:
75+
_check_plugin(plugin_fqcn, plugin_type, matcher)
76+
text = _unescape_sem_value(matcher.group(1))
77+
try:
78+
plugin_fqcn, plugin_type, option_link, option, value = parse_option(
79+
text, plugin_fqcn, plugin_type, require_plugin=False)
80+
except ValueError as exc:
81+
return _create_error(text, str(exc))
82+
if value is None:
83+
cls = 'ansible-option'
84+
text = f'{option}'
85+
strong_start = '<strong>'
86+
strong_end = '</strong>'
87+
else:
88+
cls = 'ansible-option-value'
89+
text = f'{option}={value}'
90+
strong_start = ''
91+
strong_end = ''
92+
if plugin_fqcn and plugin_type and plugin_fqcn.count('.') >= 2:
93+
# TODO: handle role arguments (entrypoint!)
94+
namespace, name, plugin = plugin_fqcn.split('.', 2)
95+
url = f'../../{namespace}/{name}/{plugin}_{plugin_type}.html'
96+
fragment = f'parameter-{quote(option_link.replace(".", "/"))}'
97+
link_start = (
98+
f'<a class="reference internal" href="{url}#{fragment}">'
99+
'<span class="std std-ref"><span class="pre">'
100+
)
101+
link_end = '</span></span></a>'
102+
else:
103+
link_start = ''
104+
link_end = ''
105+
return (
106+
f'<code class="{cls} literal notranslate">'
107+
f'{strong_start}{link_start}{text}{link_end}{strong_end}</code>'
108+
)
109+
110+
111+
def _return_value_html(plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str],
112+
matcher: 're.Match') -> str:
113+
_check_plugin(plugin_fqcn, plugin_type, matcher)
114+
text = _unescape_sem_value(matcher.group(1))
115+
try:
116+
plugin_fqcn, plugin_type, rv_link, rv, value = parse_return_value(
117+
text, plugin_fqcn, plugin_type, require_plugin=False)
118+
except ValueError as exc:
119+
return _create_error(text, str(exc))
120+
cls = 'ansible-return-value'
121+
if value is None:
122+
text = f'{rv}'
123+
else:
124+
text = f'{rv}={value}'
125+
if plugin_fqcn and plugin_type and plugin_fqcn.count('.') >= 2:
126+
namespace, name, plugin = plugin_fqcn.split('.', 2)
127+
url = f'../../{namespace}/{name}/{plugin}_{plugin_type}.html'
128+
fragment = f'return-{quote(rv_link.replace(".", "/"))}'
129+
link_start = (
130+
f'<a class="reference internal" href="{url}#{fragment}">'
131+
'<span class="std std-ref"><span class="pre">'
132+
)
133+
link_end = '</span></span></a>'
134+
else:
135+
link_start = ''
136+
link_end = ''
137+
return f'<code class="{cls} literal notranslate">{link_start}{text}{link_end}</code>'
138+
139+
140+
def _value_html(matcher: 're.Match') -> str:
141+
text = _unescape_sem_value(matcher.group(1))
142+
return f'<code class="ansible-value literal notranslate">{text}</code>'
143+
144+
145+
def _env_var_html(matcher: 're.Match') -> str:
146+
text = _unescape_sem_value(matcher.group(1))
147+
return f'<code class="xref std std-envvar literal notranslate">{text}</code>'
148+
149+
150+
@pass_context
151+
def html_ify(context: Context, text: str) -> str:
38152
''' convert symbols like I(this is in italics) to valid HTML '''
39153

40154
flog = mlog.fields(func='html_ify')
41155
flog.fields(text=text).debug('Enter')
42156
_counts = {}
43157

158+
plugin_fqcn, plugin_type = extract_plugin_data(context)
159+
44160
text = html_escape(text)
45161
text, _counts['italic'] = _ITALIC.subn(r"<em>\1</em>", text)
46162
text, _counts['bold'] = _BOLD.subn(r"<b>\1</b>", text)
47163
text, _counts['module'] = _MODULE.subn(
48164
r"<a href='../../\1/\2/\3_module.html' class='module'>\1.\2.\3</a>", text)
165+
text, _counts['plugin'] = _PLUGIN.subn(
166+
r"<a href='../../\1/\2/\3_\4.html' class='module plugin-\4'>\1.\2.\3</span>", text)
49167
text, _counts['url'] = _URL.subn(r"<a href='\1'>\1</a>", text)
50168
text, _counts['ref'] = _REF.subn(r"<span class='module'>\1</span>", text)
51169
text, _counts['link'] = _LINK.subn(r"<a href='\2'>\1</a>", text)
52170
text, _counts['const'] = _CONST.subn(
53171
r"<code class='docutils literal notranslate'>\1</code>", text)
172+
text, _counts['option-name'] = _SEM_OPTION_NAME.subn(
173+
partial(_option_name_html, plugin_fqcn, plugin_type), text)
174+
text, _counts['option-value'] = _SEM_OPTION_VALUE.subn(_value_html, text)
175+
text, _counts['environment-var'] = _SEM_ENV_VARIABLE.subn(_env_var_html, text)
176+
text, _counts['return-value'] = _SEM_RET_VALUE.subn(
177+
partial(_return_value_html, plugin_fqcn, plugin_type), text)
54178
text, _counts['ruler'] = _RULER.subn(r"<hr/>", text)
55179

56180
text = text.strip()
@@ -98,6 +222,12 @@ def _rst_ify_module(m: 're.Match') -> str:
98222
return f"\\ :ref:`{rst_escape(fqcn)} <ansible_collections.{fqcn}_module>`\\ "
99223

100224

225+
def _rst_ify_plugin(m: 're.Match') -> str:
226+
fqcn = f'{m.group(1)}.{m.group(2)}.{m.group(3)}'
227+
plugin_type = m.group(4)
228+
return f"\\ :ref:`{rst_escape(fqcn)} <ansible_collections.{fqcn}_{plugin_type}>`\\ "
229+
230+
101231
def _escape_url(url: str) -> str:
102232
# We include '<>[]{}' in safe to allow urls such as 'https://<HOST>:[PORT]/v{version}/' to
103233
# remain unmangled by percent encoding
@@ -121,20 +251,56 @@ def _rst_ify_const(m: 're.Match') -> str:
121251
return f"\\ :literal:`{rst_escape(m.group(1), escape_ending_whitespace=True)}`\\ "
122252

123253

124-
def rst_ify(text):
254+
def _rst_ify_option_name(plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str],
255+
m: 're.Match') -> str:
256+
_check_plugin(plugin_fqcn, plugin_type, m)
257+
text = _unescape_sem_value(m.group(1))
258+
text = augment_plugin_name_type(text, plugin_fqcn, plugin_type)
259+
return f"\\ :ansopt:`{rst_escape(text, escape_ending_whitespace=True)}`\\ "
260+
261+
262+
def _rst_ify_value(m: 're.Match') -> str:
263+
text = _unescape_sem_value(m.group(1))
264+
return f"\\ :ansval:`{rst_escape(text, escape_ending_whitespace=True)}`\\ "
265+
266+
267+
def _rst_ify_return_value(plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str],
268+
m: 're.Match') -> str:
269+
_check_plugin(plugin_fqcn, plugin_type, m)
270+
text = _unescape_sem_value(m.group(1))
271+
text = augment_plugin_name_type(text, plugin_fqcn, plugin_type)
272+
return f"\\ :ansretval:`{rst_escape(text, escape_ending_whitespace=True)}`\\ "
273+
274+
275+
def _rst_ify_envvar(m: 're.Match') -> str:
276+
text = _unescape_sem_value(m.group(1))
277+
return f"\\ :envvar:`{rst_escape(text, escape_ending_whitespace=True)}`\\ "
278+
279+
280+
@pass_context
281+
def rst_ify(context: Context, text: str) -> str:
125282
''' convert symbols like I(this is in italics) to valid restructured text '''
126283

127284
flog = mlog.fields(func='rst_ify')
128285
flog.fields(text=text).debug('Enter')
129286
_counts = {}
130287

288+
plugin_fqcn, plugin_type = extract_plugin_data(context)
289+
131290
text, _counts['italic'] = _ITALIC.subn(_rst_ify_italic, text)
132291
text, _counts['bold'] = _BOLD.subn(_rst_ify_bold, text)
133292
text, _counts['module'] = _MODULE.subn(_rst_ify_module, text)
293+
text, _counts['plugin'] = _PLUGIN.subn(_rst_ify_plugin, text)
134294
text, _counts['link'] = _LINK.subn(_rst_ify_link, text)
135295
text, _counts['url'] = _URL.subn(_rst_ify_url, text)
136296
text, _counts['ref'] = _REF.subn(_rst_ify_ref, text)
137297
text, _counts['const'] = _CONST.subn(_rst_ify_const, text)
298+
text, _counts['option-name'] = _SEM_OPTION_NAME.subn(
299+
partial(_rst_ify_option_name, plugin_fqcn, plugin_type), text)
300+
text, _counts['option-value'] = _SEM_OPTION_VALUE.subn(_rst_ify_value, text)
301+
text, _counts['environment-var'] = _SEM_ENV_VARIABLE.subn(_rst_ify_envvar, text)
302+
text, _counts['return-value'] = _SEM_RET_VALUE.subn(
303+
partial(_rst_ify_return_value, plugin_fqcn, plugin_type), text)
138304
text, _counts['ruler'] = _RULER.subn('\n\n.. raw:: html\n\n <hr>\n\n', text)
139305

140306
flog.fields(counts=_counts).info('Number of macros converted to rst equivalents')

src/antsibull_docs/lint_extra_docs.py

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@
1010
import re
1111
import typing as t
1212

13+
from docutils.parsers.rst import roles as docutils_roles
14+
15+
from sphinx_antsibull_ext import roles as antsibull_roles
16+
1317
from .extra_docs import (
1418
find_extra_docs,
1519
lint_required_conditions,
@@ -36,6 +40,12 @@ def lint_optional_conditions(content: str, path: str, collection_name: str
3640
return check_rst_content(content, filename=path)
3741

3842

43+
def _setup_antsibull_roles():
44+
'''Make sure that rstcheck knows about our roles.'''
45+
for name, role in antsibull_roles.ROLES.items():
46+
docutils_roles.register_local_role(name, role)
47+
48+
3949
def lint_collection_extra_docs_files(path_to_collection: str
4050
) -> t.List[t.Tuple[str, int, int, str]]:
4151
try:
@@ -47,6 +57,7 @@ def lint_collection_extra_docs_files(path_to_collection: str
4757
result = []
4858
all_labels = set()
4959
docs = find_extra_docs(path_to_collection)
60+
_setup_antsibull_roles()
5061
for doc in docs:
5162
try:
5263
# Load content

src/antsibull_docs/semantic_helper.py

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
# Author: Felix Fontein <[email protected]>
2+
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or
3+
# https://www.gnu.org/licenses/gpl-3.0.txt)
4+
# SPDX-License-Identifier: GPL-3.0-or-later
5+
# SPDX-FileCopyrightText: 2021, Ansible Project
6+
"""
7+
Helpers for parsing semantic markup.
8+
"""
9+
10+
import re
11+
12+
import typing as t
13+
14+
15+
_ARRAY_STUB_RE = re.compile(r'\[([^\]]*)\]')
16+
_FQCN_TYPE_PREFIX_RE = re.compile(r'^([^.]+\.[^.]+\.[^#]+)#([a-z]+):(.*)$')
17+
_IGNORE_MARKER = 'ignore:'
18+
19+
20+
def _remove_array_stubs(text: str) -> str:
21+
return _ARRAY_STUB_RE.sub('', text)
22+
23+
24+
def parse_option(text: str, plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str],
25+
require_plugin=False) -> t.Tuple[str, str, str, str, t.Optional[str]]:
26+
"""
27+
Given the contents of O(...) / :ansopt:`...` with potential escaping removed,
28+
split it into plugin FQCN, plugin type, option link name, option name, and option value.
29+
"""
30+
value = None
31+
if '=' in text:
32+
text, value = text.split('=', 1)
33+
m = _FQCN_TYPE_PREFIX_RE.match(text)
34+
if m:
35+
plugin_fqcn = m.group(1)
36+
plugin_type = m.group(2)
37+
text = m.group(3)
38+
elif require_plugin:
39+
raise ValueError('Cannot extract plugin name and type')
40+
elif text.startswith(_IGNORE_MARKER):
41+
plugin_fqcn = ''
42+
plugin_type = ''
43+
text = text[len(_IGNORE_MARKER):]
44+
if ':' in text or '#' in text:
45+
raise ValueError(f'Invalid option name "{text}"')
46+
return plugin_fqcn, plugin_type, _remove_array_stubs(text), text, value
47+
48+
49+
def parse_return_value(text: str, plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str],
50+
require_plugin=False) -> t.Tuple[str, str, str, str, t.Optional[str]]:
51+
"""
52+
Given the contents of RV(...) / :ansretval:`...` with potential escaping removed,
53+
split it into plugin FQCN, plugin type, option link name, option name, and option value.
54+
"""
55+
value = None
56+
if '=' in text:
57+
text, value = text.split('=', 1)
58+
m = _FQCN_TYPE_PREFIX_RE.match(text)
59+
if m:
60+
plugin_fqcn = m.group(1)
61+
plugin_type = m.group(2)
62+
text = m.group(3)
63+
elif require_plugin:
64+
raise ValueError('Cannot extract plugin name and type')
65+
elif text.startswith(_IGNORE_MARKER):
66+
plugin_fqcn = ''
67+
plugin_type = ''
68+
text = text[len(_IGNORE_MARKER):]
69+
if ':' in text or '#' in text:
70+
raise ValueError(f'Invalid return value name "{text}"')
71+
return plugin_fqcn, plugin_type, _remove_array_stubs(text), text, value
72+
73+
74+
def augment_plugin_name_type(text: str, plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str]
75+
) -> str:
76+
"""
77+
Given the text contents of O(...) or RV(...) and a plugin's FQCN and type, insert
78+
the FQCN and type if they are not already present.
79+
"""
80+
value = None
81+
if '=' in text:
82+
text, value = text.split('=', 1)
83+
if ':' not in text and plugin_fqcn and plugin_type:
84+
text = f'{plugin_fqcn}#{plugin_type}:{text}'
85+
return text if value is None else f'{text}={value}'

src/sphinx_antsibull_ext/__init__.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515

1616
from .assets import setup_assets
17+
from .roles import setup_roles
1718

1819

1920
def setup(app):
@@ -25,6 +26,9 @@ def setup(app):
2526
# Add assets
2627
setup_assets(app)
2728

29+
# Add roles
30+
setup_roles(app)
31+
2832
return dict(
2933
parallel_read_safe=True,
3034
parallel_write_safe=True,

0 commit comments

Comments
 (0)