diff --git a/changelogs/fragments/409-meta.yml b/changelogs/fragments/409-meta.yml new file mode 100644 index 00000000..e0d7217e --- /dev/null +++ b/changelogs/fragments/409-meta.yml @@ -0,0 +1,2 @@ +minor_changes: + - "Add ``ansible-output-meta`` directive that allows to apply meta actions, like resetting previous code blocks for variable references, or defining templates for ``ansible-output-data`` (https://github.com/ansible-community/antsibull-docs/pull/409)." diff --git a/docs/ansible-output.md b/docs/ansible-output.md index 3cbe8356..01e99d05 100644 --- a/docs/ansible-output.md +++ b/docs/ansible-output.md @@ -183,6 +183,116 @@ The task produces the following output: } ``` +## Controlling code block contexts + +Next to the `ansible-output-data` RST directive, antsibull-docs also provides a `ansible-output-meta` RST directive. +This meta directive allows to apply actions to the context for the next `ansible-output-data` directives. + +### Reset previous code blocks + +The `reset-previous-blocks` action resets the list of previous code blocks. +It can be used as follows: +```rst +.. ansible-output-meta:: + + actions: + - name: reset-previous-blocks +``` + +This is relevant when using `previous_code_block` variables where you specify `previous_code_block_index`. +If you want several consecutive `ansible-output-data` directives to reference the same code block, +you can reset the previous blocks directly before that code block, +and then reference that code block as the one with index `0`: +```rst +(more text with other code blocks) + +.. ansible-output-meta:: + + actions: + - name: reset-previous-blocks + +.. code-block:: yaml + + # This code block now has index 0, no matter how many other code blocks + # came before the above action. + foo: bar + +Now you can have multiple ansible-output-data directives referencing the +above ``yaml`` block as the ``yaml`` block with index 0: + +.. ansible-output-data:: + + variables: + content: + previous_code_block: yaml + previous_code_block_index: 0 + playbook: |- + - hosts: localhost + tasks: + - ansible.builtin.debug: + msg: "{{ data }}" + vars: + data: + @{{ content | indent(10) }}@ + +.. code-block:: ansible-output + + ... +``` + +### Define template for `ansible-output-data` + +The `set-template` action defines a template for all following `ansible-output-data` directives. +You can use all fields that you can also use for `ansible-output-data` in the template: +```rst +.. ansible-output-meta:: + + actions: + - name: set-template + template: + # The environment variables will be merged. If a variable is provided here, + # you do not have to provide it again in the directive - only if you want to + # override its value. + env: + ANSIBLE_STDOUT_CALLBACK: community.general.tasks_only + ANSIBLE_COLLECTIONS_TASKS_ONLY_NUMBER_OF_COLUMNS: "90" + + # Will use this value if not specified in the directive. + # If no language is provided in both the template and the directive, + # 'ansible-output' will be used. + language: console + + # Will use this value if not specified in the directive. + prepend_lines: | + $ ansible-playbook playbook.yml + + # Will use this value if not specified in the directive. + skip_first_lines: 3 + + # Will use this value if not specified in the directive. + skip_last_lines: 0 + + # The variables will be merged. If a varibale is provided here, + # you can override it in the directive by specifying a variable + # of the same name. + variables: + hosts: + value: localhost + tasks: + previous_code_block: yaml+jinja + previous_code_block_index: -1 + + # Will use this value if not specified in the directive. + postprocessors: [] + + # Will use this value if explicitly set to null/~ in the directive. + playbook: |- + (some Ansible playbook) +``` + +This can be useful to avoid repeating some definitions for multiple code blocks. +If another `ansible-output-meta` action sets a new template, the previous templates will be thrown away. + ## Post-processing ansible-playbook output Out of the box, you can post-process the `ansible-playbook` output in some ways: diff --git a/src/antsibull_docs/ansible_output/load.py b/src/antsibull_docs/ansible_output/load.py index b03e757e..707f7cc1 100644 --- a/src/antsibull_docs/ansible_output/load.py +++ b/src/antsibull_docs/ansible_output/load.py @@ -8,6 +8,7 @@ from __future__ import annotations import os +import typing as t from dataclasses import dataclass from pathlib import Path @@ -20,13 +21,21 @@ mark_antsibull_code_block, ) from docutils import nodes +from docutils.parsers.rst import Directive from yaml import MarkedYAMLError from sphinx_antsibull_ext.directive_helper import YAMLDirective from sphinx_antsibull_ext.schemas.ansible_output_data import ( AnsibleOutputData, + AnsibleOutputTemplate, NonRefPostprocessor, PostprocessorNameRef, + combine, +) +from sphinx_antsibull_ext.schemas.ansible_output_meta import ( + ActionResetPreviousBlocks, + ActionSetTemplate, + AnsibleOutputMeta, ) from ..schemas.collection_config import CollectionConfig @@ -67,7 +76,8 @@ class FileData: blocks: list[Block] -_ANSIBLE_OUTPUT_DATA_IDENTIFIER = "{}[]XXXXXX" +_ANSIBLE_OUTPUT_DATA_IDENTIFIER = "{}[]XXXXXXdata" +_ANSIBLE_OUTPUT_META_IDENTIFIER = "{}[]XXXXXXmeta" class AnsibleOutputDataDirective(YAMLDirective[AnsibleOutputData]): @@ -100,8 +110,46 @@ def _run(self, content_str: str, content: AnsibleOutputData) -> list[nodes.Node] return [literal] -_DIRECTIVES = { +class AnsibleOutputMetaDirective(YAMLDirective[AnsibleOutputMeta]): + wrap_as_data = False + schema = AnsibleOutputMeta + + def _handle_error(self, message: str, from_exc: Exception) -> list[nodes.Node]: + literal = nodes.literal_block("", "") + mark_antsibull_code_block( + literal, + language=_ANSIBLE_OUTPUT_META_IDENTIFIER, + line=self.lineno, + other={ + "error": message, + "exception": from_exc, + }, + ) + return [literal] + + def _run(self, content_str: str, content: AnsibleOutputMeta) -> list[nodes.Node]: + literal = nodes.literal_block(content_str, "") + mark_antsibull_code_block( + literal, + language=_ANSIBLE_OUTPUT_META_IDENTIFIER, + line=self.lineno, + other={ + "data": content, + }, + ) + return [literal] + + +_DIRECTIVES: dict[str, type[Directive]] = { "ansible-output-data": AnsibleOutputDataDirective, + "ansible-output-meta": AnsibleOutputMetaDirective, +} + +_DirectiveName = t.Literal["ansible-output-data", "ansible-output-meta"] + +_LANGUAGE_TO_DIRECTIVE: dict[str | None, _DirectiveName] = { + _ANSIBLE_OUTPUT_DATA_IDENTIFIER: "ansible-output-data", + _ANSIBLE_OUTPUT_META_IDENTIFIER: "ansible-output-meta", } @@ -112,7 +160,9 @@ class _AnsibleOutputDataExt: col: int -def _get_ansible_output_data_error(block: CodeBlockInfo) -> tuple[int, int, str]: +def _get_ansible_output_data_error( + block: CodeBlockInfo, *, directive: str +) -> tuple[int, int, str]: message = block.attributes["antsibull-other-error"] exc = block.attributes.get("antsibull-other-exception") line = block.row_offset + 1 @@ -125,50 +175,160 @@ def _get_ansible_output_data_error(block: CodeBlockInfo) -> tuple[int, int, str] col += exc.problem_mark.column if isinstance(exc, pydantic.ValidationError): message = ( - "Error while validating ansible-output-data directive's contents:\n" + f"Error while validating {directive} directive's contents:\n" + "\n".join(get_formatted_error_messages(exc)) ) return line, col, message -def _compose_block( - codeblock: CodeBlockInfo, - *, - path: Path, - data: _AnsibleOutputDataExt, - previous_blocks: list[CodeBlockInfo], - errors: list[Error], - environment: Environment, -) -> Block: - env = environment.env.copy() - env.update(data.data.env) - postprocessors = [] - for postprocessor in data.data.postprocessors: - if isinstance(postprocessor, PostprocessorNameRef): - ref = postprocessor.name - try: - postprocessor = environment.global_postprocessors[ref] - except KeyError: - errors.append( - Error( - path, - data.line, - data.col, - f"No global postprocessor of name {ref!r} defined", +class _BlockCollector: + def __init__( + self, *, path: Path, errors: list[Error], environment: Environment + ) -> None: + self.path = path + self.errors = errors + self.environment = environment + self.blocks: list[Block] = [] + self.data: _AnsibleOutputDataExt | None = None + self.previous_blocks: list[CodeBlockInfo] = [] + self.template = AnsibleOutputTemplate() + + def _process_reset_previous_blocks( + self, action: ActionResetPreviousBlocks # pylint: disable=unused-argument + ) -> None: + self.previous_blocks.clear() + + def _process_set_template(self, action: ActionSetTemplate) -> None: + self.template = action.template + + def process_meta( + self, + meta: AnsibleOutputMeta, + *, + line: int, # pylint: disable=unused-argument + col: int, # pylint: disable=unused-argument + ) -> None: + for action in meta.actions: + if isinstance(action, ActionResetPreviousBlocks): + self._process_reset_previous_blocks(action) + elif isinstance(action, ActionSetTemplate): + self._process_set_template(action) + else: + raise AssertionError("Unknown action") # pragma: no cover + + def process_special_block( + self, block: CodeBlockInfo, *, directive: _DirectiveName + ) -> None: + if self.data is not None: + self.errors.append( + Error( + self.path, + self.data.line, + self.data.col, + "ansible-output-data directive not used", + ) + ) + self.data = None + if "antsibull-other-error" in block.attributes: + line, col, message = _get_ansible_output_data_error( + block, directive=directive + ) + self.errors.append(Error(self.path, line, col, message)) + if "antsibull-other-data" in block.attributes: + if directive == "ansible-output-data": + try: + data = combine( + data=block.attributes["antsibull-other-data"], + template=self.template, + ) + self.data = _AnsibleOutputDataExt( + data=data, + line=block.row_offset + 1, + col=block.col_offset + 1, ) + except ValueError as exc: + self.errors.append( + Error( + self.path, + block.row_offset + 1, + block.col_offset + 1, + str(exc), + ) + ) + if directive == "ansible-output-meta": + self.process_meta( + block.attributes["antsibull-other-data"], + line=block.row_offset + 1, + col=block.col_offset + 1, ) - continue - postprocessors.append(postprocessor) - return Block( - path=path, - codeblock=codeblock, - data=data.data, - data_line=data.line, - data_col=data.col, - previous_blocks=previous_blocks, - merged_env=env, - merged_postprocessors=postprocessors, - ) + + def _add_block( + self, + codeblock: CodeBlockInfo, + *, + data: _AnsibleOutputDataExt, + ) -> None: + env = self.environment.env.copy() + env.update(data.data.env) + postprocessors = [] + error = False + if data.data.postprocessors: + for postprocessor in data.data.postprocessors: + if isinstance(postprocessor, PostprocessorNameRef): + ref = postprocessor.name + try: + postprocessor = self.environment.global_postprocessors[ref] + except KeyError: + self.errors.append( + Error( + self.path, + data.line, + data.col, + f"No global postprocessor of name {ref!r} defined", + ) + ) + error = True + continue + postprocessors.append(postprocessor) + if error: + return + self.blocks.append( + Block( + path=self.path, + codeblock=codeblock, + data=data.data, + data_line=data.line, + data_col=data.col, + previous_blocks=self.previous_blocks[:-1], + merged_env=env, + merged_postprocessors=postprocessors, + ) + ) + + def found_block(self, block: CodeBlockInfo) -> None: + directive = _LANGUAGE_TO_DIRECTIVE.get(block.language) + if directive is not None: + self.process_special_block(block, directive=directive) + return + self.previous_blocks.append(block) + if self.data is None: + return + if block.language != self.data.data.language: + return + self._add_block(block, data=self.data) + self.data = None + + def finish(self) -> list[Block]: + if self.data is not None: + self.errors.append( + Error( + self.path, + self.data.line, + self.data.col, + "ansible-output-data directive not used", + ) + ) + return self.blocks def _find_blocks( @@ -179,58 +339,12 @@ def _find_blocks( errors: list[Error], environment: Environment, ) -> list[Block]: - blocks: list[Block] = [] - data: _AnsibleOutputDataExt | None = None - previous_blocks: list[CodeBlockInfo] = [] + collector = _BlockCollector(path=path, errors=errors, environment=environment) for block in find_code_blocks( content, path=path, root_prefix=root, extra_directives=_DIRECTIVES ): - if block.language == _ANSIBLE_OUTPUT_DATA_IDENTIFIER: - if data is not None: - errors.append( - Error( - path, - data.line, - data.col, - "ansible-output-data directive not used", - ) - ) - if "antsibull-other-data" in block.attributes: - data = _AnsibleOutputDataExt( - data=block.attributes["antsibull-other-data"], - line=block.row_offset + 1, - col=block.col_offset + 1, - ) - if "antsibull-other-error" in block.attributes: - line, col, message = _get_ansible_output_data_error(block) - errors.append(Error(path, line, col, message)) - continue - previous_blocks.append(block) - if data is None: - continue - if block.language != data.data.language: - continue - blocks.append( - _compose_block( - block, - path=path, - data=data, - previous_blocks=previous_blocks[:-1], - errors=errors, - environment=environment, - ) - ) - data = None - if data is not None: - errors.append( - Error( - path, - data.line, - data.col, - "ansible-output-data directive not used", - ) - ) - return blocks + collector.found_block(block) + return collector.finish() def load_blocks_from_content( diff --git a/src/antsibull_docs/ansible_output/process.py b/src/antsibull_docs/ansible_output/process.py index 88c0e3c2..e632f050 100644 --- a/src/antsibull_docs/ansible_output/process.py +++ b/src/antsibull_docs/ansible_output/process.py @@ -61,6 +61,9 @@ def _get_variable_value( def _compose_playbook( data: AnsibleOutputData, *, previous_blocks: list[CodeBlockInfo] ) -> str: + if data.playbook is None: + raise AssertionError("playbook cannot be None at this point") + if all(s not in data.playbook for s in ("@{%", "@{{", "@{#")): return data.playbook @@ -70,18 +73,21 @@ def _compose_playbook( key=key, value=value, previous_blocks=previous_blocks ) - env = jinja2.Environment( - block_start_string="@{%", - block_end_string="%}@", - variable_start_string="@{{", - variable_end_string="}}@", - comment_start_string="@{#", - comment_end_string="#}@", - trim_blocks=True, - optimized=False, # we use every template once - ) - template = env.from_string(data.playbook) - return template.render(**variables) + try: + env = jinja2.Environment( + block_start_string="@{%", + block_end_string="%}@", + variable_start_string="@{{", + variable_end_string="}}@", + comment_start_string="@{#", + comment_end_string="#}@", + trim_blocks=True, + optimized=False, # we use every template once + ) + template = env.from_string(data.playbook) + return template.render(**variables) + except Exception as exc: + raise ValueError(f"Error while templating playbook:\n{exc}") from exc def _strip_empty_lines(lines: list[str]) -> list[str]: @@ -218,8 +224,8 @@ async def _compute_code_block_content( flog.notice("Post-process result") lines = _massage_stdout( stdout, - skip_first_lines=block.data.skip_first_lines, - skip_last_lines=block.data.skip_last_lines, + skip_first_lines=block.data.skip_first_lines or 0, + skip_last_lines=block.data.skip_last_lines or 0, prepend_lines=block.data.prepend_lines, ) for postprocessor in block.merged_postprocessors: diff --git a/src/sphinx_antsibull_ext/directives.py b/src/sphinx_antsibull_ext/directives.py index bdd4592c..de037b57 100644 --- a/src/sphinx_antsibull_ext/directives.py +++ b/src/sphinx_antsibull_ext/directives.py @@ -17,6 +17,7 @@ from .nodes import link_button from .schemas.ansible_links import AnsibleLinks from .schemas.ansible_output_data import AnsibleOutputData +from .schemas.ansible_output_meta import AnsibleOutputMeta class _OptionTypeLine(Directive): @@ -86,10 +87,25 @@ def _run(self, content_str: str, content: AnsibleOutputData) -> list[nodes.Node] return [] +class _AnsibleOutputMetaDirective(YAMLDirective[AnsibleOutputMeta]): + wrap_as_data = True + schema = AnsibleOutputMeta + + def _handle_error(self, message: str, from_exc: Exception) -> list[nodes.Node]: + # Do not report errors when simply building docs. + # Errors should be reported when running 'antsibull-docs lint-collection-docs'. + return [] + + def _run(self, content_str: str, content: AnsibleOutputMeta) -> list[nodes.Node]: + # This directive should produce no output. It is used in the ansible-output subcommand. + return [] + + DIRECTIVES = { "ansible-option-type-line": _OptionTypeLine, "ansible-links": _Links, "ansible-output-data": _AnsibleOutputDataDirective, + "ansible-output-meta": _AnsibleOutputMetaDirective, } diff --git a/src/sphinx_antsibull_ext/schemas/ansible_output_data.py b/src/sphinx_antsibull_ext/schemas/ansible_output_data.py index dee759db..149c6408 100644 --- a/src/sphinx_antsibull_ext/schemas/ansible_output_data.py +++ b/src/sphinx_antsibull_ext/schemas/ansible_output_data.py @@ -50,15 +50,15 @@ class PostprocessorNameRef(p.BaseModel): NonRefPostprocessor = t.Union[PostprocessorCLI] -class AnsibleOutputData(p.BaseModel): - playbook: str +class AnsibleOutputTemplate(p.BaseModel): + playbook: t.Optional[str] = None env: dict[str, str] = {} - prepend_lines: str = "" - language: str = "ansible-output" + prepend_lines: t.Optional[str] = None + language: t.Optional[str] = None variables: dict[str, VariableSource] = {} - skip_first_lines: int = 0 - skip_last_lines: int = 0 - postprocessors: list[Postprocessor] = [] + skip_first_lines: t.Optional[int] = None + skip_last_lines: t.Optional[int] = None + postprocessors: t.Optional[list[Postprocessor]] = None @p.field_validator("env", mode="before") @classmethod @@ -73,3 +73,39 @@ def convert_dict_values(cls, obj): if isinstance(v, int): obj[k] = str(v) return obj + + +class AnsibleOutputData(AnsibleOutputTemplate): + playbook: t.Optional[str] # no default + + +_T = t.TypeVar("_T") + + +def _coalesce(*values: _T | None) -> _T | None: + for value in values: + if value is not None: + return value + return None + + +def combine( + *, template: AnsibleOutputTemplate, data: AnsibleOutputData +) -> AnsibleOutputData: + playbook = _coalesce(data.playbook, template.playbook) + if playbook is None: + raise ValueError("Cannot use template's playbook since that is not set") + env = template.env.copy() + env.update(data.env) + variables = template.variables.copy() + variables.update(data.variables) + return AnsibleOutputData( + playbook=playbook, + env=env, + prepend_lines=data.prepend_lines or template.prepend_lines, + language=data.language or template.language or "ansible-output", + variables=variables, + skip_first_lines=_coalesce(data.skip_first_lines, template.skip_first_lines), + skip_last_lines=_coalesce(data.skip_last_lines, template.skip_last_lines), + postprocessors=_coalesce(data.postprocessors, template.postprocessors), + ) diff --git a/src/sphinx_antsibull_ext/schemas/ansible_output_meta.py b/src/sphinx_antsibull_ext/schemas/ansible_output_meta.py new file mode 100644 index 00000000..2810fc40 --- /dev/null +++ b/src/sphinx_antsibull_ext/schemas/ansible_output_meta.py @@ -0,0 +1,36 @@ +# Author: Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2025, Ansible Project +"""Schema for ansible-output-data directive.""" + +from __future__ import annotations + +import typing as t + +import pydantic as p + +from .ansible_output_data import AnsibleOutputTemplate + + +class ActionResetPreviousBlocks(p.BaseModel): + model_config = p.ConfigDict(frozen=True, extra="forbid", validate_default=True) + + name: t.Literal["reset-previous-blocks"] + + +class ActionSetTemplate(p.BaseModel): + model_config = p.ConfigDict(frozen=True, extra="forbid", validate_default=True) + + name: t.Literal["set-template"] + template: AnsibleOutputTemplate + + +AnsibleOutputAction = t.Union[ActionResetPreviousBlocks, ActionSetTemplate] + + +class AnsibleOutputMeta(p.BaseModel): + model_config = p.ConfigDict(frozen=True, extra="forbid", validate_default=True) + + actions: list[AnsibleOutputAction] diff --git a/tests/functional/test_ansible_output.py b/tests/functional/test_ansible_output.py index 628646a9..5e86c9eb 100644 --- a/tests/functional/test_ansible_output.py +++ b/tests/functional/test_ansible_output.py @@ -816,11 +816,63 @@ async def execute( ansible-playbook-failure-variables-too-many.rst:4:5: Error while validating ansible-output-data directive's contents: variables -> foo -> VariableSourceCodeBlock -> value: Extra inputs are not permitted variables -> foo -> VariableSourceValue -> previous_code_block: Extra inputs are not permitted +""", + ), + ( + "ansible-playbook-failure-variables-does-exist.rst", + """ +.. code-block:: yaml + + foo: foo! + +.. code-block:: yaml + + foo: bar + +.. ansible-output-data:: + + variables: + foo: + previous_code_block: yaml + previous_code_block_index: 1 + playbook: |- + foo @{{ foo }}@ + +.. code-block:: ansible-output + + bar +""", + AnsiblePlaybookSuccess( + ["ansible-playbook", "playbook.yml"], + {}, + [ + FileContent( + "playbook.yml", + "foo foo: bar\n", + ), + ], + "meh", + ), + 3, + r""" +Found 1 error: +ansible-playbook-failure-variables-does-exist.rst:21:5: Output would differ: + - bar + + meh """, ), ( "ansible-playbook-failure-variables-does-not-exist.rst", """ +.. code-block:: yaml + + foo: foo! + +.. ansible-output-meta:: + + actions: + - name: reset-previous-blocks + .. code-block:: yaml foo: bar @@ -832,7 +884,7 @@ async def execute( previous_code_block: yaml previous_code_block_index: 1 playbook: |- - foo @{{ foo }} + foo @{{ foo }}@ .. code-block:: ansible-output @@ -842,8 +894,76 @@ async def execute( 3, r""" Found 1 error: -ansible-playbook-failure-variables-does-not-exist.rst:8:5: Error while computing code block's expected contents: +ansible-playbook-failure-variables-does-not-exist.rst:17:5: Error while computing code block's expected contents: Found 1 previous code block(s) of language 'yaml' for variable 'foo', which does not allow index 1 +""", + ), + ( + "ansible-playbook-failure-variables-reset.rst", + """ +.. code-block:: yaml + + foo: foo! + +.. ansible-output-meta:: + + actions: + - name: reset-previous-blocks + +.. code-block:: yaml + + foo: bar + +.. ansible-output-data:: + + variables: + foo: + previous_code_block: yaml + playbook: |- + foo @{{ foo }}@ + +.. code-block:: ansible-output + + bar +""", + AnsiblePlaybookSuccess( + ["ansible-playbook", "playbook.yml"], + {}, + [ + FileContent( + "playbook.yml", + "foo foo: bar\n", + ), + ], + "meh", + ), + 3, + r""" +Found 1 error: +ansible-playbook-failure-variables-reset.rst:25:5: Output would differ: + - bar + + meh +""", + ), + ( + "ansible-playbook-bad-template.rst", + """ +.. ansible-output-data:: + + playbook: |- + foo @{{ + +.. code-block:: ansible-output + + bar +""", + None, + 3, + r""" +Found 1 error: +ansible-playbook-bad-template.rst:4:5: Error while computing code block's expected contents: + Error while templating playbook: + unexpected 'end of template' """, ), ]