From 58db8d4e2d98690bbf2b052024bb51a8fd4b0f43 Mon Sep 17 00:00:00 2001 From: Ryunosuke O'Neil Date: Tue, 4 Nov 2025 15:09:35 +0100 Subject: [PATCH 1/3] docs: settings doc fix: annoying wrapper script docs: fix task doc fix: add ci skip settings-doc-check --- .github/workflows/generate_pixi_tasks_doc.py | 2 + .pre-commit-config.yaml | 11 +- diracx-core/src/diracx/core/settings.py | 105 ++++++++++++- docs/dev/reference/env-variables.md | 153 ++++++++++++++++++- docs/dev/reference/pixi-tasks.md | 5 + pixi.toml | 22 ++- scripts/check_settings_doc.sh | 42 +++++ 7 files changed, 328 insertions(+), 12 deletions(-) create mode 100755 scripts/check_settings_doc.sh diff --git a/.github/workflows/generate_pixi_tasks_doc.py b/.github/workflows/generate_pixi_tasks_doc.py index d9c1ef321..85c6f4cd7 100644 --- a/.github/workflows/generate_pixi_tasks_doc.py +++ b/.github/workflows/generate_pixi_tasks_doc.py @@ -18,6 +18,8 @@ def get_task_group(task_name): return "Client Generation" if task_name == "shellcheck": return "Shellcheck" + if task_name.startswith("generate-settings-"): + return "Settings" return "Default" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9aa54eb6e..9d2e49583 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ default_language_version: python: python3 ci: - skip: [generate-pixi-docs] + skip: [generate-pixi-docs, settings-doc-check] default_stages: [pre-commit] @@ -84,3 +84,12 @@ repos: language: system pass_filenames: false files: ^pixi\.toml$|^pixi\.lock$ # only run if pixi files change + + - repo: local + hooks: + - id: settings-doc-check + name: Generate settings documentation + entry: scripts/check_settings_doc.sh + language: system + pass_filenames: false + files: ^diracx-core/src/diracx/core/settings\.py$ diff --git a/diracx-core/src/diracx/core/settings.py b/diracx-core/src/diracx/core/settings.py index ef11459f8..1d90ebdb6 100644 --- a/diracx-core/src/diracx/core/settings.py +++ b/diracx-core/src/diracx/core/settings.py @@ -132,11 +132,16 @@ async def lifetime_function(self) -> AsyncIterator[None]: class DevelopmentSettings(ServiceSettingsBase): """Settings for the Development Configuration that can influence run time.""" - model_config = SettingsConfigDict(env_prefix="DIRACX_DEV_") + model_config = SettingsConfigDict( + env_prefix="DIRACX_DEV_", use_attribute_docstrings=True + ) - # When then to true (only for demo/CI), crash if an access policy isn't - # called crash_on_missed_access_policy: bool = False + """When set to true (only for demo/CI), crash if an access policy isn't called. + + This is useful for development and testing to ensure all endpoints have proper + access control policies defined. + """ @classmethod def create(cls) -> Self: @@ -146,39 +151,123 @@ def create(cls) -> Self: class AuthSettings(ServiceSettingsBase): """Settings for the authentication service.""" - model_config = SettingsConfigDict(env_prefix="DIRACX_SERVICE_AUTH_") + model_config = SettingsConfigDict( + env_prefix="DIRACX_SERVICE_AUTH_", use_attribute_docstrings=True + ) dirac_client_id: str = "myDIRACClientID" - # TODO: This should be taken dynamically - # ["http://pclhcb211:8000/docs/oauth2-redirect"] + """OAuth2 client identifier for DIRAC services. + + This should match the client ID registered with the identity provider. + """ + allowed_redirects: list[str] = [] + """List of allowed redirect URLs for OAuth2 authorization flow. + + These URLs must be pre-registered and should match the redirect URIs + configured in the OAuth2 client registration. + Example: ["http://localhost:8000/docs/oauth2-redirect"] + """ + device_flow_expiration_seconds: int = 600 + """Expiration time in seconds for device flow authorization requests. + + After this time, the device code becomes invalid and users must restart + the device flow process. Default: 10 minutes. + """ + authorization_flow_expiration_seconds: int = 300 + """Expiration time in seconds for authorization code flow. + + The time window during which the authorization code remains valid + before it must be exchanged for tokens. Default: 5 minutes. + """ - # State key is used to encrypt/decrypt the state dict passed to the IAM state_key: FernetKey + """Encryption key used to encrypt/decrypt the state parameter passed to the IAM. + + This key ensures the integrity and confidentiality of state information + during OAuth2 flows. Must be a valid Fernet key. + """ token_issuer: str + """The issuer identifier for JWT tokens. + + This should be a URI that uniquely identifies the token issuer and + matches the 'iss' claim in issued JWT tokens. + """ + token_keystore: TokenSigningKeyStore + """Keystore containing the cryptographic keys used for signing JWT tokens. + + This includes both public and private keys for token signature + generation and verification. + """ + token_allowed_algorithms: list[str] = ["RS256", "EdDSA"] # noqa: S105 + """List of allowed cryptographic algorithms for JWT token signing. + + Supported algorithms include RS256 (RSA with SHA-256) and EdDSA + (Edwards-curve Digital Signature Algorithm). Default: ["RS256", "EdDSA"] + """ + access_token_expire_minutes: int = 20 + """Expiration time in minutes for access tokens. + + After this duration, access tokens become invalid and must be refreshed + or re-obtained. Default: 20 minutes. + """ + refresh_token_expire_minutes: int = 60 + """Expiration time in minutes for refresh tokens. + + The maximum lifetime of refresh tokens before they must be re-issued + through a new authentication flow. Default: 60 minutes. + """ available_properties: set[SecurityProperty] = Field( default_factory=SecurityProperty.available_properties ) + """Set of security properties available in this DIRAC installation. + + These properties define various authorization capabilities and are used + for access control decisions. Defaults to all available security properties. + """ class SandboxStoreSettings(ServiceSettingsBase): """Settings for the sandbox store.""" - model_config = SettingsConfigDict(env_prefix="DIRACX_SANDBOX_STORE_") + model_config = SettingsConfigDict( + env_prefix="DIRACX_SANDBOX_STORE_", use_attribute_docstrings=True + ) bucket_name: str + """Name of the S3 bucket used for storing job sandboxes. + + This bucket will contain input and output sandbox files for DIRAC jobs. + The bucket must exist or auto_create_bucket must be enabled. + """ + s3_client_kwargs: dict[str, str] + """Configuration parameters passed to the S3 client.""" + auto_create_bucket: bool = False + """Whether to automatically create the S3 bucket if it doesn't exist.""" + url_validity_seconds: int = 5 * 60 + """Validity duration in seconds for pre-signed S3 URLs. + + This determines how long generated download/upload URLs remain valid + before expiring. Default: 300 seconds (5 minutes). + """ + se_name: str = "SandboxSE" + """Logical name of the Storage Element for the sandbox store. + + This name is used within DIRAC to refer to this sandbox storage + endpoint in job descriptions and file catalogs. + """ _client: S3Client = PrivateAttr() @contextlib.asynccontextmanager diff --git a/docs/dev/reference/env-variables.md b/docs/dev/reference/env-variables.md index efbcb3330..41cc34b0b 100644 --- a/docs/dev/reference/env-variables.md +++ b/docs/dev/reference/env-variables.md @@ -1,5 +1,154 @@ # List of development environment variables -## Development: +*This page is auto-generated. Do not edit directly.* -- `DIRACX_DEV_CRASH_ON_MISSED_ACCESS_POLICY`: If set to true, the server will crash if an access policy is not called. + + +## `DIRACX_DEV_CRASH_ON_MISSED_ACCESS_POLICY` + +*Optional*, default value: `False` + +When set to true (only for demo/CI), crash if an access policy isn't called. + +This is useful for development and testing to ensure all endpoints have proper +access control policies defined. + +## `DIRACX_SERVICE_AUTH_DIRAC_CLIENT_ID` + +*Optional*, default value: `myDIRACClientID` + +OAuth2 client identifier for DIRAC services. + +This should match the client ID registered with the identity provider. + +## `DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS` + +*Optional*, default value: `[]` + +List of allowed redirect URLs for OAuth2 authorization flow. + +These URLs must be pre-registered and should match the redirect URIs +configured in the OAuth2 client registration. +Example: ["http://localhost:8000/docs/oauth2-redirect"] + +## `DIRACX_SERVICE_AUTH_DEVICE_FLOW_EXPIRATION_SECONDS` + +*Optional*, default value: `600` + +Expiration time in seconds for device flow authorization requests. + +After this time, the device code becomes invalid and users must restart +the device flow process. Default: 10 minutes. + +## `DIRACX_SERVICE_AUTH_AUTHORIZATION_FLOW_EXPIRATION_SECONDS` + +*Optional*, default value: `300` + +Expiration time in seconds for authorization code flow. + +The time window during which the authorization code remains valid +before it must be exchanged for tokens. Default: 5 minutes. + +## `DIRACX_SERVICE_AUTH_STATE_KEY` + +**Required** + +Encryption key used to encrypt/decrypt the state parameter passed to the IAM. + +This key ensures the integrity and confidentiality of state information +during OAuth2 flows. Must be a valid Fernet key. + +## `DIRACX_SERVICE_AUTH_TOKEN_ISSUER` + +**Required** + +The issuer identifier for JWT tokens. + +This should be a URI that uniquely identifies the token issuer and +matches the 'iss' claim in issued JWT tokens. + +## `DIRACX_SERVICE_AUTH_TOKEN_KEYSTORE` + +**Required** + +Keystore containing the cryptographic keys used for signing JWT tokens. + +This includes both public and private keys for token signature +generation and verification. + +## `DIRACX_SERVICE_AUTH_TOKEN_ALLOWED_ALGORITHMS` + +*Optional*, default value: `['RS256', 'EdDSA']` + +List of allowed cryptographic algorithms for JWT token signing. + +Supported algorithms include RS256 (RSA with SHA-256) and EdDSA +(Edwards-curve Digital Signature Algorithm). Default: ["RS256", "EdDSA"] + +## `DIRACX_SERVICE_AUTH_ACCESS_TOKEN_EXPIRE_MINUTES` + +*Optional*, default value: `20` + +Expiration time in minutes for access tokens. + +After this duration, access tokens become invalid and must be refreshed +or re-obtained. Default: 20 minutes. + +## `DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES` + +*Optional*, default value: `60` + +Expiration time in minutes for refresh tokens. + +The maximum lifetime of refresh tokens before they must be re-issued +through a new authentication flow. Default: 60 minutes. + +## `DIRACX_SERVICE_AUTH_AVAILABLE_PROPERTIES` + +*Optional* + +Set of security properties available in this DIRAC installation. + +These properties define various authorization capabilities and are used +for access control decisions. Defaults to all available security properties. + +## `DIRACX_SANDBOX_STORE_BUCKET_NAME` + +**Required** + +Name of the S3 bucket used for storing job sandboxes. + +This bucket will contain input and output sandbox files for DIRAC jobs. +The bucket must exist or auto_create_bucket must be enabled. + +## `DIRACX_SANDBOX_STORE_S3_CLIENT_KWARGS` + +**Required** + +Configuration parameters passed to the S3 client. + +## `DIRACX_SANDBOX_STORE_AUTO_CREATE_BUCKET` + +*Optional*, default value: `False` + +Whether to automatically create the S3 bucket if it doesn't exist. + +## `DIRACX_SANDBOX_STORE_URL_VALIDITY_SECONDS` + +*Optional*, default value: `300` + +Validity duration in seconds for pre-signed S3 URLs. + +This determines how long generated download/upload URLs remain valid +before expiring. Default: 300 seconds (5 minutes). + +## `DIRACX_SANDBOX_STORE_SE_NAME` + +*Optional*, default value: `SandboxSE` + +Logical name of the Storage Element for the sandbox store. + +This name is used within DIRAC to refer to this sandbox storage +endpoint in job descriptions and file catalogs. + + diff --git a/docs/dev/reference/pixi-tasks.md b/docs/dev/reference/pixi-tasks.md index f2737acde..b7866fde8 100644 --- a/docs/dev/reference/pixi-tasks.md +++ b/docs/dev/reference/pixi-tasks.md @@ -46,6 +46,11 @@ This page documents the available pixi tasks. - `pre-commit`: pre-commit +## Settings Tasks + +- `generate-settings-doc`: Generate settings documentation in Markdown format +- `generate-settings-dotenv`: Generate a dotenv file from settings + ## Shellcheck Tasks - `shellcheck`: Run shellcheck on all shell scripts diff --git a/pixi.toml b/pixi.toml index e9ae48d61..ab64784bb 100644 --- a/pixi.toml +++ b/pixi.toml @@ -30,6 +30,10 @@ diracx-logic = { path = "diracx-logic", editable = true, extras = ["testing"] } [feature.diracx-routers.pypi-dependencies] diracx-routers = { path = "diracx-routers", editable = true, extras = ["testing"] } +# Settings documentation feature +[feature.settings-doc.pypi-dependencies] +settings-doc = "*" + # DiracX features for providing tasks. This is needed to make it so that running # "pixi run pytest-diracx-core -vvv --pdb" passes the arguments as expected. # See: https://github.com/prefix-dev/pixi/issues/1519#issuecomment-2651078457 @@ -57,6 +61,22 @@ description = "Run the tests for diracx-logic" cmd = "cd diracx-routers/ && pytest" description = "Run the tests for diracx-routers" +# Settings documentation task +[feature.settings-doc.tasks.generate-settings-doc] +cmd = """settings-doc generate \ + --module diracx.core.settings \ + --output-format markdown \ + --update docs/dev/reference/env-variables.md \ + --between "" "" \ + --heading-offset 1""" +description = "Generate settings documentation in Markdown format" + +[feature.settings-doc.tasks.generate-settings-dotenv] +cmd = """settings-doc generate \ + --module diracx.core.settings \ + --output-format dotenv > .env""" +description = "Generate a dotenv file from settings" + # Gubbins features for providing dependencies [feature.gubbins.pypi-dependencies] gubbins = { path = "extensions/gubbins", editable = true, extras = ["testing"] } @@ -142,7 +162,7 @@ description = "Run shellcheck on all shell scripts" [environments] # DiracX environments -default = {features = ["task-diracx", "diracx", "diracx-core", "diracx-api", "diracx-cli", "diracx-client", "diracx-db", "diracx-logic", "diracx-routers"], solve-group = "diracx"} +default = {features = ["task-diracx", "diracx", "diracx-core", "diracx-api", "diracx-cli", "diracx-client", "diracx-db", "diracx-logic", "diracx-routers", "settings-doc"], solve-group = "diracx"} diracx-core = {features = ["task-diracx-core", "diracx-core"], solve-group = "diracx"} diracx-api = {features = ["task-diracx-api", "diracx-api", "diracx-client", "diracx-core"], solve-group = "diracx"} diracx-cli = {features = ["task-diracx-cli", "diracx-cli", "diracx-api", "diracx-client", "diracx-core"], solve-group = "diracx"} diff --git a/scripts/check_settings_doc.sh b/scripts/check_settings_doc.sh new file mode 100755 index 000000000..dc0b18b2c --- /dev/null +++ b/scripts/check_settings_doc.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Run the settings-doc generator via pixi and fail if the committed file would change. +# This allows a pre-commit hook to check that generated documentation is up-to-date +# without letting the hook itself modify the repository and cause the hook to fail. + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +cd "$repo_root" + +TARGET="docs/dev/reference/env-variables.md" + +# Save the current state of the target file +cp "$TARGET" "${TARGET}.backup" + +echo "Running settings-doc generator (pixi run generate-settings-doc)..." +pixi run generate-settings-doc + +# Normalize both files by removing empty lines, then compare +# This handles cases where settings-doc adds/removes blank lines inconsistently +normalize_file() { + grep -v '^[[:space:]]*$' "$1" || true +} + +if diff -q <(normalize_file "$TARGET") <(normalize_file "${TARGET}.backup") > /dev/null 2>&1; then + # Files are equivalent ignoring blank lines, restore original and pass + cp "${TARGET}.backup" "$TARGET" + rm "${TARGET}.backup" + echo "Generated documentation is up-to-date." + exit 0 +fi + +# Files have meaningful differences +echo +echo "ERROR: Generated settings documentation is out of date." +echo "Run 'pixi run generate-settings-doc' and commit the updated file." +echo +echo "Diff (generated vs current):" +diff "$TARGET" "${TARGET}.backup" || true +# Restore the original file so pre-commit doesn't see modifications +mv "${TARGET}.backup" "$TARGET" +exit 1 From 0170a87ece5bdada429061f0122a1f8d2a760d17 Mon Sep 17 00:00:00 2001 From: Ryunosuke O'Neil Date: Fri, 7 Nov 2025 14:44:39 +0100 Subject: [PATCH 2/3] docs: env variable / settings documentation generator fix: cleanups --- .env.example | 121 +++++++ .gitignore | 1 + .pre-commit-config.yaml | 4 +- docs/admin/reference/env-variables.md | 218 ++++++++++-- docs/admin/reference/env-variables.md.j2 | 11 + docs/dev/reference/env-variables.md | 150 +------- docs/dev/reference/env-variables.md.j2 | 7 + docs/dev/reference/pixi-tasks.md | 3 +- docs/templates/_render_class.jinja | 40 +++ pixi.toml | 15 +- scripts/check_settings_doc.sh | 42 --- scripts/generate_settings_docs.py | 422 +++++++++++++++++++++++ 12 files changed, 794 insertions(+), 240 deletions(-) create mode 100644 .env.example create mode 100644 docs/admin/reference/env-variables.md.j2 create mode 100644 docs/dev/reference/env-variables.md.j2 create mode 100644 docs/templates/_render_class.jinja delete mode 100755 scripts/check_settings_doc.sh create mode 100644 scripts/generate_settings_docs.py diff --git a/.env.example b/.env.example new file mode 100644 index 000000000..632a502c3 --- /dev/null +++ b/.env.example @@ -0,0 +1,121 @@ +# Auto-generated .env file with all DiracX settings +# This file contains all available environment variables. +# Uncomment and set the values you need. + +# AuthSettings (from diracx.core.settings) +# OAuth2 client identifier for DIRAC services. +# +# This should match the client ID registered with the identity provider. +# DIRACX_SERVICE_AUTH_DIRAC_CLIENT_ID=myDIRACClientID + +# List of allowed redirect URLs for OAuth2 authorization flow. +# +# These URLs must be pre-registered and should match the redirect URIs +# configured in the OAuth2 client registration. +# Example: ["http://localhost:8000/docs/oauth2-redirect"] +# DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS=[] + +# Expiration time in seconds for device flow authorization requests. +# +# After this time, the device code becomes invalid and users must restart +# the device flow process. Default: 10 minutes. +# DIRACX_SERVICE_AUTH_DEVICE_FLOW_EXPIRATION_SECONDS=600 + +# Expiration time in seconds for authorization code flow. +# +# The time window during which the authorization code remains valid +# before it must be exchanged for tokens. Default: 5 minutes. +# DIRACX_SERVICE_AUTH_AUTHORIZATION_FLOW_EXPIRATION_SECONDS=300 + +# Encryption key used to encrypt/decrypt the state parameter passed to the IAM. +# +# This key ensures the integrity and confidentiality of state information +# during OAuth2 flows. Must be a valid Fernet key. +DIRACX_SERVICE_AUTH_STATE_KEY= + +# The issuer identifier for JWT tokens. +# +# This should be a URI that uniquely identifies the token issuer and +# matches the 'iss' claim in issued JWT tokens. +DIRACX_SERVICE_AUTH_TOKEN_ISSUER= + +# Keystore containing the cryptographic keys used for signing JWT tokens. +# +# This includes both public and private keys for token signature +# generation and verification. +DIRACX_SERVICE_AUTH_TOKEN_KEYSTORE= + +# List of allowed cryptographic algorithms for JWT token signing. +# +# Supported algorithms include RS256 (RSA with SHA-256) and EdDSA +# (Edwards-curve Digital Signature Algorithm). Default: ["RS256", "EdDSA"] +# DIRACX_SERVICE_AUTH_TOKEN_ALLOWED_ALGORITHMS=['RS256', 'EdDSA'] + +# Expiration time in minutes for access tokens. +# +# After this duration, access tokens become invalid and must be refreshed +# or re-obtained. Default: 20 minutes. +# DIRACX_SERVICE_AUTH_ACCESS_TOKEN_EXPIRE_MINUTES=20 + +# Expiration time in minutes for refresh tokens. +# +# The maximum lifetime of refresh tokens before they must be re-issued +# through a new authentication flow. Default: 60 minutes. +# DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES=60 + +# Set of security properties available in this DIRAC installation. +# +# These properties define various authorization capabilities and are used +# for access control decisions. Defaults to all available security properties. +# DIRACX_SERVICE_AUTH_AVAILABLE_PROPERTIES= + + + +# DevelopmentSettings (from diracx.core.settings) +# When set to true (only for demo/CI), crash if an access policy isn't called. +# +# This is useful for development and testing to ensure all endpoints have proper +# access control policies defined. +# DIRACX_DEV_CRASH_ON_MISSED_ACCESS_POLICY=False + + + +# OTELSettings (from diracx.routers.otel) +# DIRACX_OTEL_ENABLED=False + +# DIRACX_OTEL_APPLICATION_NAME=diracx + +# DIRACX_OTEL_GRPC_ENDPOINT= + +# DIRACX_OTEL_GRPC_INSECURE=True + +# DIRACX_OTEL_HEADERS= + + + +# SandboxStoreSettings (from diracx.core.settings) +# Name of the S3 bucket used for storing job sandboxes. +# +# This bucket will contain input and output sandbox files for DIRAC jobs. +# The bucket must exist or auto_create_bucket must be enabled. +DIRACX_SANDBOX_STORE_BUCKET_NAME= + +# Configuration parameters passed to the S3 client. +DIRACX_SANDBOX_STORE_S3_CLIENT_KWARGS= + +# Whether to automatically create the S3 bucket if it doesn't exist. +# DIRACX_SANDBOX_STORE_AUTO_CREATE_BUCKET=False + +# Validity duration in seconds for pre-signed S3 URLs. +# +# This determines how long generated download/upload URLs remain valid +# before expiring. Default: 300 seconds (5 minutes). +# DIRACX_SANDBOX_STORE_URL_VALIDITY_SECONDS=300 + +# Logical name of the Storage Element for the sandbox store. +# +# This name is used within DIRAC to refer to this sandbox storage +# endpoint in job descriptions and file catalogs. +# DIRACX_SANDBOX_STORE_SE_NAME=SandboxSE + + diff --git a/.gitignore b/.gitignore index 900a0e643..b898f7da7 100644 --- a/.gitignore +++ b/.gitignore @@ -96,3 +96,4 @@ docs/source/_build .pixi pixi.lock *.egg-info +docs/templates/_builtin_markdown.jinja diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9d2e49583..31501d1d2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -89,7 +89,7 @@ repos: hooks: - id: settings-doc-check name: Generate settings documentation - entry: scripts/check_settings_doc.sh + entry: pixi run -e default python scripts/generate_settings_docs.py language: system pass_filenames: false - files: ^diracx-core/src/diracx/core/settings\.py$ + files: ^(diracx-.*/src/diracx/.*/settings\.py|docs/.*\.j2|docs/templates/.*\.jinja|scripts/generate_settings_docs\.py)$ diff --git a/docs/admin/reference/env-variables.md b/docs/admin/reference/env-variables.md index c96cc7a90..9ac4ba5af 100644 --- a/docs/admin/reference/env-variables.md +++ b/docs/admin/reference/env-variables.md @@ -1,39 +1,183 @@ # List of environment variables -## Core - -- `DIRACX_CONFIG_BACKEND_URL`: The URL of the configuration backend. - -## Services: - -- `DIRACX_SERVICE_AUTH_TOKEN_ISSUER`: The issuer for the auth tokens. -- `DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS`: A JSON-encoded list of allowed redirect URIs for the authorization code - flow. -- `DIRACX_SERVICE_AUTH_DEVICE_FLOW_EXPIRATION_SECONDS`: The expiration time for the device flow in seconds. -- `DIRACX_SERVICE_AUTH_AUTHORIZATION_FLOW_EXPIRATION_SECONDS`: The expiration time for the authorization flow in - seconds. -- `DIRACX_SERVICE_AUTH_STATE_KEY`: The key used to encrypt the state in the authorization code flow. -- `DIRACX_SERVICE_AUTH_TOKEN_KEYSTORE`: The path to the JWKS file containing the token signing keys. -- `DIRACX_SERVICE_AUTH_TOKEN_ALLOWED_ALGORITHMS`: A JSON-encoded list of allowed algorithms for token signing. -- `DIRACX_SERVICE_AUTH_ACCESS_TOKEN_EXPIRE_MINUTES`: The expiration time for the access token in minutes. -- `DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES`: The expiration time for the refresh token in minutes. -- `DIRACX_SANDBOX_STORE_BUCKET_NAME`: The name of the S3 bucket for the sandbox store. -- `DIRACX_SANDBOX_STORE_S3_CLIENT_KWARGS`: A JSON-encoded dictionary of keyword arguments for the S3 client. -- `DIRACX_SANDBOX_STORE_AUTO_CREATE_BUCKET`: Whether to automatically create the S3 bucket if it doesn't exist. -- `DIRACX_SANDBOX_STORE_URL_VALIDITY_SECONDS`: The validity of the presigned URLs for the sandbox store in seconds. -- `DIRACX_SANDBOX_STORE_SE_NAME`: The name of the storage element for the sandbox store. -- `DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY`: The hashed API key for the legacy exchange endpoint. -- `DIRACX_SERVICE_JOBS_ENABLED`: Whether the jobs service is enabled. - -## Databases: - -- `DIRACX_DB_URL_`: The URL for the SQL database ``. -- `DIRACX_OS_DB_`: A JSON-encoded dictionary of connection keyword arguments for the OpenSearch database `` - -## OTEL: - -- `DIRACX_OTEL_ENABLED`: Whether OpenTelemetry is enabled. -- `DIRACX_OTEL_APPLICATION_NAME`: The name of the application for OpenTelemetry. -- `DIRACX_OTEL_GRPC_ENDPOINT`: The gRPC endpoint for the OpenTelemetry collector. -- `DIRACX_OTEL_GRPC_INSECURE`: Whether to use an insecure gRPC connection for the OpenTelemetry collector. -- `DIRACX_OTEL_HEADERS`: A JSON-encoded dictionary of headers to pass to the OpenTelemetry collector. +*This page is auto-generated from the settings classes in `diracx.core.settings`.* + + +## AuthSettings + +Settings for the authentication service. + + +### `DIRACX_SERVICE_AUTH_DIRAC_CLIENT_ID` + +*Optional*, default value: `myDIRACClientID` + +OAuth2 client identifier for DIRAC services. + +This should match the client ID registered with the identity provider. + +### `DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS` + +*Optional*, default value: `[]` + +List of allowed redirect URLs for OAuth2 authorization flow. + +These URLs must be pre-registered and should match the redirect URIs +configured in the OAuth2 client registration. +Example: ["http://localhost:8000/docs/oauth2-redirect"] + +### `DIRACX_SERVICE_AUTH_DEVICE_FLOW_EXPIRATION_SECONDS` + +*Optional*, default value: `600` + +Expiration time in seconds for device flow authorization requests. + +After this time, the device code becomes invalid and users must restart +the device flow process. Default: 10 minutes. + +### `DIRACX_SERVICE_AUTH_AUTHORIZATION_FLOW_EXPIRATION_SECONDS` + +*Optional*, default value: `300` + +Expiration time in seconds for authorization code flow. + +The time window during which the authorization code remains valid +before it must be exchanged for tokens. Default: 5 minutes. + +### `DIRACX_SERVICE_AUTH_STATE_KEY` + +**Required** + +Encryption key used to encrypt/decrypt the state parameter passed to the IAM. + +This key ensures the integrity and confidentiality of state information +during OAuth2 flows. Must be a valid Fernet key. + +### `DIRACX_SERVICE_AUTH_TOKEN_ISSUER` + +**Required** + +The issuer identifier for JWT tokens. + +This should be a URI that uniquely identifies the token issuer and +matches the 'iss' claim in issued JWT tokens. + +### `DIRACX_SERVICE_AUTH_TOKEN_KEYSTORE` + +**Required** + +Keystore containing the cryptographic keys used for signing JWT tokens. + +This includes both public and private keys for token signature +generation and verification. + +### `DIRACX_SERVICE_AUTH_TOKEN_ALLOWED_ALGORITHMS` + +*Optional*, default value: `['RS256', 'EdDSA']` + +List of allowed cryptographic algorithms for JWT token signing. + +Supported algorithms include RS256 (RSA with SHA-256) and EdDSA +(Edwards-curve Digital Signature Algorithm). Default: ["RS256", "EdDSA"] + +### `DIRACX_SERVICE_AUTH_ACCESS_TOKEN_EXPIRE_MINUTES` + +*Optional*, default value: `20` + +Expiration time in minutes for access tokens. + +After this duration, access tokens become invalid and must be refreshed +or re-obtained. Default: 20 minutes. + +### `DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES` + +*Optional*, default value: `60` + +Expiration time in minutes for refresh tokens. + +The maximum lifetime of refresh tokens before they must be re-issued +through a new authentication flow. Default: 60 minutes. + +### `DIRACX_SERVICE_AUTH_AVAILABLE_PROPERTIES` + +*Optional* + +Set of security properties available in this DIRAC installation. + +These properties define various authorization capabilities and are used +for access control decisions. Defaults to all available security properties. + + + + +## SandboxStoreSettings + +Settings for the sandbox store. + + +### `DIRACX_SANDBOX_STORE_BUCKET_NAME` + +**Required** + +Name of the S3 bucket used for storing job sandboxes. + +This bucket will contain input and output sandbox files for DIRAC jobs. +The bucket must exist or auto_create_bucket must be enabled. + +### `DIRACX_SANDBOX_STORE_S3_CLIENT_KWARGS` + +**Required** + +Configuration parameters passed to the S3 client. + +### `DIRACX_SANDBOX_STORE_AUTO_CREATE_BUCKET` + +*Optional*, default value: `False` + +Whether to automatically create the S3 bucket if it doesn't exist. + +### `DIRACX_SANDBOX_STORE_URL_VALIDITY_SECONDS` + +*Optional*, default value: `300` + +Validity duration in seconds for pre-signed S3 URLs. + +This determines how long generated download/upload URLs remain valid +before expiring. Default: 300 seconds (5 minutes). + +### `DIRACX_SANDBOX_STORE_SE_NAME` + +*Optional*, default value: `SandboxSE` + +Logical name of the Storage Element for the sandbox store. + +This name is used within DIRAC to refer to this sandbox storage +endpoint in job descriptions and file catalogs. + + + + +## OTELSettings + +Settings for the Open Telemetry Configuration. + + +### `DIRACX_OTEL_ENABLED` + +*Optional*, default value: `False` + +### `DIRACX_OTEL_APPLICATION_NAME` + +*Optional*, default value: `diracx` + +### `DIRACX_OTEL_GRPC_ENDPOINT` + +*Optional*, default value: `` + +### `DIRACX_OTEL_GRPC_INSECURE` + +*Optional*, default value: `True` + +### `DIRACX_OTEL_HEADERS` + +*Optional*, default value: `None` diff --git a/docs/admin/reference/env-variables.md.j2 b/docs/admin/reference/env-variables.md.j2 new file mode 100644 index 000000000..eecae741e --- /dev/null +++ b/docs/admin/reference/env-variables.md.j2 @@ -0,0 +1,11 @@ +{% from '_render_class.jinja' import render_class %} + +# List of environment variables + +*This page is auto-generated from the settings classes in `diracx.core.settings`.* + +{{ render_class('AuthSettings') }} + +{{ render_class('SandboxStoreSettings') }} + +{{ render_class('OTELSettings') }} diff --git a/docs/dev/reference/env-variables.md b/docs/dev/reference/env-variables.md index 41cc34b0b..949a06234 100644 --- a/docs/dev/reference/env-variables.md +++ b/docs/dev/reference/env-variables.md @@ -1,10 +1,12 @@ -# List of development environment variables +# List of environment variables -*This page is auto-generated. Do not edit directly.* +*This page is auto-generated from the DevelopmentSettings class in `diracx.core.settings`.* - +## DevelopmentSettings -## `DIRACX_DEV_CRASH_ON_MISSED_ACCESS_POLICY` +Settings for the Development Configuration that can influence run time. + +### `DIRACX_DEV_CRASH_ON_MISSED_ACCESS_POLICY` *Optional*, default value: `False` @@ -12,143 +14,3 @@ When set to true (only for demo/CI), crash if an access policy isn't called. This is useful for development and testing to ensure all endpoints have proper access control policies defined. - -## `DIRACX_SERVICE_AUTH_DIRAC_CLIENT_ID` - -*Optional*, default value: `myDIRACClientID` - -OAuth2 client identifier for DIRAC services. - -This should match the client ID registered with the identity provider. - -## `DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS` - -*Optional*, default value: `[]` - -List of allowed redirect URLs for OAuth2 authorization flow. - -These URLs must be pre-registered and should match the redirect URIs -configured in the OAuth2 client registration. -Example: ["http://localhost:8000/docs/oauth2-redirect"] - -## `DIRACX_SERVICE_AUTH_DEVICE_FLOW_EXPIRATION_SECONDS` - -*Optional*, default value: `600` - -Expiration time in seconds for device flow authorization requests. - -After this time, the device code becomes invalid and users must restart -the device flow process. Default: 10 minutes. - -## `DIRACX_SERVICE_AUTH_AUTHORIZATION_FLOW_EXPIRATION_SECONDS` - -*Optional*, default value: `300` - -Expiration time in seconds for authorization code flow. - -The time window during which the authorization code remains valid -before it must be exchanged for tokens. Default: 5 minutes. - -## `DIRACX_SERVICE_AUTH_STATE_KEY` - -**Required** - -Encryption key used to encrypt/decrypt the state parameter passed to the IAM. - -This key ensures the integrity and confidentiality of state information -during OAuth2 flows. Must be a valid Fernet key. - -## `DIRACX_SERVICE_AUTH_TOKEN_ISSUER` - -**Required** - -The issuer identifier for JWT tokens. - -This should be a URI that uniquely identifies the token issuer and -matches the 'iss' claim in issued JWT tokens. - -## `DIRACX_SERVICE_AUTH_TOKEN_KEYSTORE` - -**Required** - -Keystore containing the cryptographic keys used for signing JWT tokens. - -This includes both public and private keys for token signature -generation and verification. - -## `DIRACX_SERVICE_AUTH_TOKEN_ALLOWED_ALGORITHMS` - -*Optional*, default value: `['RS256', 'EdDSA']` - -List of allowed cryptographic algorithms for JWT token signing. - -Supported algorithms include RS256 (RSA with SHA-256) and EdDSA -(Edwards-curve Digital Signature Algorithm). Default: ["RS256", "EdDSA"] - -## `DIRACX_SERVICE_AUTH_ACCESS_TOKEN_EXPIRE_MINUTES` - -*Optional*, default value: `20` - -Expiration time in minutes for access tokens. - -After this duration, access tokens become invalid and must be refreshed -or re-obtained. Default: 20 minutes. - -## `DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES` - -*Optional*, default value: `60` - -Expiration time in minutes for refresh tokens. - -The maximum lifetime of refresh tokens before they must be re-issued -through a new authentication flow. Default: 60 minutes. - -## `DIRACX_SERVICE_AUTH_AVAILABLE_PROPERTIES` - -*Optional* - -Set of security properties available in this DIRAC installation. - -These properties define various authorization capabilities and are used -for access control decisions. Defaults to all available security properties. - -## `DIRACX_SANDBOX_STORE_BUCKET_NAME` - -**Required** - -Name of the S3 bucket used for storing job sandboxes. - -This bucket will contain input and output sandbox files for DIRAC jobs. -The bucket must exist or auto_create_bucket must be enabled. - -## `DIRACX_SANDBOX_STORE_S3_CLIENT_KWARGS` - -**Required** - -Configuration parameters passed to the S3 client. - -## `DIRACX_SANDBOX_STORE_AUTO_CREATE_BUCKET` - -*Optional*, default value: `False` - -Whether to automatically create the S3 bucket if it doesn't exist. - -## `DIRACX_SANDBOX_STORE_URL_VALIDITY_SECONDS` - -*Optional*, default value: `300` - -Validity duration in seconds for pre-signed S3 URLs. - -This determines how long generated download/upload URLs remain valid -before expiring. Default: 300 seconds (5 minutes). - -## `DIRACX_SANDBOX_STORE_SE_NAME` - -*Optional*, default value: `SandboxSE` - -Logical name of the Storage Element for the sandbox store. - -This name is used within DIRAC to refer to this sandbox storage -endpoint in job descriptions and file catalogs. - - diff --git a/docs/dev/reference/env-variables.md.j2 b/docs/dev/reference/env-variables.md.j2 new file mode 100644 index 000000000..6bc891a90 --- /dev/null +++ b/docs/dev/reference/env-variables.md.j2 @@ -0,0 +1,7 @@ +{% from '_render_class.jinja' import render_class %} + +# List of environment variables + +*This page is auto-generated from the DevelopmentSettings class in `diracx.core.settings`.* + +{{ render_class('DevelopmentSettings') }} diff --git a/docs/dev/reference/pixi-tasks.md b/docs/dev/reference/pixi-tasks.md index b7866fde8..34ad9bc4f 100644 --- a/docs/dev/reference/pixi-tasks.md +++ b/docs/dev/reference/pixi-tasks.md @@ -48,8 +48,7 @@ This page documents the available pixi tasks. ## Settings Tasks -- `generate-settings-doc`: Generate settings documentation in Markdown format -- `generate-settings-dotenv`: Generate a dotenv file from settings +- `generate-settings-doc`: Auto-discover and generate settings documentation with validation ## Shellcheck Tasks diff --git a/docs/templates/_render_class.jinja b/docs/templates/_render_class.jinja new file mode 100644 index 000000000..242d6fb4b --- /dev/null +++ b/docs/templates/_render_class.jinja @@ -0,0 +1,40 @@ +{# +Reusable macro to render a settings class with all its environment variables. + +This macro encapsulates all the complexity of rendering fields, so the main +template can be kept simple and clean. +#} + +{% macro render_class(class_name) %} +{# Find the class by name from the classes dict #} +{% set cls = namespace(found=none) %} +{% for c, field_list in classes.items() %} + {% if c.__name__ == class_name %} + {% set cls.found = c %} + {% endif %} +{% endfor %} + +{% if cls.found %} +## {{ cls.found.__name__ }} + +{{ cls.found.__doc__ or "*No description available.*" }} + +{# Build the fields list in the format expected by the built-in template #} +{% set ns = namespace(fields=[]) %} +{% for field_name, field_info in cls.found.model_fields.items() %} + {% set env_prefix = cls.found.model_config.get('env_prefix', '') %} + {% set env_name = (env_prefix ~ field_name).upper() %} + {% set _ = ns.fields.append((env_name, field_info)) %} +{% endfor %} + +{# Set the fields variable and include the built-in settings_doc template #} +{% set fields = ns.fields %} +{% include '_builtin_markdown.jinja' with context %} +{% else %} +{# Class not found - provide a helpful error #} +**Error: Class '{{ class_name }}' not found in diracx.core.settings** + +Available classes: {{ classes.keys() | map(attribute='__name__') | list | join(', ') }} +{% endif %} + +{% endmacro %} diff --git a/pixi.toml b/pixi.toml index ab64784bb..df37211bc 100644 --- a/pixi.toml +++ b/pixi.toml @@ -63,19 +63,8 @@ description = "Run the tests for diracx-routers" # Settings documentation task [feature.settings-doc.tasks.generate-settings-doc] -cmd = """settings-doc generate \ - --module diracx.core.settings \ - --output-format markdown \ - --update docs/dev/reference/env-variables.md \ - --between "" "" \ - --heading-offset 1""" -description = "Generate settings documentation in Markdown format" - -[feature.settings-doc.tasks.generate-settings-dotenv] -cmd = """settings-doc generate \ - --module diracx.core.settings \ - --output-format dotenv > .env""" -description = "Generate a dotenv file from settings" +cmd = "python scripts/generate_settings_docs.py" +description = "Auto-discover and generate settings documentation with validation" # Gubbins features for providing dependencies [feature.gubbins.pypi-dependencies] diff --git a/scripts/check_settings_doc.sh b/scripts/check_settings_doc.sh deleted file mode 100755 index dc0b18b2c..000000000 --- a/scripts/check_settings_doc.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Run the settings-doc generator via pixi and fail if the committed file would change. -# This allows a pre-commit hook to check that generated documentation is up-to-date -# without letting the hook itself modify the repository and cause the hook to fail. - -repo_root="$(cd "$(dirname "$0")/.." && pwd)" -cd "$repo_root" - -TARGET="docs/dev/reference/env-variables.md" - -# Save the current state of the target file -cp "$TARGET" "${TARGET}.backup" - -echo "Running settings-doc generator (pixi run generate-settings-doc)..." -pixi run generate-settings-doc - -# Normalize both files by removing empty lines, then compare -# This handles cases where settings-doc adds/removes blank lines inconsistently -normalize_file() { - grep -v '^[[:space:]]*$' "$1" || true -} - -if diff -q <(normalize_file "$TARGET") <(normalize_file "${TARGET}.backup") > /dev/null 2>&1; then - # Files are equivalent ignoring blank lines, restore original and pass - cp "${TARGET}.backup" "$TARGET" - rm "${TARGET}.backup" - echo "Generated documentation is up-to-date." - exit 0 -fi - -# Files have meaningful differences -echo -echo "ERROR: Generated settings documentation is out of date." -echo "Run 'pixi run generate-settings-doc' and commit the updated file." -echo -echo "Diff (generated vs current):" -diff "$TARGET" "${TARGET}.backup" || true -# Restore the original file so pre-commit doesn't see modifications -mv "${TARGET}.backup" "$TARGET" -exit 1 diff --git a/scripts/generate_settings_docs.py b/scripts/generate_settings_docs.py new file mode 100644 index 000000000..c7532e7a1 --- /dev/null +++ b/scripts/generate_settings_docs.py @@ -0,0 +1,422 @@ +#!/usr/bin/env python3 +"""Automatically discover and validate settings documentation. + +This script: +1. Syncs the built-in settings_doc template to avoid recursion +2. Discovers all Settings classes across the DiracX codebase +3. Checks which classes are documented in templates +4. Warns about undocumented classes +5. Generates documentation for all templates +6. Generates .env.example file with all settings +""" + +from __future__ import annotations + +import importlib +import inspect +import pkgutil +import re +import shutil +import sys +from pathlib import Path +from typing import Any + +import settings_doc +from jinja2 import Environment, FileSystemLoader, select_autoescape +from pydantic_settings import BaseSettings +from settings_doc import OutputFormat, importing, render +from settings_doc.main import _model_fields +from settings_doc.template_functions import JINJA_ENV_GLOBALS + +from diracx.core.settings import ServiceSettingsBase + + +def sync_builtin_template(docs_dir: Path) -> None: + """Sync the built-in settings_doc markdown template. + + This copies the built-in template to '_builtin_markdown.jinja' so our custom + 'markdown.jinja' can include it without causing recursion issues in Jinja2. + + Args: + docs_dir: The docs directory where templates are stored + + """ + # Get the path to the built-in settings_doc template + builtin_template_dir = Path(settings_doc.__file__).parent / "templates" + builtin_markdown = builtin_template_dir / "markdown.jinja" + + # Define our custom templates directory + custom_template_dir = docs_dir / "templates" + custom_template_dir.mkdir(parents=True, exist_ok=True) + + # Copy the built-in template with a different name to avoid recursion + target_template = custom_template_dir / "_builtin_markdown.jinja" + shutil.copy2(builtin_markdown, target_template) + + +def discover_all_settings_classes() -> dict[str, dict[str, Any]]: + """Automatically discover all Settings classes in the DiracX codebase. + + Uses pkgutil.walk_packages() to walk all diracx.* packages. + + Returns: + Dict mapping class names to their info (module, class object, etc.) + + """ + settings_classes = {} + + import diracx + + # Use pkgutil to walk all packages and subpackages + # This is the canonical way to discover modules in a package + if hasattr(diracx, "__path__"): + for _, module_name, _ in pkgutil.walk_packages( + path=diracx.__path__, + prefix="diracx.", + onerror=lambda x: None, # Silently skip modules with import errors + ): + try: + module = importlib.import_module(module_name) + + # Inspect the module for Settings classes + for name, obj in inspect.getmembers(module, inspect.isclass): + # Check if it's a subclass of ServiceSettingsBase + if ( + issubclass(obj, ServiceSettingsBase) + and obj is not ServiceSettingsBase + and obj is not BaseSettings + # Only include classes defined in this module (not imported) + and obj.__module__ == module_name + ): + settings_classes[name] = { + "module": module_name, + "class": obj, + "file": inspect.getfile(obj), + } + except (ImportError, AttributeError, TypeError): + # Skip modules that can't be imported or inspected + continue + else: + raise ImportError("Cannot find diracx package paths") + + return settings_classes + + +def discover_templates(docs_dir: Path) -> dict[str, Path]: + """Discover all Jinja2 template files in the docs directory. + + Returns: + Dict mapping template names to their paths + + """ + templates = {} + + # Look for .j2 and .jinja files + for pattern in ["**/*.j2", "**/*.jinja"]: + for template_file in docs_dir.glob(pattern): + # Skip the helper templates (those starting with _) + if template_file.name.startswith("_"): + continue + + # Use relative path from docs dir as the key + rel_path = template_file.relative_to(docs_dir) + templates[str(rel_path)] = template_file + + return templates + + +def extract_documented_classes(template_file: Path) -> set[str]: + """Extract which Settings classes are documented in a template. + + Looks for {{ render_class('ClassName') }} patterns. + """ + content = template_file.read_text() + + # Pattern to match render_class('ClassName') or render_class("ClassName") + pattern = r"render_class\(['\"](\w+)['\"]\)" + + matches = re.findall(pattern, content) + return set(matches) + + +def validate_documentation( + settings_classes: dict[str, dict], + templates: dict[str, Path], +) -> tuple[bool, list[str]]: + """Validate that all Settings classes are documented. + + Returns: + Tuple of (all_documented, warnings) + + """ + warnings = [] + all_documented = True + + # Track which classes are documented in which templates + class_to_templates: dict[str, list[str]] = {} + + # Get all documented classes across all templates + for template_name, template_file in templates.items(): + documented = extract_documented_classes(template_file) + + if documented: + print(f"šŸ“„ {template_name}: documents {len(documented)} classes") + for cls_name in sorted(documented): + # Track which template documents this class + if cls_name not in class_to_templates: + class_to_templates[cls_name] = [] + class_to_templates[cls_name].append(template_name) + + # Check if class exists + if cls_name not in settings_classes: + warnings.append( + f" āš ļø Template '{template_name}' references unknown class: {cls_name}" + ) + + # Check for classes documented in multiple templates + for cls_name, template_list in class_to_templates.items(): + if len(template_list) > 1: + warnings.append( + f"\nāš ļø Class '{cls_name}' is documented in multiple templates:" + ) + for tmpl in template_list: + warnings.append(f" - {tmpl}") + warnings.append( + " This might be intentional, but verify it's not a mistake." + ) + + # Get all documented classes (from any template) + all_documented_classes = set(class_to_templates.keys()) + + # Check for undocumented classes + undocumented = set(settings_classes.keys()) - all_documented_classes + + # Exclude ServiceSettingsBase as it's the base class + undocumented.discard("ServiceSettingsBase") + + if undocumented: + all_documented = False + warnings.append("\nāŒ Undocumented Settings classes found:") + for cls_name in sorted(undocumented): + info = settings_classes[cls_name] + warnings.append(f" - {cls_name} (in {info['module']})") + warnings.append( + f" Add to template: {{{{ render_class('{cls_name}') }}}}" + ) + + return all_documented, warnings + + +def generate_all_templates( + templates: dict[str, Path], + settings_classes: dict[str, dict], + docs_dir: Path, + repo_root: Path, +) -> None: + """Generate documentation for all discovered templates.""" + # Collect all unique modules that contain settings + modules = sorted(set(info["module"] for info in settings_classes.values())) + + # Import settings classes from all modules + settings = importing.import_module_path(tuple(modules)) + + if not settings: + raise ValueError(f"No settings classes found in {modules}") + + # Prepare data structures for templates + fields = list( + (env_name, field) for cls in settings for env_name, field in _model_fields(cls) + ) + + classes = {cls: list(cls.model_fields.values()) for cls in settings} + + # Set up Jinja2 environment + env = Environment( + loader=FileSystemLoader([str(docs_dir), str(docs_dir / "templates")]), + autoescape=select_autoescape(), + trim_blocks=True, + lstrip_blocks=True, + keep_trailing_newline=True, + ) + + env.globals.update(JINJA_ENV_GLOBALS) + env.globals.update( + { + "heading_offset": 2, + "fields": fields, + "classes": classes, + "all_classes": settings, + } + ) + + # Generate each template to .bak file + generated_files = [] + for template_name, template_file in templates.items(): + template = env.get_template(str(template_name)) + rendered = template.render().strip() + "\n" + + # Output file is the template file without the .j2/.jinja extension + if template_file.suffix == ".j2": + output_file = template_file.with_suffix("") + elif template_file.suffix == ".jinja": + output_file = template_file.with_name(template_file.stem + ".md") + else: + output_file = template_file.with_suffix(".md") + + # Write to .bak file + backup_file = output_file.with_suffix(output_file.suffix + ".bak") + backup_file.write_text(rendered) + generated_files.append((output_file, backup_file)) + + return generated_files + + +def generate_dotenv_file( + settings_classes: dict[str, dict[str, Any]], + output_path: Path, +) -> Path: + """Generate a .env file with all Settings classes. + + Args: + settings_classes: Dictionary of discovered Settings classes + output_path: Path where the .env file should be written + + Returns: + Path to the .bak file that was created + + """ + dotenv_sections = [] + + # Generate .env content for each Settings class + for class_name in sorted(settings_classes.keys()): + info = settings_classes[class_name] + + # Use settings_doc.render to generate DOTENV format + # class_path should be in format "module.ClassName" + class_path = f"{info['module']}.{class_name}" + dotenv_content = render( + output_format=OutputFormat.DOTENV, + class_path=(class_path,), # Must be a tuple + ) + + # Add a header comment for the class + section = f"# {class_name} (from {info['module']})\n{dotenv_content}\n" + dotenv_sections.append(section) + + # Combine all sections + full_content = "# Auto-generated .env file with all DiracX settings\n" + full_content += "# This file contains all available environment variables.\n" + full_content += "# Uncomment and set the values you need.\n\n" + full_content += "\n".join(dotenv_sections) + + # Write to .bak file + backup_path = output_path.with_suffix(output_path.suffix + ".bak") + backup_path.write_text(full_content) + + return backup_path + + +def compare_and_update_files( + generated_files: list[tuple[Path, Path]], repo_root: Path +) -> bool: + """Compare generated .bak files with originals and update if needed. + + Args: + generated_files: List of (original_path, backup_path) tuples + repo_root: Repository root for relative path display + + Returns: + True if all files were already up to date, False if any were updated + + """ + + # Helper to normalize content (remove empty lines for comparison) + def normalize(text: str) -> str: + return "\n".join(line for line in text.splitlines() if line.strip()) + + all_up_to_date = True + + for original_file, backup_file in generated_files: + if original_file.exists(): + original_content = normalize(original_file.read_text()) + backup_content = normalize(backup_file.read_text()) + + if original_content != backup_content: + # Replace original with backup + backup_file.replace(original_file) + print(f"āœ“ Updated {original_file.relative_to(repo_root)}") + all_up_to_date = False + else: + # Files match, remove backup + backup_file.unlink() + print(f" No changes: {original_file.relative_to(repo_root)}") + else: + # New file - move backup to original + backup_file.replace(original_file) + print(f"āœ“ Created {original_file.relative_to(repo_root)}") + all_up_to_date = False + + return all_up_to_date + + +def main(): + """Main entry point.""" + repo_root = Path(__file__).parent.parent + docs_dir = repo_root / "docs" + + print("šŸ”„ Syncing built-in template...") + sync_builtin_template(docs_dir) + + print("šŸ” Discovering Settings classes...") + settings_classes = discover_all_settings_classes() + + print(f"āœ“ Found {len(settings_classes)} Settings classes:") + for name, info in sorted(settings_classes.items()): + print(f" - {name} ({info['module']})") + + print("\nšŸ” Discovering documentation templates...") + templates = discover_templates(docs_dir) + + print(f"āœ“ Found {len(templates)} template(s):") + for name in sorted(templates.keys()): + print(f" - {name}") + + print("\nšŸ” Validating documentation coverage...") + all_documented, warnings = validate_documentation(settings_classes, templates) + + if warnings: + for warning in warnings: + print(warning) + + if not all_documented: + print("\nāŒ Documentation is incomplete!") + print(" Add missing classes to your templates.") + return 1 + + print("\nāœ“ All Settings classes are documented!") + + # Generate documentation (always to .bak files first) + print("\nšŸ“ Generating documentation from templates...") + generated_files = generate_all_templates( + templates, settings_classes, docs_dir, repo_root + ) + + print("\nšŸ“ Generating .env file...") + dotenv_path = repo_root / ".env.example" + dotenv_backup = generate_dotenv_file(settings_classes, dotenv_path) + generated_files.append((dotenv_path, dotenv_backup)) + + # Update files if they're different + print("\nšŸ“ Updating files...") + all_up_to_date = compare_and_update_files(generated_files, repo_root) + + if all_up_to_date: + print("\nāœ… All files were already up to date!") + else: + print("\nāœ… Documentation updated successfully!") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 228865b76e426fc5cea16c46974b4d77eedf6c0d Mon Sep 17 00:00:00 2001 From: Ryunosuke O'Neil Date: Fri, 7 Nov 2025 14:58:44 +0100 Subject: [PATCH 3/3] fix: use separate tooling env for settings-doc fix: well that didn't work fix: update tasks fix: format settings doc files --- .env.example | 121 -------------------------- .gitignore | 3 + docs/admin/reference/env-variables.md | 12 +-- pixi.toml | 26 ++++-- scripts/generate_settings_docs.py | 28 ++++-- 5 files changed, 44 insertions(+), 146 deletions(-) delete mode 100644 .env.example diff --git a/.env.example b/.env.example deleted file mode 100644 index 632a502c3..000000000 --- a/.env.example +++ /dev/null @@ -1,121 +0,0 @@ -# Auto-generated .env file with all DiracX settings -# This file contains all available environment variables. -# Uncomment and set the values you need. - -# AuthSettings (from diracx.core.settings) -# OAuth2 client identifier for DIRAC services. -# -# This should match the client ID registered with the identity provider. -# DIRACX_SERVICE_AUTH_DIRAC_CLIENT_ID=myDIRACClientID - -# List of allowed redirect URLs for OAuth2 authorization flow. -# -# These URLs must be pre-registered and should match the redirect URIs -# configured in the OAuth2 client registration. -# Example: ["http://localhost:8000/docs/oauth2-redirect"] -# DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS=[] - -# Expiration time in seconds for device flow authorization requests. -# -# After this time, the device code becomes invalid and users must restart -# the device flow process. Default: 10 minutes. -# DIRACX_SERVICE_AUTH_DEVICE_FLOW_EXPIRATION_SECONDS=600 - -# Expiration time in seconds for authorization code flow. -# -# The time window during which the authorization code remains valid -# before it must be exchanged for tokens. Default: 5 minutes. -# DIRACX_SERVICE_AUTH_AUTHORIZATION_FLOW_EXPIRATION_SECONDS=300 - -# Encryption key used to encrypt/decrypt the state parameter passed to the IAM. -# -# This key ensures the integrity and confidentiality of state information -# during OAuth2 flows. Must be a valid Fernet key. -DIRACX_SERVICE_AUTH_STATE_KEY= - -# The issuer identifier for JWT tokens. -# -# This should be a URI that uniquely identifies the token issuer and -# matches the 'iss' claim in issued JWT tokens. -DIRACX_SERVICE_AUTH_TOKEN_ISSUER= - -# Keystore containing the cryptographic keys used for signing JWT tokens. -# -# This includes both public and private keys for token signature -# generation and verification. -DIRACX_SERVICE_AUTH_TOKEN_KEYSTORE= - -# List of allowed cryptographic algorithms for JWT token signing. -# -# Supported algorithms include RS256 (RSA with SHA-256) and EdDSA -# (Edwards-curve Digital Signature Algorithm). Default: ["RS256", "EdDSA"] -# DIRACX_SERVICE_AUTH_TOKEN_ALLOWED_ALGORITHMS=['RS256', 'EdDSA'] - -# Expiration time in minutes for access tokens. -# -# After this duration, access tokens become invalid and must be refreshed -# or re-obtained. Default: 20 minutes. -# DIRACX_SERVICE_AUTH_ACCESS_TOKEN_EXPIRE_MINUTES=20 - -# Expiration time in minutes for refresh tokens. -# -# The maximum lifetime of refresh tokens before they must be re-issued -# through a new authentication flow. Default: 60 minutes. -# DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES=60 - -# Set of security properties available in this DIRAC installation. -# -# These properties define various authorization capabilities and are used -# for access control decisions. Defaults to all available security properties. -# DIRACX_SERVICE_AUTH_AVAILABLE_PROPERTIES= - - - -# DevelopmentSettings (from diracx.core.settings) -# When set to true (only for demo/CI), crash if an access policy isn't called. -# -# This is useful for development and testing to ensure all endpoints have proper -# access control policies defined. -# DIRACX_DEV_CRASH_ON_MISSED_ACCESS_POLICY=False - - - -# OTELSettings (from diracx.routers.otel) -# DIRACX_OTEL_ENABLED=False - -# DIRACX_OTEL_APPLICATION_NAME=diracx - -# DIRACX_OTEL_GRPC_ENDPOINT= - -# DIRACX_OTEL_GRPC_INSECURE=True - -# DIRACX_OTEL_HEADERS= - - - -# SandboxStoreSettings (from diracx.core.settings) -# Name of the S3 bucket used for storing job sandboxes. -# -# This bucket will contain input and output sandbox files for DIRAC jobs. -# The bucket must exist or auto_create_bucket must be enabled. -DIRACX_SANDBOX_STORE_BUCKET_NAME= - -# Configuration parameters passed to the S3 client. -DIRACX_SANDBOX_STORE_S3_CLIENT_KWARGS= - -# Whether to automatically create the S3 bucket if it doesn't exist. -# DIRACX_SANDBOX_STORE_AUTO_CREATE_BUCKET=False - -# Validity duration in seconds for pre-signed S3 URLs. -# -# This determines how long generated download/upload URLs remain valid -# before expiring. Default: 300 seconds (5 minutes). -# DIRACX_SANDBOX_STORE_URL_VALIDITY_SECONDS=300 - -# Logical name of the Storage Element for the sandbox store. -# -# This name is used within DIRAC to refer to this sandbox storage -# endpoint in job descriptions and file catalogs. -# DIRACX_SANDBOX_STORE_SE_NAME=SandboxSE - - diff --git a/.gitignore b/.gitignore index b898f7da7..6aac865c8 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,6 @@ docs/source/_build pixi.lock *.egg-info docs/templates/_builtin_markdown.jinja + +# docs site +site diff --git a/docs/admin/reference/env-variables.md b/docs/admin/reference/env-variables.md index 9ac4ba5af..ddd1304e9 100644 --- a/docs/admin/reference/env-variables.md +++ b/docs/admin/reference/env-variables.md @@ -2,12 +2,10 @@ *This page is auto-generated from the settings classes in `diracx.core.settings`.* - ## AuthSettings Settings for the authentication service. - ### `DIRACX_SERVICE_AUTH_DIRAC_CLIENT_ID` *Optional*, default value: `myDIRACClientID` @@ -107,14 +105,10 @@ Set of security properties available in this DIRAC installation. These properties define various authorization capabilities and are used for access control decisions. Defaults to all available security properties. - - - ## SandboxStoreSettings Settings for the sandbox store. - ### `DIRACX_SANDBOX_STORE_BUCKET_NAME` **Required** @@ -154,14 +148,10 @@ Logical name of the Storage Element for the sandbox store. This name is used within DIRAC to refer to this sandbox storage endpoint in job descriptions and file catalogs. - - - ## OTELSettings Settings for the Open Telemetry Configuration. - ### `DIRACX_OTEL_ENABLED` *Optional*, default value: `False` @@ -172,7 +162,7 @@ Settings for the Open Telemetry Configuration. ### `DIRACX_OTEL_GRPC_ENDPOINT` -*Optional*, default value: `` +*Optional*, default value: \`\` ### `DIRACX_OTEL_GRPC_INSECURE` diff --git a/pixi.toml b/pixi.toml index df37211bc..f3a670153 100644 --- a/pixi.toml +++ b/pixi.toml @@ -30,10 +30,6 @@ diracx-logic = { path = "diracx-logic", editable = true, extras = ["testing"] } [feature.diracx-routers.pypi-dependencies] diracx-routers = { path = "diracx-routers", editable = true, extras = ["testing"] } -# Settings documentation feature -[feature.settings-doc.pypi-dependencies] -settings-doc = "*" - # DiracX features for providing tasks. This is needed to make it so that running # "pixi run pytest-diracx-core -vvv --pdb" passes the arguments as expected. # See: https://github.com/prefix-dev/pixi/issues/1519#issuecomment-2651078457 @@ -61,11 +57,6 @@ description = "Run the tests for diracx-logic" cmd = "cd diracx-routers/ && pytest" description = "Run the tests for diracx-routers" -# Settings documentation task -[feature.settings-doc.tasks.generate-settings-doc] -cmd = "python scripts/generate_settings_docs.py" -description = "Auto-discover and generate settings documentation with validation" - # Gubbins features for providing dependencies [feature.gubbins.pypi-dependencies] gubbins = { path = "extensions/gubbins", editable = true, extras = ["testing"] } @@ -149,6 +140,22 @@ shellcheck = "*" cmd = "find . -not -wholename './.pixi/*' -name '*.sh' -print -exec shellcheck --exclude=SC1090,SC1091 --external-source '{}' ';'" description = "Run shellcheck on all shell scripts" +# Settings documentation feature +[feature.settings-doc.dependencies] +python = "*" +[feature.settings-doc.pypi-dependencies] +settings-doc = "*" +mdformat = "*" +mdformat-mkdocs = "*" +mdformat-gfm = "*" +mdformat-black = "*" +diracx-core = { path = "diracx-core", editable = true } +diracx-routers = { path = "diracx-routers", editable = true } +# Settings documentation task +[feature.settings-doc.tasks.generate-settings-doc] +cmd = "python scripts/generate_settings_docs.py" +description = "Auto-discover and generate settings documentation with validation" + [environments] # DiracX environments default = {features = ["task-diracx", "diracx", "diracx-core", "diracx-api", "diracx-cli", "diracx-client", "diracx-db", "diracx-logic", "diracx-routers", "settings-doc"], solve-group = "diracx"} @@ -177,6 +184,7 @@ gubbins-generate-client = {features = ["client-gen", "diracx-client", "gubbins-c mkdocs = {features = ["mkdocs"], no-default-feature = true} shellcheck = {features = ["shellcheck"], no-default-feature = true} pre-commit = {features = ["pre-commit"], no-default-feature = true} +settings-doc = {features = ["settings-doc"], no-default-feature = true} # Meta-tasks for running many tests at once [tasks.pytest-diracx-all-one-by-one] diff --git a/scripts/generate_settings_docs.py b/scripts/generate_settings_docs.py index c7532e7a1..0c86c9d61 100644 --- a/scripts/generate_settings_docs.py +++ b/scripts/generate_settings_docs.py @@ -17,6 +17,7 @@ import pkgutil import re import shutil +import subprocess import sys from pathlib import Path from typing import Any @@ -212,7 +213,7 @@ def generate_all_templates( settings_classes: dict[str, dict], docs_dir: Path, repo_root: Path, -) -> None: +) -> list: """Generate documentation for all discovered templates.""" # Collect all unique modules that contain settings modules = sorted(set(info["module"] for info in settings_classes.values())) @@ -334,6 +335,23 @@ def compare_and_update_files( def normalize(text: str) -> str: return "\n".join(line for line in text.splitlines() if line.strip()) + # First, run mdformat on all .bak files to ensure consistent formatting + backup_files = [backup for _, backup in generated_files] + if backup_files: + try: + subprocess.run( # noqa: S603 + ["mdformat", "--number"] + [str(f) for f in backup_files], + check=True, + capture_output=True, + text=True, + ) + except subprocess.CalledProcessError as e: + print(f"āš ļø mdformat failed: {e.stderr}") + return False + except FileNotFoundError: + print("āš ļø mdformat not found in environment") + return False + all_up_to_date = True for original_file, backup_file in generated_files: @@ -401,10 +419,10 @@ def main(): templates, settings_classes, docs_dir, repo_root ) - print("\nšŸ“ Generating .env file...") - dotenv_path = repo_root / ".env.example" - dotenv_backup = generate_dotenv_file(settings_classes, dotenv_path) - generated_files.append((dotenv_path, dotenv_backup)) + # print("\nšŸ“ Generating .env file...") + # dotenv_path = repo_root / ".env.example" + # dotenv_backup = generate_dotenv_file(settings_classes, dotenv_path) + # generated_files.append((dotenv_path, dotenv_backup)) # Update files if they're different print("\nšŸ“ Updating files...")