From 5fe637a8ba1cebc3153d874932acc85fc4884f14 Mon Sep 17 00:00:00 2001 From: Paco Valdez Date: Wed, 16 Jul 2025 12:44:51 -0700 Subject: [PATCH 1/2] Add python API --- .pydoc-markdown.yml | 9 + CLAUDE.md | 213 +++++++++++++- README.md | 178 +++++++++++- docs/lkml2cube_converter.md | 32 +++ docs/lkml2cube_main.md | 113 ++++++++ docs/lkml2cube_parser.md | 430 ++++++++++++++++++++++++++++ lkml2cube/converter.py | 313 ++++++++++++++++++++ lkml2cube/main.py | 117 +++++++- lkml2cube/parser/cube_api.py | 106 ++++++- lkml2cube/parser/explores.py | 131 +++++++++ lkml2cube/parser/loader.py | 213 +++++++++++++- lkml2cube/parser/types.py | 65 +++++ lkml2cube/parser/views.py | 31 ++ pdm.lock | 433 +++++++++++++++++++++++++++- pyproject.toml | 1 + requirements.txt | 1 + scripts/generate_docs.py | 217 ++++++++++++++ tests/test_converter.py | 535 +++++++++++++++++++++++++++++++++++ 18 files changed, 3096 insertions(+), 42 deletions(-) create mode 100644 .pydoc-markdown.yml create mode 100644 docs/lkml2cube_converter.md create mode 100644 docs/lkml2cube_main.md create mode 100644 docs/lkml2cube_parser.md create mode 100644 lkml2cube/converter.py create mode 100644 scripts/generate_docs.py create mode 100644 tests/test_converter.py diff --git a/.pydoc-markdown.yml b/.pydoc-markdown.yml new file mode 100644 index 0000000..6a2c8bf --- /dev/null +++ b/.pydoc-markdown.yml @@ -0,0 +1,9 @@ + + loaders: + - type: python + search_path: [lkml2cube] + renderer: + type: markdown + render_toc: true + render_module_header: true + \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index ffc7866..2791f6a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,10 +2,27 @@ This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +> **🚨 IMPORTANT**: Always check `docs/lkml2cube_*.md` files BEFORE reading source code. Update docstrings and run `python scripts/generate_docs.py` after any code changes. + ## Project Overview lkml2cube is a Python CLI tool that converts LookML models into Cube data models. It uses the `lkml` library to parse LookML files and generates YAML-based Cube definitions. +## 📚 Documentation-First Development + +**CRITICAL**: Before reading source code, always consult the generated documentation: + +1. **Primary Reference**: Check `docs/lkml2cube_*.md` files first +2. **Implementation Details**: Read source code only if docs are insufficient +3. **Always Update**: Maintain docstrings and regenerate docs for any changes + +📖 **Documentation Files**: +- `docs/lkml2cube_main.md` - CLI commands and usage +- `docs/lkml2cube_converter.md` - LookMLConverter Python API +- `docs/lkml2cube_parser.md` - Parser module functions and classes + +⚠️ **Required Workflow**: When modifying code → Update docstrings → Run `python scripts/generate_docs.py` + ## Development Commands ### Environment Setup @@ -15,9 +32,17 @@ lkml2cube is a Python CLI tool that converts LookML models into Cube data models ### Testing - Tests are located in `tests/` directory -- Main test file: `tests/test_e2e.py` +- Main test files: `tests/test_e2e.py`, `tests/test_converter.py`, `tests/test_explores_command.py` - Test samples are in `tests/samples/` with both `lkml/` and `cubeml/` subdirectories - Tests compare generated output against expected YAML files +- `test_converter.py` provides comprehensive unit tests for the `LookMLConverter` class + +### Documentation Generation +- **Generate docs**: `python scripts/generate_docs.py` +- **MANDATORY**: Run after any function/method changes +- **Output**: Updates `docs/lkml2cube_*.md` files +- **Fallback**: Uses manual generation if pydoc-markdown fails +- **Optimization**: Optimizes output for LLM consumption ### CLI Usage The tool provides three main commands: @@ -36,8 +61,9 @@ The tool provides three main commands: - `cube_api.py` - Interfaces with Cube meta API, correctly separates cubes vs views - `types.py` - Custom YAML types for proper formatting -#### Main Entry Point +#### Main Entry Points - `main.py` - Typer-based CLI with three commands: cubes, views, explores +- `converter.py` - Python API class `LookMLConverter` for programmatic usage - Uses Rich for console output formatting ### Key Concepts @@ -67,4 +93,185 @@ Common options across commands: - `--parseonly` - Shows parsed LookML as Python dict - `--printonly` - Prints generated YAML to stdout - `--outputdir` - Directory for output files -- `--rootdir` - Base path for resolving includes \ No newline at end of file +- `--rootdir` - Base path for resolving includes + +## Python API Usage + +### LookMLConverter Class +The `LookMLConverter` class provides a Python API for programmatic usage without requiring CLI interaction: + +```python +from lkml2cube.converter import LookMLConverter + +# Initialize with configuration +converter = LookMLConverter( + outputdir="/tmp/output", + rootdir="/lookml/models", + parseonly=False, + printonly=False, + use_explores_name=False +) + +# Convert LookML views to Cube definitions +result = converter.cubes("models/orders.lkml") + +# Convert LookML explores to Cube definitions with views +result = converter.views("models/explores.lkml") + +# Generate LookML explores from Cube meta API +result = converter.explores("https://api.cube.dev/v1/meta", "jwt-token") +``` + +#### Key Methods +- `cubes(file_path)` - Equivalent to `lkml2cube cubes` command +- `views(file_path)` - Equivalent to `lkml2cube views` command +- `explores(metaurl, token)` - Equivalent to `lkml2cube explores` command +- `set_config(**kwargs)` - Update configuration options +- `get_config()` - Get current configuration +- `validate_files(file_paths)` - Validate that files can be loaded + +#### Configuration Management +The converter maintains state and can be reconfigured: + +```python +# Update configuration +converter.set_config(parseonly=True, outputdir="/new/path") + +# Get current configuration +config = converter.get_config() +``` + +#### Return Values +All methods return a dictionary with relevant data: +- `parseonly=True`: Returns parsed model structure +- `printonly=True`: Returns YAML output string +- Default: Returns file generation summary + +## Documentation and Code Guidelines + +### Documentation-First Approach + +**IMPORTANT**: Always refer to the generated documentation in the `docs/` directory before reading source code: + +1. **First**: Check `docs/lkml2cube_*.md` files for API documentation +2. **Second**: If implementation details are needed, then read the source code +3. **Always**: Maintain and update documentation when making changes + +### Documentation Files + +The project maintains auto-generated documentation: +- `docs/lkml2cube_main.md` - CLI commands and usage +- `docs/lkml2cube_converter.md` - LookMLConverter Python API +- `docs/lkml2cube_parser.md` - Parser module functions and classes + +### Documentation Maintenance Workflow + +**MANDATORY**: When adding new functions or modifying existing ones: + +1. **Add/Update Google-style Docstrings**: + ```python + def my_function(param1: str, param2: int = 5) -> dict: + """Brief one-line description. + + Detailed description if needed. + + Args: + param1 (str): Description of param1. + param2 (int, optional): Description of param2. Defaults to 5. + + Returns: + dict: Description of return value. + + Raises: + ValueError: Description of when this is raised. + + Example: + >>> result = my_function("test", 10) + >>> print(result['key']) + 'value' + """ + ``` + +2. **Run Documentation Generation**: + ```bash + python scripts/generate_docs.py + ``` + +3. **Verify Documentation**: + - Check that `docs/` files are updated + - Ensure docstrings are properly formatted + - Verify examples work correctly + +### Google-style Docstring Requirements + +All public functions, methods, and classes MUST have Google-style docstrings including: +- **Clear one-line description** +- **Complete parameter documentation with types** +- **Return value descriptions** +- **Raised exceptions** +- **Simple usage examples** + +### Documentation Generation Script + +The `scripts/generate_docs.py` script: +- Automatically extracts docstrings from source code +- Generates markdown files in `docs/` directory +- Uses pydoc-markdown with manual fallback +- Optimizes output for LLM consumption +- Must be run after any function signature changes + +### When to Update Documentation + +Run `python scripts/generate_docs.py` when: +- Adding new functions or methods +- Modifying function signatures +- Changing parameter types or defaults +- Adding or removing classes +- Updating docstrings for clarity + +### Code Review Checklist + +Before committing changes: +- [ ] All new functions have Google-style docstrings +- [ ] Documentation generation script has been run +- [ ] Generated docs reflect the changes +- [ ] Examples in docstrings are accurate +- [ ] Parameter types and descriptions are correct + +## 🔒 Enforcement Rules + +**MANDATORY REQUIREMENTS**: + +1. **Documentation First**: NEVER read source code without first checking the generated documentation in `docs/` +2. **Google-style Docstrings**: ALL public functions, methods, and classes MUST have complete Google-style docstrings +3. **Documentation Generation**: ALWAYS run `python scripts/generate_docs.py` after any code changes +4. **No Exceptions**: These rules apply to ALL code changes, no matter how small + +**VIOLATION CONSEQUENCES**: +- Code changes without proper docstrings will be rejected +- Failure to generate documentation will result in incomplete assistance +- Not following documentation-first approach will lead to suboptimal code understanding + +**COMPLIANCE VERIFICATION**: +- Check that `docs/` files are updated after changes +- Verify docstrings follow Google-style format exactly +- Ensure examples in docstrings are working and accurate +- Confirm all parameters and return values are documented + +--- + +## 📝 Quick Reference + +**Documentation Workflow**: +1. 📖 Check `docs/lkml2cube_*.md` first +2. 📝 Add/update Google-style docstrings +3. 🔄 Run `python scripts/generate_docs.py` +4. ✅ Verify documentation is updated + +**Key Files**: +- `docs/lkml2cube_main.md` - CLI documentation +- `docs/lkml2cube_converter.md` - Python API documentation +- `docs/lkml2cube_parser.md` - Parser module documentation +- `scripts/generate_docs.py` - Documentation generation script + +**Remember**: Documentation first, code second. Always maintain docstrings! \ No newline at end of file diff --git a/README.md b/README.md index 2745ff5..e060a8f 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,19 @@ A comprehensive tool for bidirectional conversion between LookML and Cube data m pip install lkml2cube ``` -## Commands +## Usage + +lkml2cube can be used both as a command-line tool and as a Python library: + +### Command Line Interface + +Use the CLI commands for quick conversions and automation: + +#### Commands lkml2cube provides three main commands for different conversion scenarios: -### 1. `cubes` - LookML Views → Cube Models +##### 1. `cubes` - LookML Views → Cube Models Converts LookML view files into Cube YAML definitions (cubes only). @@ -35,7 +43,7 @@ lkml2cube cubes --parseonly path/to/orders.view.lkml lkml2cube cubes views/orders.view.lkml --outputdir models/ --rootdir ../my_project/ ``` -### 2. `views` - LookML Explores → Cube Models +##### 2. `views` - LookML Explores → Cube Models Converts LookML explore files into Cube YAML definitions (cubes + views with joins). @@ -47,7 +55,7 @@ lkml2cube views path/to/sales_analysis.explore.lkml --outputdir examples/ lkml2cube views --printonly path/to/sales_analysis.explore.lkml ``` -### 3. `explores` - Cube Meta API → LookML ✨ **NEW** +##### 3. `explores` - Cube Meta API → LookML ✨ **NEW** Generates production-ready LookML files from Cube's meta API endpoint. @@ -68,6 +76,79 @@ lkml2cube explores "https://your-cube.com/cubejs-api/v1/meta" \ --printonly ``` +### Python API + +For programmatic usage, import and use the `LookMLConverter` class: + +```python +from lkml2cube.converter import LookMLConverter + +# Initialize converter with options +converter = LookMLConverter( + outputdir="./output", + rootdir="./models", + parseonly=False, + printonly=False, + use_explores_name=False +) + +# Convert LookML views to Cube definitions +result = converter.cubes("path/to/orders.view.lkml") +print(f"Generated {len(result['summary']['cubes'])} cube files") + +# Convert LookML explores to Cube definitions with views +result = converter.views("path/to/explores.lkml") +print(f"Generated {len(result['summary']['views'])} view files") + +# Generate LookML from Cube API +result = converter.explores("https://api.cube.dev/v1/meta", "jwt-token") +print(f"Generated {len(result['summary']['views'])} LookML views") +``` + +#### Configuration Management + +The converter maintains state and can be reconfigured: + +```python +# Update configuration +converter.set_config(parseonly=True, outputdir="/tmp/new-output") + +# Get current configuration +config = converter.get_config() +print(f"Current output directory: {config['outputdir']}") + +# Validate files before processing +file_paths = ["model1.lkml", "model2.lkml"] +validation_results = converter.validate_files(file_paths) +valid_files = [f for f, valid in validation_results.items() if valid] +``` + +#### Return Values + +All conversion methods return a dictionary with: + +- **parseonly=True**: `{'lookml_model': dict, 'parsed_model': str}` +- **printonly=True**: `{'lookml_model': dict, 'cube_def': dict, 'yaml_output': str}` +- **Default**: `{'lookml_model': dict, 'cube_def': dict, 'summary': dict}` + +The `summary` contains details about generated files: + +```python +{ + 'cubes': [{'name': 'orders', 'path': '/output/cubes/orders.yml'}], + 'views': [{'name': 'orders_view', 'path': '/output/views/orders_view.yml'}] +} +``` + +#### Why Use the Python API? + +- **State Management**: Maintain configuration across multiple conversions +- **Programmatic Control**: Integrate conversions into data pipelines +- **Validation**: Check file validity before processing +- **Error Handling**: Catch and handle conversion errors gracefully +- **Batch Processing**: Process multiple files efficiently +- **Custom Workflows**: Build complex conversion workflows + ## What Gets Generated ### From Cube Cubes → LookML Views @@ -140,6 +221,7 @@ explore order_analysis { The tool automatically handles LookML `include` statements and can resolve relative paths: +**CLI:** ```sh # Use --rootdir to resolve include paths lkml2cube views explores/sales.explore.lkml \ @@ -147,10 +229,21 @@ lkml2cube views explores/sales.explore.lkml \ --rootdir /path/to/lookml/project/ ``` +**Python API:** +```python +# Set rootdir for include resolution +converter = LookMLConverter( + rootdir="/path/to/lookml/project/", + outputdir="output/" +) +result = converter.views("explores/sales.explore.lkml") +``` + ### Authentication for Cube API The `explores` command requires a valid JWT token for Cube authentication: +**CLI:** ```sh # Get your token from Cube's authentication export CUBE_TOKEN="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9..." @@ -160,6 +253,83 @@ lkml2cube explores "https://your-cube.com/cubejs-api/v1/meta" \ --outputdir looker_models/ ``` +**Python API:** +```python +# Use environment variables or pass token directly +import os +converter = LookMLConverter(outputdir="looker_models/") +result = converter.explores( + "https://your-cube.com/cubejs-api/v1/meta", + os.getenv("CUBE_TOKEN") +) +``` + +### Batch Processing + +The Python API makes it easy to process multiple files: + +```python +from lkml2cube.converter import LookMLConverter +from pathlib import Path + +converter = LookMLConverter(outputdir="output/") + +# Process all LookML files in a directory +lookml_dir = Path("models/") +for lkml_file in lookml_dir.glob("*.lkml"): + try: + print(f"Processing {lkml_file}...") + result = converter.cubes(str(lkml_file)) + print(f" ✓ Generated {len(result['summary']['cubes'])} cubes") + except Exception as e: + print(f" ✗ Error processing {lkml_file}: {e}") + +# Validate files before processing +file_paths = [str(f) for f in lookml_dir.glob("*.lkml")] +validation_results = converter.validate_files(file_paths) +valid_files = [f for f, valid in validation_results.items() if valid] +print(f"Found {len(valid_files)} valid LookML files") +``` + +### Pipeline Integration + +Integrate lkml2cube into your data pipeline: + +```python +from lkml2cube.converter import LookMLConverter +import logging + +def sync_cube_to_lookml(cube_api_url: str, token: str, output_dir: str): + """Sync Cube models to LookML files.""" + converter = LookMLConverter(outputdir=output_dir) + + try: + # Generate LookML from Cube API + result = converter.explores(cube_api_url, token) + + # Log results + views_count = len(result['summary']['views']) + explores_count = len(result['summary']['explores']) + + logging.info(f"Generated {views_count} LookML views") + logging.info(f"Generated {explores_count} LookML explores") + + return result['summary'] + + except Exception as e: + logging.error(f"Failed to sync Cube to LookML: {e}") + raise + +# Use in your pipeline +if __name__ == "__main__": + summary = sync_cube_to_lookml( + "https://your-cube.com/cubejs-api/v1/meta", + "your-jwt-token", + "looker_models/" + ) + print(f"Sync complete: {summary}") +``` + ## Output Structure The tool creates organized directory structures: diff --git a/docs/lkml2cube_converter.md b/docs/lkml2cube_converter.md new file mode 100644 index 0000000..d942955 --- /dev/null +++ b/docs/lkml2cube_converter.md @@ -0,0 +1,32 @@ +# lkml2cube.converter + +Main converter class for lkml2cube providing a Python API for LookML to Cube conversion. + +This module provides a high-level interface for converting LookML models to Cube definitions +without requiring CLI usage. It maintains configuration state and provides the same +functionality as the CLI commands. + +#### Classes + +##### LookMLConverter +Main converter class for LookML to Cube conversion operations. + + This class provides a Python API for converting LookML models to Cube definitions, + maintaining configuration state and providing the same functionality as the CLI commands. + + Attributes: + outputdir (str): Directory where output files will be written. + rootdir (str | None): Root directory for resolving LookML includes. + parseonly (bool): If True, only parse and return Python dict representation. + printonly (bool): If True, print YAML output to stdout instead of writing files. + use_explores_name (bool): Whether to use explore names for cube view names. + +**Methods:** + +- `__init__()`: Initialize the LookML converter with configuration options +- `cubes()`: Generate cube definitions from LookML views +- `explores()`: Generate LookML explores from Cube meta API +- `get_config()`: Get current converter configuration +- `set_config()`: Update converter configuration options +- `validate_files()`: Validate that LookML files exist and can be loaded +- `views()`: Generate cube definitions with views from LookML explores diff --git a/docs/lkml2cube_main.md b/docs/lkml2cube_main.md new file mode 100644 index 0000000..08ae850 --- /dev/null +++ b/docs/lkml2cube_main.md @@ -0,0 +1,113 @@ +# lkml2cube.main + +Main CLI module for lkml2cube - LookML to Cube bidirectional converter. + +This module provides the command-line interface for lkml2cube, offering three main commands: +- cubes: Convert LookML views to Cube definitions +- views: Convert LookML explores to Cube definitions with views +- explores: Generate LookML explores from Cube meta API + +The CLI is built using Typer and provides rich console output with proper error handling. +Each command supports various options for parsing, output formatting, and file generation. + +#### Functions + +##### callback() +Main callback function for the lkml2cube CLI application. + + serves as the entry point for the CLI and provides + general information about the tool. It sets up the global context + for all subcommands. + + Note: + is called automatically by Typer when the CLI is invoked. + It doesn't perform any specific actions but serves as a placeholder + for global CLI configuration. + + Example: + $ lkml2cube --help + # Shows help information for the entire CLI + +##### cubes() +Generate Cube model definitions from LookML view files. + + Converts LookML view files into Cube YAML definitions, handling dimensions, + measures, and basic join relationships. This command focuses on generating + cube definitions only (no views). + + Args: + file_path (str): Path to the LookML file to process (supports glob patterns). + parseonly (bool, optional): If True, only displays the parsed LookML as Python dict. Defaults to False. + outputdir (str, optional): Directory where output files will be written. Defaults to ".". + printonly (bool, optional): If True, prints YAML to stdout instead of writing files. Defaults to False. + rootdir (str | None, optional): Root directory for resolving LookML includes. Defaults to None. + + Raises: + typer.Exit: If no files are found at the specified path. + + Example: + $ lkml2cube cubes models/orders.view.lkml --outputdir output/ + # Generates cube definitions in output/cubes/ + + $ lkml2cube cubes models/orders.view.lkml --parseonly + # Shows parsed LookML structure + + $ lkml2cube cubes models/orders.view.lkml --printonly + # Prints YAML to console + +##### explores() +Generate LookML explores and views from Cube meta API. + + Fetches Cube model definitions from the meta API and converts them to + production-ready LookML files. This command correctly maps: + - Cube cubes (with sql_table/sql) → LookML views + - Cube views (with aliasMember joins) → LookML explores + + Args: + metaurl (str): URL to the Cube meta API endpoint (e.g., https://api.cube.dev/v1/meta). + token (str): JWT authentication token for the Cube meta API. + parseonly (bool, optional): If True, only displays the parsed Cube model as Python dict. Defaults to False. + outputdir (str, optional): Directory where output files will be written. Defaults to ".". + printonly (bool, optional): If True, prints YAML to stdout instead of writing files. Defaults to False. + + Raises: + typer.Exit: If no response is received from the meta API. + ValueError: If the token is invalid or API request fails. + + Example: + $ lkml2cube explores "https://api.cube.dev/v1/meta" --token "jwt-token" --outputdir lookml/ + # Generates LookML views and explores in lookml/views/ and lookml/explores/ + + $ lkml2cube explores "https://api.cube.dev/v1/meta" --token "jwt-token" --parseonly + # Shows parsed Cube model structure + + $ lkml2cube explores "https://api.cube.dev/v1/meta" --token "jwt-token" --printonly + # Prints generated LookML to console + +##### views() +Generate Cube model definitions with views from LookML explore files. + + Converts LookML explore files into Cube YAML definitions, creating both + cube definitions and view definitions with join relationships. This command + generates a complete Cube model with views that define how cubes relate to each other. + + Args: + file_path (str): Path to the LookML explore file to process (supports glob patterns). + parseonly (bool, optional): If True, only displays the parsed LookML as Python dict. Defaults to False. + outputdir (str, optional): Directory where output files will be written. Defaults to ".". + printonly (bool, optional): If True, prints YAML to stdout instead of writing files. Defaults to False. + rootdir (str | None, optional): Root directory for resolving LookML includes. Defaults to None. + use_explores_name (bool, optional): If True, uses explore names for cube view names. Defaults to False. + + Raises: + typer.Exit: If no files are found at the specified path. + + Example: + $ lkml2cube views models/explores.lkml --outputdir output/ + # Generates cubes and views in output/cubes/ and output/views/ + + $ lkml2cube views models/explores.lkml --use-explores-name + # Uses explore names for view naming + + $ lkml2cube views models/explores.lkml --parseonly + # Shows parsed LookML structure diff --git a/docs/lkml2cube_parser.md b/docs/lkml2cube_parser.md new file mode 100644 index 0000000..4002fac --- /dev/null +++ b/docs/lkml2cube_parser.md @@ -0,0 +1,430 @@ +# lkml2cube.parser + +## Submodules + +### lkml2cube.parser.cube_api + +#### Functions + +##### meta_loader() +Load the Cube meta API and return the model as a dictionary. + + Args: + meta_url (str): URL to the Cube meta API endpoint. + token (str): Authentication token for the API. + + Returns: + dict: Cube model data from the meta API. + + Raises: + ValueError: If no valid token is provided. + Exception: If the API request fails or returns non-200 status. + + Example: + >>> model = meta_loader('https://api.cube.dev/v1/meta', 'my-token') + >>> print(model['cubes'][0]['name']) + 'orders' + +##### parse_members() +Parse measures and dimensions from the Cube meta model. + + Args: + members (list): List of dimension or measure definitions from Cube meta. + + Returns: + list: List of parsed members in LookML format. + + Example: + >>> members = [{'name': 'total_sales', 'type': 'sum', 'sql': 'amount'}] + >>> parsed = parse_members(members) + >>> print(parsed[0]['name']) + 'total_sales' + +##### parse_meta() +Parse the Cube meta model and return a simplified version. + + Separates Cube cubes (-> LookML views) from Cube views (-> LookML explores). + + Args: + cube_model (dict): Complete Cube model from meta API. + + Returns: + dict: LookML model with structure: + {'views': list, 'explores': list} + + Example: + >>> cube_model = {'cubes': [{'name': 'orders', 'sql_table': 'orders'}]} + >>> lookml_model = parse_meta(cube_model) + >>> print(lookml_model['views'][0]['name']) + 'orders' + + +### lkml2cube.parser.explores + +#### Functions + +##### build_cube_name_look_up() +Build a lookup dictionary for cube names in the cube definition. + + Args: + cube_def (dict): Cube definition containing 'cubes' list. + + Note: + modifies the cube_def dictionary in place by adding + a 'cube_name_look_up' key if it doesn't exist. + + Example: + >>> cube_def = {'cubes': [{'name': 'orders'}, {'name': 'customers'}]} + >>> build_cube_name_look_up(cube_def) + >>> print('orders' in cube_def['cube_name_look_up']) + True + +##### generate_cube_joins() +Generate cube join definitions from LookML explores. + + Args: + cube_def (dict): Existing cube definition to modify. + lookml_model (dict): LookML model containing explores with joins. + + Returns: + dict: Updated cube definition with join information added to cubes. + + Raises: + Exception: If cube referenced in explores is not found. + + Example: + >>> cube_def = {'cubes': [{'name': 'orders'}, {'name': 'customers'}]} + >>> lookml_model = {'explores': [{'joins': [{'name': 'customers', 'sql_on': '${orders.customer_id} = ${customers.id}', 'relationship': 'many_to_one'}]}]} + >>> updated_def = generate_cube_joins(cube_def, lookml_model) + >>> print(updated_def['cubes'][1]['joins'][0]['name']) + 'orders' + +##### generate_cube_views() +Generate Cube view definitions from LookML explores. + + Args: + cube_def (dict): Cube definition to add views to. + lookml_model (dict): LookML model containing explores. + use_explores_name (bool, optional): Whether to use explore names as view names. Defaults to False. + + Returns: + dict: Updated cube definition with view definitions added. + + Example: + >>> cube_def = {'cubes': [{'name': 'orders'}]} + >>> lookml_model = {'explores': [{'name': 'orders_explore', 'label': 'Orders Analysis'}]} + >>> updated_def = generate_cube_views(cube_def, lookml_model) + >>> print(updated_def['views'][0]['name']) + 'orders_analysis' + +##### get_cube_from_cube_def() +Get a cube definition by name from the cube definition. + + Args: + cube_def (dict): Cube definition containing 'cubes' list. + cube_name (str): Name of the cube to retrieve. + + Returns: + dict | None: Cube definition if found, None otherwise. + + Example: + >>> cube_def = {'cubes': [{'name': 'orders', 'sql_table': 'orders'}]} + >>> cube = get_cube_from_cube_def(cube_def, 'orders') + >>> print(cube['sql_table']) + 'orders' + +##### get_cube_names_from_join_condition() +Extract cube names from a join condition SQL string. + + Args: + join_condition (str): SQL join condition containing cube references. + + Returns: + list[str]: List of cube names found in the join condition. + + Example: + >>> join_condition = '${orders.customer_id} = ${customers.id}' + >>> get_cube_names_from_join_condition(join_condition) + ['orders', 'customers'] + +##### parse_explores() +Parse LookML explores into Cube definitions with joins and views. + + Args: + lookml_model (dict): LookML model containing views and explores. + use_explores_name (bool, optional): Whether to use explore names as view names. Defaults to False. + + Returns: + dict: Complete cube definition with cubes, joins, and views. + + Raises: + Exception: If no explores are found in the LookML model. + + Example: + >>> lookml_model = { + ... 'views': [{'name': 'orders', 'sql_table_name': 'orders'}], + ... 'explores': [{'name': 'orders_explore', 'joins': [{'name': 'customers', 'sql_on': '${orders.customer_id} = ${customers.id}', 'relationship': 'many_to_one'}]}] + ... } + >>> cube_def = parse_explores(lookml_model) + >>> print(len(cube_def['cubes'])) + 1 + >>> print(len(cube_def['views'])) + 1 + +##### snakify() +Convert a string to snake_case format. + + Args: + s (str): String to convert to snake_case. + + Returns: + str: Snake_case version of the input string. + + Example: + >>> snakify('MyViewName') + 'my_view_name' + >>> snakify('Order-Details') + 'order_details' + +##### traverse_graph() +Find the shortest path between two cubes using BFS traversal. + + Args: + join_paths (dict): Dictionary mapping cube names to their connected cubes. + cube_left (str): Starting cube name. + cube_right (str): Target cube name. + + Returns: + str: Dot-separated path from cube_left to cube_right. + + Example: + >>> join_paths = {'orders': ['customers'], 'customers': ['addresses']} + >>> traverse_graph(join_paths, 'orders', 'addresses') + 'orders.customers.addresses' + + +### lkml2cube.parser.loader + +#### Functions + +##### file_loader() +Load LookML files and resolve includes recursively. + + Args: + file_path_input (str): File path pattern to load (supports glob patterns). + rootdir_param (str | None): Root directory for resolving includes. + namespace (dict | None): Existing namespace to merge content into. + + Returns: + dict: Loaded LookML model with resolved includes. + + Raises: + FileNotFoundError: If specified file path cannot be found. + ValueError: If LookML file cannot be parsed. + + Example: + >>> namespace = file_loader('models/*.lkml', '/path/to/root') + >>> print(namespace['views'][0]['name']) + 'my_view' + +##### print_summary() +Print a formatted summary of generated files using Rich tables. + + Args: + summary (dict): Summary dictionary containing file information with keys + 'cubes', 'views', and/or 'explores', each containing lists of + {'name': str, 'path': str} dictionaries. + + Example: + >>> summary = {'cubes': [{'name': 'orders', 'path': '/output/cubes/orders.yml'}]} + >>> print_summary(summary) + # Displays a formatted table showing the generated files + +##### update_namespace() +Update namespace with new file content, merging lists and handling conflicts. + + Args: + namespace (dict | None): Existing namespace dictionary or None. + new_file (dict): New file content to merge into namespace. + + Returns: + dict: Updated namespace with merged content. + + Example: + >>> namespace = {'views': [{'name': 'view1'}]} + >>> new_file = {'views': [{'name': 'view2'}]} + >>> update_namespace(namespace, new_file) + {'views': [{'name': 'view1'}, {'name': 'view2'}]} + +##### write_files() +Write cube definitions to separate files organized by type. + + Args: + cube_def (dict): Cube definitions containing 'cubes' and/or 'views' keys. + outputdir (str): Output directory path. + + Returns: + dict: Summary of written files with structure: + {'cubes': [{'name': str, 'path': str}], 'views': [{'name': str, 'path': str}]} + + Raises: + Exception: If no cube definition is provided. + OSError: If output directory cannot be created or files cannot be written. + + Example: + >>> cube_def = {'cubes': [{'name': 'orders'}], 'views': [{'name': 'orders_view'}]} + >>> summary = write_files(cube_def, '/output') + >>> print(summary['cubes'][0]['name']) + 'orders' + +##### write_lookml_files() +Write LookML model to files in the output directory. + + Args: + lookml_model (dict): LookML model containing 'views' and/or 'explores' keys. + outputdir (str): Output directory path. + + Returns: + dict: Summary of written files with structure: + {'views': [{'name': str, 'path': str}], 'explores': [{'name': str, 'path': str}]} + + Raises: + Exception: If no LookML model is provided. + OSError: If output directory cannot be created or files cannot be written. + + Example: + >>> lookml_model = {'views': [{'name': 'orders'}], 'explores': [{'name': 'orders_explore'}]} + >>> summary = write_lookml_files(lookml_model, '/output') + >>> print(summary['views'][0]['name']) + 'orders' + +##### write_single_file() +Write a single cube definition to a YAML file. + + Args: + cube_def (dict): Cube definition to write. + outputdir (str): Output directory path. + subdir (str, optional): Subdirectory within output directory. Defaults to "cubes". + file_name (str, optional): Name of the output file. Defaults to "my_cubes.yml". + + Raises: + OSError: If output directory cannot be created or file cannot be written. + + Example: + >>> cube_def = {'cubes': [{'name': 'orders', 'sql_table': 'orders'}]} + >>> write_single_file(cube_def, '/output', 'cubes', 'orders.yml') + + +### lkml2cube.parser.types + +#### Classes + +##### Console +Simple console wrapper for printing messages. + + This class provides a basic print interface compatible with Rich console + while falling back to standard print functionality. + +**Methods:** + +- `print()`: Print a message to the console + +##### folded_unicode +String subclass for YAML folded scalar representation. + + This class marks strings that should be represented as folded scalars + in YAML output (using the '>' style). + + Example: + >>> text = folded_unicode("This is a long +string that will be folded") + >>> # When dumped to YAML, will use '>' style + +##### literal_unicode +String subclass for YAML literal scalar representation. + + This class marks strings that should be represented as literal scalars + in YAML output (using the '|' style). + + Example: + >>> sql = literal_unicode("SELECT * +FROM table +WHERE id = 1") + >>> # When dumped to YAML, will use '|' style preserving line breaks + +#### Functions + +##### folded_unicode_representer() +YAML representer for folded_unicode strings. + + Args: + dumper: YAML dumper instance. + data (folded_unicode): String data to represent. + + Returns: + Scalar representation with folded style. + + Example: + >>> import yaml + >>> yaml.add_representer(folded_unicode, folded_unicode_representer) + >>> yaml.dump(folded_unicode("long text")) + '> long text +' + +##### literal_unicode_representer() +YAML representer for literal_unicode strings. + + Args: + dumper: YAML dumper instance. + data (literal_unicode): String data to represent. + + Returns: + Scalar representation with literal style. + + Example: + >>> import yaml + >>> yaml.add_representer(literal_unicode, literal_unicode_representer) + >>> yaml.dump(literal_unicode("SELECT * +FROM table")) + '| + SELECT * + FROM table +' + + +### lkml2cube.parser.views + +#### Functions + +##### parse_view() +Parse LookML views into Cube definitions. + + Converts LookML view definitions into Cube format, handling dimensions, measures, + view inheritance, and various LookML-specific features like tiers and drill fields. + + Args: + lookml_model (dict): LookML model containing views to parse. + raise_when_views_not_present (bool, optional): Whether to raise an exception + when no views are found. Defaults to True. + + Returns: + dict: Cube definitions with structure: + {'cubes': [{'name': str, 'description': str, 'dimensions': list, 'measures': list, 'joins': list}]} + + Raises: + Exception: If raise_when_views_not_present is True and no views are found, + or if required dimension properties are missing. + + Example: + >>> lookml_model = { + ... 'views': [{ + ... 'name': 'orders', + ... 'sql_table_name': 'public.orders', + ... 'dimensions': [{'name': 'id', 'type': 'number', 'sql': '${TABLE}.id'}], + ... 'measures': [{'name': 'count', 'type': 'count'}] + ... }] + ... } + >>> cube_def = parse_view(lookml_model) + >>> print(cube_def['cubes'][0]['name']) + 'orders' + diff --git a/lkml2cube/converter.py b/lkml2cube/converter.py new file mode 100644 index 0000000..51d9362 --- /dev/null +++ b/lkml2cube/converter.py @@ -0,0 +1,313 @@ +""" +Main converter class for lkml2cube providing a Python API for LookML to Cube conversion. + +This module provides a high-level interface for converting LookML models to Cube definitions +without requiring CLI usage. It maintains configuration state and provides the same +functionality as the CLI commands. +""" + +import pprint +import yaml +from typing import Optional, Dict, Any, List + +from lkml2cube.parser.cube_api import meta_loader, parse_meta +from lkml2cube.parser.explores import parse_explores, generate_cube_joins +from lkml2cube.parser.loader import file_loader, write_files, write_lookml_files, print_summary +from lkml2cube.parser.views import parse_view +from lkml2cube.parser.types import ( + folded_unicode, + literal_unicode, + folded_unicode_representer, + literal_unicode_representer, + console, +) + + +class LookMLConverter: + """Main converter class for LookML to Cube conversion operations. + + This class provides a Python API for converting LookML models to Cube definitions, + maintaining configuration state and providing the same functionality as the CLI commands. + + Attributes: + outputdir (str): Directory where output files will be written. + rootdir (str | None): Root directory for resolving LookML includes. + parseonly (bool): If True, only parse and return Python dict representation. + printonly (bool): If True, print YAML output to stdout instead of writing files. + use_explores_name (bool): Whether to use explore names for cube view names. + """ + + def __init__( + self, + outputdir: str = ".", + rootdir: Optional[str] = None, + parseonly: bool = False, + printonly: bool = False, + use_explores_name: bool = False, + ): + """Initialize the LookML converter with configuration options. + + Args: + outputdir (str, optional): Directory where output files will be written. Defaults to ".". + rootdir (str | None, optional): Root directory for resolving LookML includes. Defaults to None. + parseonly (bool, optional): If True, only parse and return Python dict representation. Defaults to False. + printonly (bool, optional): If True, print YAML output to stdout instead of writing files. Defaults to False. + use_explores_name (bool, optional): Whether to use explore names for cube view names. Defaults to False. + + Example: + >>> converter = LookMLConverter(outputdir="/tmp/output", rootdir="/lookml/models") + >>> result = converter.cubes("models/*.lkml") + >>> print(result['summary']['cubes'][0]['name']) + 'orders' + """ + self.outputdir = outputdir + self.rootdir = rootdir + self.parseonly = parseonly + self.printonly = printonly + self.use_explores_name = use_explores_name + + # Configure YAML representers for proper formatting + yaml.add_representer(folded_unicode, folded_unicode_representer) + yaml.add_representer(literal_unicode, literal_unicode_representer) + + def cubes(self, file_path: str) -> Dict[str, Any]: + """Generate cube definitions from LookML views. + + Converts LookML views into Cube cube definitions, handling dimensions, measures, + and basic join relationships. + + Args: + file_path (str): Path to LookML file(s) to process (supports glob patterns). + + Returns: + dict: Result dictionary containing: + - 'lookml_model': Parsed LookML model (if parseonly=True) + - 'cube_def': Generated cube definitions + - 'yaml_output': YAML string representation (if printonly=True) + - 'summary': File generation summary (if files written) + + Raises: + ValueError: If no files are found at the specified path. + + Example: + >>> converter = LookMLConverter() + >>> result = converter.cubes("models/orders.lkml") + >>> print(result['cube_def']['cubes'][0]['name']) + 'orders' + """ + lookml_model = file_loader(file_path, self.rootdir) + + if lookml_model is None: + raise ValueError(f"No files were found on path: {file_path}") + + result = {'lookml_model': lookml_model} + + if self.parseonly: + result['parsed_model'] = pprint.pformat(lookml_model) + return result + + cube_def = parse_view(lookml_model) + cube_def = generate_cube_joins(cube_def, lookml_model) + result['cube_def'] = cube_def + + if self.printonly: + yaml_output = yaml.dump(cube_def, allow_unicode=True) + result['yaml_output'] = yaml_output + console.print(yaml_output) + return result + + summary = write_files(cube_def, outputdir=self.outputdir) + result['summary'] = summary + print_summary(summary) + + return result + + def views(self, file_path: str) -> Dict[str, Any]: + """Generate cube definitions with views from LookML explores. + + Converts LookML explores into Cube definitions including both cubes and views + with join relationships. + + Args: + file_path (str): Path to LookML file(s) to process (supports glob patterns). + + Returns: + dict: Result dictionary containing: + - 'lookml_model': Parsed LookML model (if parseonly=True) + - 'cube_def': Generated cube definitions with views + - 'yaml_output': YAML string representation (if printonly=True) + - 'summary': File generation summary (if files written) + + Raises: + ValueError: If no files are found at the specified path. + + Example: + >>> converter = LookMLConverter(use_explores_name=True) + >>> result = converter.views("models/explores.lkml") + >>> print(len(result['cube_def']['views'])) + 2 + """ + lookml_model = file_loader(file_path, self.rootdir) + + if lookml_model is None: + raise ValueError(f"No files were found on path: {file_path}") + + result = {'lookml_model': lookml_model} + + if self.parseonly: + result['parsed_model'] = pprint.pformat(lookml_model) + return result + + cube_def = parse_explores(lookml_model, self.use_explores_name) + result['cube_def'] = cube_def + + if self.printonly: + yaml_output = yaml.dump(cube_def, allow_unicode=True) + result['yaml_output'] = yaml_output + console.print(yaml_output) + return result + + summary = write_files(cube_def, outputdir=self.outputdir) + result['summary'] = summary + print_summary(summary) + + return result + + def explores(self, metaurl: str, token: str) -> Dict[str, Any]: + """Generate LookML explores from Cube meta API. + + Fetches Cube model from meta API and converts it to LookML explores, + correctly mapping Cube cubes to LookML views and Cube views to LookML explores. + + Args: + metaurl (str): URL to the Cube meta API endpoint. + token (str): JWT token for Cube meta API authentication. + + Returns: + dict: Result dictionary containing: + - 'cube_model': Raw Cube model from meta API (if parseonly=True) + - 'lookml_model': Converted LookML model + - 'yaml_output': YAML string representation (if printonly=True) + - 'summary': File generation summary (if files written) + + Raises: + ValueError: If no response is received from the meta API. + Exception: If API request fails or token is invalid. + + Example: + >>> converter = LookMLConverter(outputdir="/tmp/lookml") + >>> result = converter.explores("https://api.cube.dev/v1/meta", "jwt-token") + >>> print(len(result['lookml_model']['explores'])) + 3 + """ + cube_model = meta_loader(meta_url=metaurl, token=token) + + if cube_model is None: + raise ValueError(f"No response received from: {metaurl}") + + result = {'cube_model': cube_model} + + if self.parseonly: + result['parsed_model'] = pprint.pformat(cube_model) + return result + + lookml_model = parse_meta(cube_model) + result['lookml_model'] = lookml_model + + if self.printonly: + yaml_output = yaml.dump(lookml_model, allow_unicode=True) + result['yaml_output'] = yaml_output + console.print(yaml_output) + return result + + summary = write_lookml_files(lookml_model, outputdir=self.outputdir) + result['summary'] = summary + print_summary(summary) + + return result + + def set_config( + self, + outputdir: Optional[str] = None, + rootdir: Optional[str] = None, + parseonly: Optional[bool] = None, + printonly: Optional[bool] = None, + use_explores_name: Optional[bool] = None, + ) -> None: + """Update converter configuration options. + + Args: + outputdir (str | None, optional): Directory where output files will be written. + rootdir (str | None, optional): Root directory for resolving LookML includes. + parseonly (bool | None, optional): If True, only parse and return Python dict representation. + printonly (bool | None, optional): If True, print YAML output to stdout instead of writing files. + use_explores_name (bool | None, optional): Whether to use explore names for cube view names. + + Example: + >>> converter = LookMLConverter() + >>> converter.set_config(outputdir="/new/path", parseonly=True) + >>> result = converter.cubes("models/*.lkml") + # Will now parse only and use the new output directory + """ + if outputdir is not None: + self.outputdir = outputdir + if rootdir is not None: + self.rootdir = rootdir + if parseonly is not None: + self.parseonly = parseonly + if printonly is not None: + self.printonly = printonly + if use_explores_name is not None: + self.use_explores_name = use_explores_name + + def get_config(self) -> Dict[str, Any]: + """Get current converter configuration. + + Returns: + dict: Current configuration settings. + + Example: + >>> converter = LookMLConverter(outputdir="/tmp") + >>> config = converter.get_config() + >>> print(config['outputdir']) + '/tmp' + """ + return { + 'outputdir': self.outputdir, + 'rootdir': self.rootdir, + 'parseonly': self.parseonly, + 'printonly': self.printonly, + 'use_explores_name': self.use_explores_name, + } + + def validate_files(self, file_paths: List[str]) -> Dict[str, bool]: + """Validate that LookML files exist and can be loaded. + + Args: + file_paths (list[str]): List of file paths to validate. + + Returns: + dict: Dictionary mapping file paths to validation results. + + Example: + >>> converter = LookMLConverter() + >>> results = converter.validate_files(["models/orders.lkml", "models/missing.lkml"]) + >>> print(results["models/orders.lkml"]) + True + """ + results = {} + for file_path in file_paths: + try: + lookml_model = file_loader(file_path, self.rootdir) + results[file_path] = lookml_model is not None + except Exception: + results[file_path] = False + return results + + def __repr__(self) -> str: + """Return string representation of the converter.""" + return ( + f"LookMLConverter(outputdir='{self.outputdir}', " + f"rootdir='{self.rootdir}', parseonly={self.parseonly}, " + f"printonly={self.printonly}, use_explores_name={self.use_explores_name})" + ) \ No newline at end of file diff --git a/lkml2cube/main.py b/lkml2cube/main.py index daa60b2..95e0724 100644 --- a/lkml2cube/main.py +++ b/lkml2cube/main.py @@ -1,3 +1,15 @@ +""" +Main CLI module for lkml2cube - LookML to Cube bidirectional converter. + +This module provides the command-line interface for lkml2cube, offering three main commands: +- cubes: Convert LookML views to Cube definitions +- views: Convert LookML explores to Cube definitions with views +- explores: Generate LookML explores from Cube meta API + +The CLI is built using Typer and provides rich console output with proper error handling. +Each command supports various options for parsing, output formatting, and file generation. +""" + import pprint import rich import typer @@ -23,8 +35,20 @@ @app.callback() def callback(): - """ - lkml2cube is a tool to convert LookML models into Cube data models. + """Main callback function for the lkml2cube CLI application. + + This function serves as the entry point for the CLI and provides + general information about the tool. It sets up the global context + for all subcommands. + + Note: + This function is called automatically by Typer when the CLI is invoked. + It doesn't perform any specific actions but serves as a placeholder + for global CLI configuration. + + Example: + $ lkml2cube --help + # Shows help information for the entire CLI """ # console.print(("lkml2cube is a tool to convert LookML models into Cube data models.\n" # "Use lkml2cube --help to see usage.")) @@ -53,8 +77,31 @@ def cubes( str, typer.Option(help="The path to prepend to include paths") ] = None, ): - """ - Generate cubes-only given a LookML file that contains LookML Views. + """Generate Cube model definitions from LookML view files. + + Converts LookML view files into Cube YAML definitions, handling dimensions, + measures, and basic join relationships. This command focuses on generating + cube definitions only (no views). + + Args: + file_path (str): Path to the LookML file to process (supports glob patterns). + parseonly (bool, optional): If True, only displays the parsed LookML as Python dict. Defaults to False. + outputdir (str, optional): Directory where output files will be written. Defaults to ".". + printonly (bool, optional): If True, prints YAML to stdout instead of writing files. Defaults to False. + rootdir (str | None, optional): Root directory for resolving LookML includes. Defaults to None. + + Raises: + typer.Exit: If no files are found at the specified path. + + Example: + $ lkml2cube cubes models/orders.view.lkml --outputdir output/ + # Generates cube definitions in output/cubes/ + + $ lkml2cube cubes models/orders.view.lkml --parseonly + # Shows parsed LookML structure + + $ lkml2cube cubes models/orders.view.lkml --printonly + # Prints YAML to console """ lookml_model = file_loader(file_path, rootdir) @@ -103,8 +150,32 @@ def views( bool, typer.Option(help="Use explore names for cube view names") ] = False, ): - """ - Generate cubes-only given a LookML file that contains LookML Views. + """Generate Cube model definitions with views from LookML explore files. + + Converts LookML explore files into Cube YAML definitions, creating both + cube definitions and view definitions with join relationships. This command + generates a complete Cube model with views that define how cubes relate to each other. + + Args: + file_path (str): Path to the LookML explore file to process (supports glob patterns). + parseonly (bool, optional): If True, only displays the parsed LookML as Python dict. Defaults to False. + outputdir (str, optional): Directory where output files will be written. Defaults to ".". + printonly (bool, optional): If True, prints YAML to stdout instead of writing files. Defaults to False. + rootdir (str | None, optional): Root directory for resolving LookML includes. Defaults to None. + use_explores_name (bool, optional): If True, uses explore names for cube view names. Defaults to False. + + Raises: + typer.Exit: If no files are found at the specified path. + + Example: + $ lkml2cube views models/explores.lkml --outputdir output/ + # Generates cubes and views in output/cubes/ and output/views/ + + $ lkml2cube views models/explores.lkml --use-explores-name + # Uses explore names for view naming + + $ lkml2cube views models/explores.lkml --parseonly + # Shows parsed LookML structure """ lookml_model = file_loader(file_path, rootdir) @@ -147,8 +218,33 @@ def explores( bool, typer.Option(help="Print to stdout the parsed files") ] = False, ): - """ - Generate cubes-only given a LookML file that contains LookML Views. + """Generate LookML explores and views from Cube meta API. + + Fetches Cube model definitions from the meta API and converts them to + production-ready LookML files. This command correctly maps: + - Cube cubes (with sql_table/sql) → LookML views + - Cube views (with aliasMember joins) → LookML explores + + Args: + metaurl (str): URL to the Cube meta API endpoint (e.g., https://api.cube.dev/v1/meta). + token (str): JWT authentication token for the Cube meta API. + parseonly (bool, optional): If True, only displays the parsed Cube model as Python dict. Defaults to False. + outputdir (str, optional): Directory where output files will be written. Defaults to ".". + printonly (bool, optional): If True, prints YAML to stdout instead of writing files. Defaults to False. + + Raises: + typer.Exit: If no response is received from the meta API. + ValueError: If the token is invalid or API request fails. + + Example: + $ lkml2cube explores "https://api.cube.dev/v1/meta" --token "jwt-token" --outputdir lookml/ + # Generates LookML views and explores in lookml/views/ and lookml/explores/ + + $ lkml2cube explores "https://api.cube.dev/v1/meta" --token "jwt-token" --parseonly + # Shows parsed Cube model structure + + $ lkml2cube explores "https://api.cube.dev/v1/meta" --token "jwt-token" --printonly + # Prints generated LookML to console """ cube_model = meta_loader( @@ -175,4 +271,9 @@ def explores( if __name__ == "__main__": + """Entry point for the CLI application when run as a script. + + This block is executed when the module is run directly (not imported). + It starts the Typer application which handles command parsing and routing. + """ app() diff --git a/lkml2cube/parser/cube_api.py b/lkml2cube/parser/cube_api.py index ed5d87d..732788d 100644 --- a/lkml2cube/parser/cube_api.py +++ b/lkml2cube/parser/cube_api.py @@ -6,8 +6,23 @@ def meta_loader( meta_url: str, token: str, ) -> dict: - """ - Load the Cube meta API and return the model as a dictionary. + """Load the Cube meta API and return the model as a dictionary. + + Args: + meta_url (str): URL to the Cube meta API endpoint. + token (str): Authentication token for the API. + + Returns: + dict: Cube model data from the meta API. + + Raises: + ValueError: If no valid token is provided. + Exception: If the API request fails or returns non-200 status. + + Example: + >>> model = meta_loader('https://api.cube.dev/v1/meta', 'my-token') + >>> print(model['cubes'][0]['name']) + 'orders' """ if not token: @@ -27,8 +42,19 @@ def meta_loader( def parse_members(members: list) -> list: - """ - Parse measures and dimensions from the Cube meta model. + """Parse measures and dimensions from the Cube meta model. + + Args: + members (list): List of dimension or measure definitions from Cube meta. + + Returns: + list: List of parsed members in LookML format. + + Example: + >>> members = [{'name': 'total_sales', 'type': 'sum', 'sql': 'amount'}] + >>> parsed = parse_members(members) + >>> print(parsed[0]['name']) + 'total_sales' """ rpl_table = ( @@ -64,9 +90,22 @@ def parse_members(members: list) -> list: def parse_meta(cube_model: dict) -> dict: - """ - Parse the Cube meta model and return a simplified version. + """Parse the Cube meta model and return a simplified version. + Separates Cube cubes (-> LookML views) from Cube views (-> LookML explores). + + Args: + cube_model (dict): Complete Cube model from meta API. + + Returns: + dict: LookML model with structure: + {'views': list, 'explores': list} + + Example: + >>> cube_model = {'cubes': [{'name': 'orders', 'sql_table': 'orders'}]} + >>> lookml_model = parse_meta(cube_model) + >>> print(lookml_model['views'][0]['name']) + 'orders' """ lookml_model = { @@ -91,9 +130,23 @@ def parse_meta(cube_model: dict) -> dict: def _is_cube_view(model: dict) -> bool: - """ - Determine if a Cube model is a view (has joins) or a cube (has its own data source). + """Determine if a Cube model is a view (has joins) or a cube (has its own data source). + Views typically have aliasMember references and no sql_table/sql property. + + Args: + model (dict): Cube model definition. + + Returns: + bool: True if the model is a view, False if it's a cube. + + Example: + >>> model = {'dimensions': [{'aliasMember': 'orders.id'}]} + >>> _is_cube_view(model) + True + >>> model = {'sql_table': 'orders', 'dimensions': [{'name': 'id'}]} + >>> _is_cube_view(model) + False """ # Check if any dimensions or measures use aliasMember (indicating joins) has_alias_members = False @@ -116,8 +169,21 @@ def _is_cube_view(model: dict) -> bool: def _parse_cube_to_view(model: dict) -> dict: - """ - Parse a Cube cube into a LookML view. + """Parse a Cube cube into a LookML view. + + Args: + model (dict): Cube model definition. + + Returns: + dict: LookML view definition with dimensions, measures, and metadata. + + Example: + >>> model = {'name': 'orders', 'sql_table': 'orders', 'dimensions': [{'name': 'id', 'type': 'string'}]} + >>> view = _parse_cube_to_view(model) + >>> print(view['name']) + 'orders' + >>> print(view['sql_table_name']) + 'orders' """ view = { "name": model.get("name"), @@ -146,8 +212,24 @@ def _parse_cube_to_view(model: dict) -> dict: def _parse_cube_view_to_explore(model: dict) -> dict: - """ - Parse a Cube view into a LookML explore with joins. + """Parse a Cube view into a LookML explore with joins. + + Args: + model (dict): Cube view model definition with aliasMember references. + + Returns: + dict: LookML explore definition with joins based on referenced cubes. + + Example: + >>> model = { + ... 'name': 'orders_analysis', + ... 'dimensions': [{'aliasMember': 'orders.id'}, {'aliasMember': 'customers.name'}] + ... } + >>> explore = _parse_cube_view_to_explore(model) + >>> print(explore['name']) + 'orders_analysis' + >>> print(len(explore['joins'])) + 1 """ explore = { "name": model.get("name"), diff --git a/lkml2cube/parser/explores.py b/lkml2cube/parser/explores.py index 2cc875c..724fa33 100644 --- a/lkml2cube/parser/explores.py +++ b/lkml2cube/parser/explores.py @@ -10,6 +10,20 @@ def snakify(s): + """Convert a string to snake_case format. + + Args: + s (str): String to convert to snake_case. + + Returns: + str: Snake_case version of the input string. + + Example: + >>> snakify('MyViewName') + 'my_view_name' + >>> snakify('Order-Details') + 'order_details' + """ return "_".join( re.sub( "([A-Z][a-z]+)", r" \1", re.sub("([A-Z]+)", r" \1", s.replace("-", " ")) @@ -18,6 +32,21 @@ def snakify(s): def build_cube_name_look_up(cube_def): + """Build a lookup dictionary for cube names in the cube definition. + + Args: + cube_def (dict): Cube definition containing 'cubes' list. + + Note: + This function modifies the cube_def dictionary in place by adding + a 'cube_name_look_up' key if it doesn't exist. + + Example: + >>> cube_def = {'cubes': [{'name': 'orders'}, {'name': 'customers'}]} + >>> build_cube_name_look_up(cube_def) + >>> print('orders' in cube_def['cube_name_look_up']) + True + """ if "cube_name_look_up" in cube_def: return cube_name_look_up = {} @@ -27,6 +56,21 @@ def build_cube_name_look_up(cube_def): def get_cube_from_cube_def(cube_def, cube_name): + """Get a cube definition by name from the cube definition. + + Args: + cube_def (dict): Cube definition containing 'cubes' list. + cube_name (str): Name of the cube to retrieve. + + Returns: + dict | None: Cube definition if found, None otherwise. + + Example: + >>> cube_def = {'cubes': [{'name': 'orders', 'sql_table': 'orders'}]} + >>> cube = get_cube_from_cube_def(cube_def, 'orders') + >>> print(cube['sql_table']) + 'orders' + """ if "cube_name_look_up" not in cube_def: build_cube_name_look_up(cube_def) if cube_name in cube_def["cube_name_look_up"]: @@ -35,10 +79,38 @@ def get_cube_from_cube_def(cube_def, cube_name): def get_cube_names_from_join_condition(join_condition): + """Extract cube names from a join condition SQL string. + + Args: + join_condition (str): SQL join condition containing cube references. + + Returns: + list[str]: List of cube names found in the join condition. + + Example: + >>> join_condition = '${orders.customer_id} = ${customers.id}' + >>> get_cube_names_from_join_condition(join_condition) + ['orders', 'customers'] + """ return [cube.split(".")[0] for cube in re.findall(snake_case, join_condition)] def traverse_graph(join_paths, cube_left, cube_right): + """Find the shortest path between two cubes using BFS traversal. + + Args: + join_paths (dict): Dictionary mapping cube names to their connected cubes. + cube_left (str): Starting cube name. + cube_right (str): Target cube name. + + Returns: + str: Dot-separated path from cube_left to cube_right. + + Example: + >>> join_paths = {'orders': ['customers'], 'customers': ['addresses']} + >>> traverse_graph(join_paths, 'orders', 'addresses') + 'orders.customers.addresses' + """ # Create a queue for BFS queue = [] queue.append([cube_left]) @@ -66,6 +138,25 @@ def traverse_graph(join_paths, cube_left, cube_right): def generate_cube_joins(cube_def, lookml_model): + """Generate cube join definitions from LookML explores. + + Args: + cube_def (dict): Existing cube definition to modify. + lookml_model (dict): LookML model containing explores with joins. + + Returns: + dict: Updated cube definition with join information added to cubes. + + Raises: + Exception: If cube referenced in explores is not found. + + Example: + >>> cube_def = {'cubes': [{'name': 'orders'}, {'name': 'customers'}]} + >>> lookml_model = {'explores': [{'joins': [{'name': 'customers', 'sql_on': '${orders.customer_id} = ${customers.id}', 'relationship': 'many_to_one'}]}]} + >>> updated_def = generate_cube_joins(cube_def, lookml_model) + >>> print(updated_def['cubes'][1]['joins'][0]['name']) + 'orders' + """ if "explores" not in lookml_model or not lookml_model["explores"]: return cube_def for explore in lookml_model["explores"]: @@ -122,6 +213,23 @@ def generate_cube_joins(cube_def, lookml_model): def generate_cube_views(cube_def, lookml_model, use_explores_name=False): + """Generate Cube view definitions from LookML explores. + + Args: + cube_def (dict): Cube definition to add views to. + lookml_model (dict): LookML model containing explores. + use_explores_name (bool, optional): Whether to use explore names as view names. Defaults to False. + + Returns: + dict: Updated cube definition with view definitions added. + + Example: + >>> cube_def = {'cubes': [{'name': 'orders'}]} + >>> lookml_model = {'explores': [{'name': 'orders_explore', 'label': 'Orders Analysis'}]} + >>> updated_def = generate_cube_views(cube_def, lookml_model) + >>> print(updated_def['views'][0]['name']) + 'orders_analysis' + """ if "views" not in cube_def: cube_def["views"] = [] if "explores" not in lookml_model or not lookml_model["explores"]: @@ -184,6 +292,29 @@ def generate_cube_views(cube_def, lookml_model, use_explores_name=False): def parse_explores(lookml_model, use_explores_name=False): + """Parse LookML explores into Cube definitions with joins and views. + + Args: + lookml_model (dict): LookML model containing views and explores. + use_explores_name (bool, optional): Whether to use explore names as view names. Defaults to False. + + Returns: + dict: Complete cube definition with cubes, joins, and views. + + Raises: + Exception: If no explores are found in the LookML model. + + Example: + >>> lookml_model = { + ... 'views': [{'name': 'orders', 'sql_table_name': 'orders'}], + ... 'explores': [{'name': 'orders_explore', 'joins': [{'name': 'customers', 'sql_on': '${orders.customer_id} = ${customers.id}', 'relationship': 'many_to_one'}]}] + ... } + >>> cube_def = parse_explores(lookml_model) + >>> print(len(cube_def['cubes'])) + 1 + >>> print(len(cube_def['views'])) + 1 + """ # First we read all possible lookml views. cube_def = parse_view(lookml_model, raise_when_views_not_present=False) if "explores" not in lookml_model: diff --git a/lkml2cube/parser/loader.py b/lkml2cube/parser/loader.py index 6833581..9b1a8e5 100644 --- a/lkml2cube/parser/loader.py +++ b/lkml2cube/parser/loader.py @@ -14,6 +14,21 @@ def update_namespace(namespace, new_file): + """Update namespace with new file content, merging lists and handling conflicts. + + Args: + namespace (dict | None): Existing namespace dictionary or None. + new_file (dict): New file content to merge into namespace. + + Returns: + dict: Updated namespace with merged content. + + Example: + >>> namespace = {'views': [{'name': 'view1'}]} + >>> new_file = {'views': [{'name': 'view2'}]} + >>> update_namespace(namespace, new_file) + {'views': [{'name': 'view1'}, {'name': 'view2'}]} + """ if namespace is None: return new_file @@ -32,6 +47,25 @@ def update_namespace(namespace, new_file): def file_loader(file_path_input, rootdir_param, namespace=None): + """Load LookML files and resolve includes recursively. + + Args: + file_path_input (str): File path pattern to load (supports glob patterns). + rootdir_param (str | None): Root directory for resolving includes. + namespace (dict | None): Existing namespace to merge content into. + + Returns: + dict: Loaded LookML model with resolved includes. + + Raises: + FileNotFoundError: If specified file path cannot be found. + ValueError: If LookML file cannot be parsed. + + Example: + >>> namespace = file_loader('models/*.lkml', '/path/to/root') + >>> print(namespace['views'][0]['name']) + 'my_view' + """ file_paths = glob.glob(file_path_input) for file_path in file_paths: @@ -65,6 +99,21 @@ def write_single_file( subdir: str = "cubes", file_name: str = "my_cubes.yml", ): + """Write a single cube definition to a YAML file. + + Args: + cube_def (dict): Cube definition to write. + outputdir (str): Output directory path. + subdir (str, optional): Subdirectory within output directory. Defaults to "cubes". + file_name (str, optional): Name of the output file. Defaults to "my_cubes.yml". + + Raises: + OSError: If output directory cannot be created or file cannot be written. + + Example: + >>> cube_def = {'cubes': [{'name': 'orders', 'sql_table': 'orders'}]} + >>> write_single_file(cube_def, '/output', 'cubes', 'orders.yml') + """ f = open(join(outputdir, subdir, file_name), "w") f.write(yaml.dump(cube_def, allow_unicode=True)) @@ -72,6 +121,26 @@ def write_single_file( def write_files(cube_def, outputdir): + """Write cube definitions to separate files organized by type. + + Args: + cube_def (dict): Cube definitions containing 'cubes' and/or 'views' keys. + outputdir (str): Output directory path. + + Returns: + dict: Summary of written files with structure: + {'cubes': [{'name': str, 'path': str}], 'views': [{'name': str, 'path': str}]} + + Raises: + Exception: If no cube definition is provided. + OSError: If output directory cannot be created or files cannot be written. + + Example: + >>> cube_def = {'cubes': [{'name': 'orders'}], 'views': [{'name': 'orders_view'}]} + >>> summary = write_files(cube_def, '/output') + >>> print(summary['cubes'][0]['name']) + 'orders' + """ summary = {"cubes": [], "views": []} @@ -128,8 +197,25 @@ def write_files(cube_def, outputdir): def write_lookml_files(lookml_model, outputdir): - """ - Write LookML model to files in the output directory. + """Write LookML model to files in the output directory. + + Args: + lookml_model (dict): LookML model containing 'views' and/or 'explores' keys. + outputdir (str): Output directory path. + + Returns: + dict: Summary of written files with structure: + {'views': [{'name': str, 'path': str}], 'explores': [{'name': str, 'path': str}]} + + Raises: + Exception: If no LookML model is provided. + OSError: If output directory cannot be created or files cannot be written. + + Example: + >>> lookml_model = {'views': [{'name': 'orders'}], 'explores': [{'name': 'orders_explore'}]} + >>> summary = write_lookml_files(lookml_model, '/output') + >>> print(summary['views'][0]['name']) + 'orders' """ summary = {"views": [], "explores": []} @@ -169,8 +255,23 @@ def write_lookml_files(lookml_model, outputdir): def _generate_lookml_content(element, element_type, includes=None): - """ - Generate LookML content for a view or explore element. + """Generate LookML content for a view or explore element. + + Args: + element (dict): View or explore element definition. + element_type (str): Type of element ('view' or 'explore'). + includes (list[str] | None): List of include statements to add. + + Returns: + str: Generated LookML content as a string. + + Example: + >>> element = {'name': 'orders', 'sql_table_name': 'orders'} + >>> content = _generate_lookml_content(element, 'view') + >>> print(content) + view orders { + sql_table_name: orders ;; + } """ lines = [] name = element.get("name", "unnamed") @@ -247,8 +348,23 @@ def _generate_lookml_content(element, element_type, includes=None): def _generate_dimension_lines(dimension): - """ - Generate LookML lines for a dimension. + """Generate LookML lines for a dimension. + + Args: + dimension (dict): Dimension definition containing name, type, sql, etc. + + Returns: + list[str]: List of LookML lines representing the dimension. + + Example: + >>> dimension = {'name': 'order_id', 'type': 'number', 'sql': '${TABLE}.id'} + >>> lines = _generate_dimension_lines(dimension) + >>> print('\n'.join(lines)) + dimension: order_id { + type: number + primary_key: yes + sql: ${TABLE}.id ;; + } """ lines = [] name = dimension.get("name", "unnamed") @@ -287,8 +403,23 @@ def _generate_dimension_lines(dimension): def _generate_measure_lines(measure): - """ - Generate LookML lines for a measure. + """Generate LookML lines for a measure. + + Args: + measure (dict): Measure definition containing name, type, sql, etc. + + Returns: + list[str]: List of LookML lines representing the measure. + + Example: + >>> measure = {'name': 'total_orders', 'type': 'count', 'sql': '${TABLE}.id'} + >>> lines = _generate_measure_lines(measure) + >>> print('\n'.join(lines)) + measure: total_orders { + type: count + sql: ${TABLE}.id ;; + drill_fields: [id, name] + } """ lines = [] name = measure.get("name", "unnamed") @@ -325,8 +456,22 @@ def _generate_measure_lines(measure): def _generate_filter_lines(filter_def): - """ - Generate LookML lines for a filter. + """Generate LookML lines for a filter. + + Args: + filter_def (dict): Filter definition containing name, type, description, etc. + + Returns: + list[str]: List of LookML lines representing the filter. + + Example: + >>> filter_def = {'name': 'date_filter', 'type': 'date', 'description': 'Filter by date'} + >>> lines = _generate_filter_lines(filter_def) + >>> print('\n'.join(lines)) + filter: date_filter { + description: "Filter by date" + type: date + } """ lines = [] name = filter_def.get("name", "unnamed") @@ -346,8 +491,23 @@ def _generate_filter_lines(filter_def): def _generate_join_lines(join): - """ - Generate LookML lines for a join. + """Generate LookML lines for a join. + + Args: + join (dict): Join definition containing name, type, relationship, sql_on, etc. + + Returns: + list[str]: List of LookML lines representing the join. + + Example: + >>> join = {'name': 'customers', 'type': 'left_outer', 'relationship': 'many_to_one', 'sql_on': '${orders.customer_id} = ${customers.id}'} + >>> lines = _generate_join_lines(join) + >>> print('\n'.join(lines)) + join: customers { + type: left_outer + relationship: many_to_one + sql_on: ${orders.customer_id} = ${customers.id} ;; + } """ lines = [] name = join.get("name", "unnamed") @@ -370,8 +530,21 @@ def _generate_join_lines(join): def _generate_includes_for_explore(explore, lookml_model): - """ - Generate include statements for an explore based on the views it references. + """Generate include statements for an explore based on the views it references. + + Args: + explore (dict): Explore definition containing view_name and joins. + lookml_model (dict): Complete LookML model to check for view existence. + + Returns: + list[str]: List of include file paths for the explore. + + Example: + >>> explore = {'name': 'orders', 'view_name': 'orders', 'joins': [{'name': 'customers'}]} + >>> lookml_model = {'views': [{'name': 'orders'}, {'name': 'customers'}]} + >>> includes = _generate_includes_for_explore(explore, lookml_model) + >>> print(includes) + ['/views/orders.view.lkml', '/views/customers.view.lkml'] """ includes = [] referenced_views = set() @@ -396,6 +569,18 @@ def _generate_includes_for_explore(explore, lookml_model): def print_summary(summary): + """Print a formatted summary of generated files using Rich tables. + + Args: + summary (dict): Summary dictionary containing file information with keys + 'cubes', 'views', and/or 'explores', each containing lists of + {'name': str, 'path': str} dictionaries. + + Example: + >>> summary = {'cubes': [{'name': 'orders', 'path': '/output/cubes/orders.yml'}]} + >>> print_summary(summary) + # Displays a formatted table showing the generated files + """ # Use the proper Rich console for table rendering rich_console = rich.console.Console() diff --git a/lkml2cube/parser/types.py b/lkml2cube/parser/types.py index 913f95b..aa7c9e1 100644 --- a/lkml2cube/parser/types.py +++ b/lkml2cube/parser/types.py @@ -3,7 +3,24 @@ # console = rich.console.Console() class Console: + """Simple console wrapper for printing messages. + + This class provides a basic print interface compatible with Rich console + while falling back to standard print functionality. + """ + def print(self, s, *args): + """Print a message to the console. + + Args: + s (str): Message to print. + *args: Additional arguments (currently ignored). + + Example: + >>> console = Console() + >>> console.print("Hello world", style="bold") + Hello world + """ print(s) @@ -39,16 +56,64 @@ def print(self, s, *args): class folded_unicode(str): + """String subclass for YAML folded scalar representation. + + This class marks strings that should be represented as folded scalars + in YAML output (using the '>' style). + + Example: + >>> text = folded_unicode("This is a long\nstring that will be folded") + >>> # When dumped to YAML, will use '>' style + """ pass class literal_unicode(str): + """String subclass for YAML literal scalar representation. + + This class marks strings that should be represented as literal scalars + in YAML output (using the '|' style). + + Example: + >>> sql = literal_unicode("SELECT *\nFROM table\nWHERE id = 1") + >>> # When dumped to YAML, will use '|' style preserving line breaks + """ pass def folded_unicode_representer(dumper, data): + """YAML representer for folded_unicode strings. + + Args: + dumper: YAML dumper instance. + data (folded_unicode): String data to represent. + + Returns: + Scalar representation with folded style. + + Example: + >>> import yaml + >>> yaml.add_representer(folded_unicode, folded_unicode_representer) + >>> yaml.dump(folded_unicode("long text")) + '> long text\n' + """ return dumper.represent_scalar("tag:yaml.org,2002:str", data, style=">") def literal_unicode_representer(dumper, data): + """YAML representer for literal_unicode strings. + + Args: + dumper: YAML dumper instance. + data (literal_unicode): String data to represent. + + Returns: + Scalar representation with literal style. + + Example: + >>> import yaml + >>> yaml.add_representer(literal_unicode, literal_unicode_representer) + >>> yaml.dump(literal_unicode("SELECT *\nFROM table")) + '|\n SELECT *\n FROM table\n' + """ return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") diff --git a/lkml2cube/parser/views.py b/lkml2cube/parser/views.py index 4882f42..3476310 100644 --- a/lkml2cube/parser/views.py +++ b/lkml2cube/parser/views.py @@ -6,6 +6,37 @@ def parse_view(lookml_model, raise_when_views_not_present=True): + """Parse LookML views into Cube definitions. + + Converts LookML view definitions into Cube format, handling dimensions, measures, + view inheritance, and various LookML-specific features like tiers and drill fields. + + Args: + lookml_model (dict): LookML model containing views to parse. + raise_when_views_not_present (bool, optional): Whether to raise an exception + when no views are found. Defaults to True. + + Returns: + dict: Cube definitions with structure: + {'cubes': [{'name': str, 'description': str, 'dimensions': list, 'measures': list, 'joins': list}]} + + Raises: + Exception: If raise_when_views_not_present is True and no views are found, + or if required dimension properties are missing. + + Example: + >>> lookml_model = { + ... 'views': [{ + ... 'name': 'orders', + ... 'sql_table_name': 'public.orders', + ... 'dimensions': [{'name': 'id', 'type': 'number', 'sql': '${TABLE}.id'}], + ... 'measures': [{'name': 'count', 'type': 'count'}] + ... }] + ... } + >>> cube_def = parse_view(lookml_model) + >>> print(cube_def['cubes'][0]['name']) + 'orders' + """ cubes = [] cube_def = {"cubes": cubes} rpl_table = lambda s: s.replace("${TABLE}", "{CUBE}").replace("${", "{") diff --git a/pdm.lock b/pdm.lock index 5d5ad8e..910b760 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,11 +5,46 @@ groups = ["default", "test"] strategy = ["cross_platform"] lock_version = "4.5.0" -content_hash = "sha256:bcc1a3aab3bab7a4b535dec41ee5a87d23eb296cb705064437db21ca37daad76" +content_hash = "sha256:8ea04e10efb4f16feba07d3681045b11e38fa781e11734a294f979de670883d6" [[metadata.targets]] requires_python = ">=3.10" +[[package]] +name = "black" +version = "25.1.0" +requires_python = ">=3.9" +summary = "The uncompromising code formatter." +dependencies = [ + "click>=8.0.0", + "mypy-extensions>=0.4.3", + "packaging>=22.0", + "pathspec>=0.9.0", + "platformdirs>=2", + "tomli>=1.1.0; python_version < \"3.11\"", + "typing-extensions>=4.0.1; python_version < \"3.11\"", +] +files = [ + {file = "black-25.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:759e7ec1e050a15f89b770cefbf91ebee8917aac5c20483bc2d80a6c3a04df32"}, + {file = "black-25.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e519ecf93120f34243e6b0054db49c00a35f84f195d5bce7e9f5cfc578fc2da"}, + {file = "black-25.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:055e59b198df7ac0b7efca5ad7ff2516bca343276c466be72eb04a3bcc1f82d7"}, + {file = "black-25.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:db8ea9917d6f8fc62abd90d944920d95e73c83a5ee3383493e35d271aca872e9"}, + {file = "black-25.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a39337598244de4bae26475f77dda852ea00a93bd4c728e09eacd827ec929df0"}, + {file = "black-25.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:96c1c7cd856bba8e20094e36e0f948718dc688dba4a9d78c3adde52b9e6c2299"}, + {file = "black-25.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bce2e264d59c91e52d8000d507eb20a9aca4a778731a08cfff7e5ac4a4bb7096"}, + {file = "black-25.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:172b1dbff09f86ce6f4eb8edf9dede08b1fce58ba194c87d7a4f1a5aa2f5b3c2"}, + {file = "black-25.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4b60580e829091e6f9238c848ea6750efed72140b91b048770b64e74fe04908b"}, + {file = "black-25.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1e2978f6df243b155ef5fa7e558a43037c3079093ed5d10fd84c43900f2d8ecc"}, + {file = "black-25.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b48735872ec535027d979e8dcb20bf4f70b5ac75a8ea99f127c106a7d7aba9f"}, + {file = "black-25.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:ea0213189960bda9cf99be5b8c8ce66bb054af5e9e861249cd23471bd7b0b3ba"}, + {file = "black-25.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f0b18a02996a836cc9c9c78e5babec10930862827b1b724ddfe98ccf2f2fe4f"}, + {file = "black-25.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:afebb7098bfbc70037a053b91ae8437c3857482d3a690fefc03e9ff7aa9a5fd3"}, + {file = "black-25.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:030b9759066a4ee5e5aca28c3c77f9c64789cdd4de8ac1df642c40b708be6171"}, + {file = "black-25.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:a22f402b410566e2d1c950708c77ebf5ebd5d0d88a6a2e87c86d9fb48afa0d18"}, + {file = "black-25.1.0-py3-none-any.whl", hash = "sha256:95e8176dae143ba9097f351d174fdaf0ccd29efb414b362ae3fd72bf0f710717"}, + {file = "black-25.1.0.tar.gz", hash = "sha256:33496d5cd1222ad73391352b4ae8da15253c5de89b93a80b3e2c8d9a19ec2666"}, +] + [[package]] name = "certifi" version = "2025.7.14" @@ -105,6 +140,102 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "databind" +version = "4.5.2" +requires_python = "<4.0.0,>=3.8.0" +summary = "Databind is a library inspired by jackson-databind to de-/serialize Python dataclasses. The `databind` package will install the full suite of databind packages. Compatible with Python 3.8 and newer." +dependencies = [ + "Deprecated<2.0.0,>=1.2.12", + "nr-date<3.0.0,>=2.0.0", + "nr-stream<2.0.0,>=1.0.0", + "setuptools>=40.8.0; python_version < \"3.10\"", + "typeapi<3,>=2.0.1", + "typing-extensions<5,>=3.10.0", +] +files = [ + {file = "databind-4.5.2-py3-none-any.whl", hash = "sha256:b9c3a03c0414aa4567f095d7218ac904bd2b267b58e3763dac28e83d64b69770"}, + {file = "databind-4.5.2.tar.gz", hash = "sha256:0a8aa0ff130a0306581c559388f5ef65e0fae7ef4b86412eacb1f4a0420006c4"}, +] + +[[package]] +name = "databind-core" +version = "4.5.2" +requires_python = "<4.0.0,>=3.8.0" +summary = "Databind is a library inspired by jackson-databind to de-/serialize Python dataclasses. Compatible with Python 3.8 and newer. Deprecated, use `databind` package." +dependencies = [ + "databind<5.0.0,>=4.5.2", +] +files = [ + {file = "databind.core-4.5.2-py3-none-any.whl", hash = "sha256:a1dd1c6bd8ca9907d1292d8df9ec763ce91543e27f7eda4268e4a1a84fcd1c42"}, + {file = "databind.core-4.5.2.tar.gz", hash = "sha256:b8ac8127bc5d6b239a2a81aeddb268b0c4cadd53fbce7e8b2c7a9ef6413bccb3"}, +] + +[[package]] +name = "databind-json" +version = "4.5.2" +requires_python = "<4.0.0,>=3.8.0" +summary = "De-/serialize Python dataclasses to or from JSON payloads. Compatible with Python 3.8 and newer. Deprecated, use `databind` module instead." +dependencies = [ + "databind<5.0.0,>=4.5.2", +] +files = [ + {file = "databind.json-4.5.2-py3-none-any.whl", hash = "sha256:a803bf440634685984361cb2a5a975887e487c854ed48d81ff7aaf3a1ed1e94c"}, + {file = "databind.json-4.5.2.tar.gz", hash = "sha256:6cc9b5c6fddaebd49b2433932948eb3be8a41633b90aa37998d7922504b8f165"}, +] + +[[package]] +name = "deprecated" +version = "1.2.18" +requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +summary = "Python @deprecated decorator to deprecate old python classes, functions or methods." +dependencies = [ + "wrapt<2,>=1.10", +] +files = [ + {file = "Deprecated-1.2.18-py2.py3-none-any.whl", hash = "sha256:bd5011788200372a32418f888e326a09ff80d0214bd961147cfed01b5c018eec"}, + {file = "deprecated-1.2.18.tar.gz", hash = "sha256:422b6f6d859da6f2ef57857761bfb392480502a64c3028ca9bbe86085d72115d"}, +] + +[[package]] +name = "docspec" +version = "2.2.1" +requires_python = ">=3.7,<4.0" +summary = "Docspec is a JSON object specification for representing API documentation of programming languages." +dependencies = [ + "Deprecated<2.0.0,>=1.2.12", + "databind-core<5.0.0,>=4.2.6", + "databind-json<5.0.0,>=4.2.6", +] +files = [ + {file = "docspec-2.2.1-py3-none-any.whl", hash = "sha256:7538f750095a9688c6980ff9a4e029a823a500f64bd00b6b4bdb27951feb31cb"}, + {file = "docspec-2.2.1.tar.gz", hash = "sha256:4854e77edc0e2de40e785e57e95880f7095a05fe978f8b54cef7a269586e15ff"}, +] + +[[package]] +name = "docspec-python" +version = "2.2.2" +requires_python = ">=3.8" +summary = "A parser based on lib2to3 producing docspec data from Python source code." +dependencies = [ + "black>=24.8.0", + "docspec==2.2.1", + "nr-util>=0.8.12", +] +files = [ + {file = "docspec_python-2.2.2-py3-none-any.whl", hash = "sha256:caa32dc1e8c470af8a5ecad67cca614e68c1563ac01dab0c0486c4d7f709d6b1"}, + {file = "docspec_python-2.2.2.tar.gz", hash = "sha256:429be834d09549461b95bf45eb53c16859f3dfb3e9220408b3bfb12812ccb3fb"}, +] + +[[package]] +name = "docstring-parser" +version = "0.11" +requires_python = ">=3.6" +summary = "\"Parse Python docstrings in reST, Google and Numpydoc format\"" +files = [ + {file = "docstring_parser-0.11.tar.gz", hash = "sha256:93b3f8f481c7d24e37c5d9f30293c89e2933fa209421c8abd731dd3ef0715ecb"}, +] + [[package]] name = "exceptiongroup" version = "1.3.0" @@ -138,6 +269,19 @@ files = [ {file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"}, ] +[[package]] +name = "jinja2" +version = "3.1.6" +requires_python = ">=3.7" +summary = "A very fast and expressive template engine." +dependencies = [ + "MarkupSafe>=2.0", +] +files = [ + {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, + {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, +] + [[package]] name = "lkml" version = "1.3.7" @@ -160,6 +304,65 @@ files = [ {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, ] +[[package]] +name = "markupsafe" +version = "3.0.2" +requires_python = ">=3.9" +summary = "Safely add untrusted strings to HTML/XML markup." +files = [ + {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-win32.whl", hash = "sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-win32.whl", hash = "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-win32.whl", hash = "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f"}, + {file = "markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0"}, +] + [[package]] name = "mdurl" version = "0.1.2" @@ -170,6 +373,53 @@ files = [ {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, ] +[[package]] +name = "mypy-extensions" +version = "1.1.0" +requires_python = ">=3.8" +summary = "Type system extensions for programs checked with the mypy type checker." +files = [ + {file = "mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505"}, + {file = "mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558"}, +] + +[[package]] +name = "nr-date" +version = "2.1.0" +requires_python = ">=3.6,<4.0" +summary = "" +dependencies = [ + "dataclasses<0.9,>=0.8; python_version == \"3.6\"", +] +files = [ + {file = "nr_date-2.1.0-py3-none-any.whl", hash = "sha256:bd672a9dfbdcf7c4b9289fea6750c42490eaee08036a72059dcc78cb236ed568"}, + {file = "nr_date-2.1.0.tar.gz", hash = "sha256:0643aea13bcdc2a8bc56af9d5e6a89ef244c9744a1ef00cdc735902ba7f7d2e6"}, +] + +[[package]] +name = "nr-stream" +version = "1.1.5" +requires_python = ">=3.6,<4.0" +summary = "" +files = [ + {file = "nr_stream-1.1.5-py3-none-any.whl", hash = "sha256:47e12150b331ad2cb729cfd9d2abd281c9949809729ba461c6aa87dd9927b2d4"}, + {file = "nr_stream-1.1.5.tar.gz", hash = "sha256:eb0216c6bfc61a46d4568dba3b588502c610ec8ddef4ac98f3932a2bd7264f65"}, +] + +[[package]] +name = "nr-util" +version = "0.8.12" +requires_python = ">=3.7,<4.0" +summary = "General purpose Python utility library." +dependencies = [ + "deprecated<2.0.0,>=1.2.0", + "typing-extensions>=3.0.0", +] +files = [ + {file = "nr.util-0.8.12-py3-none-any.whl", hash = "sha256:91da02ac9795eb8e015372275c1efe54bac9051231ee9b0e7e6f96b0b4e7d2bb"}, + {file = "nr.util-0.8.12.tar.gz", hash = "sha256:a4549c2033d99d2f0379b3f3d233fd2a8ade286bbf0b3ad0cc7cea16022214f4"}, +] + [[package]] name = "packaging" version = "25.0" @@ -180,6 +430,26 @@ files = [ {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, ] +[[package]] +name = "pathspec" +version = "0.12.1" +requires_python = ">=3.8" +summary = "Utility library for gitignore style pattern matching of file paths." +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] + +[[package]] +name = "platformdirs" +version = "4.3.8" +requires_python = ">=3.9" +summary = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." +files = [ + {file = "platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4"}, + {file = "platformdirs-4.3.8.tar.gz", hash = "sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc"}, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -190,6 +460,32 @@ files = [ {file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"}, ] +[[package]] +name = "pydoc-markdown" +version = "4.8.2" +requires_python = ">=3.7,<4.0" +summary = "Create Python API documentation in Markdown format." +dependencies = [ + "PyYAML<7.0,>=5.0", + "click<9.0,>=7.1", + "databind-core<5.0.0,>=4.4.0", + "databind-json<5.0.0,>=4.4.0", + "docspec-python<3.0.0,>=2.2.1", + "docspec<3.0.0,>=2.2.1", + "docstring-parser<0.12,>=0.11", + "jinja2<4.0.0,>=3.0.0", + "nr-util<1.0.0,>=0.7.5", + "requests<3.0.0,>=2.23.0", + "tomli-w<2.0.0,>=1.0.0", + "tomli<3.0.0,>=2.0.0", + "watchdog", + "yapf>=0.30.0", +] +files = [ + {file = "pydoc_markdown-4.8.2-py3-none-any.whl", hash = "sha256:203f74119e6bb2f9deba43d452422de7c8ec31955b61e0620fa4dd8c2611715f"}, + {file = "pydoc_markdown-4.8.2.tar.gz", hash = "sha256:fb6c927e31386de17472d42f9bd3d3be2905977d026f6216881c65145aa67f0b"}, +] + [[package]] name = "pygments" version = "2.19.2" @@ -345,6 +641,29 @@ files = [ {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, ] +[[package]] +name = "tomli-w" +version = "1.2.0" +requires_python = ">=3.9" +summary = "A lil' TOML writer" +files = [ + {file = "tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90"}, + {file = "tomli_w-1.2.0.tar.gz", hash = "sha256:2dd14fac5a47c27be9cd4c976af5a12d87fb1f0b4512f81d69cce3b35ae25021"}, +] + +[[package]] +name = "typeapi" +version = "2.2.4" +requires_python = ">=3.8" +summary = "" +dependencies = [ + "typing-extensions>=3.0.0", +] +files = [ + {file = "typeapi-2.2.4-py3-none-any.whl", hash = "sha256:bd6d5e5907fa47e0303bf254e7cc8712d4be4eb26d7ffaedb67c9e7844c53bb8"}, + {file = "typeapi-2.2.4.tar.gz", hash = "sha256:daa80767520c0957a320577e4f729c0ba6921c708def31f4c6fd8d611908fd7b"}, +] + [[package]] name = "typer" version = "0.16.0" @@ -394,3 +713,115 @@ files = [ {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, ] + +[[package]] +name = "watchdog" +version = "6.0.0" +requires_python = ">=3.9" +summary = "Filesystem events monitoring" +files = [ + {file = "watchdog-6.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d1cdb490583ebd691c012b3d6dae011000fe42edb7a82ece80965b42abd61f26"}, + {file = "watchdog-6.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bc64ab3bdb6a04d69d4023b29422170b74681784ffb9463ed4870cf2f3e66112"}, + {file = "watchdog-6.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c897ac1b55c5a1461e16dae288d22bb2e412ba9807df8397a635d88f671d36c3"}, + {file = "watchdog-6.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6eb11feb5a0d452ee41f824e271ca311a09e250441c262ca2fd7ebcf2461a06c"}, + {file = "watchdog-6.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ef810fbf7b781a5a593894e4f439773830bdecb885e6880d957d5b9382a960d2"}, + {file = "watchdog-6.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:afd0fe1b2270917c5e23c2a65ce50c2a4abb63daafb0d419fde368e272a76b7c"}, + {file = "watchdog-6.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdd4e6f14b8b18c334febb9c4425a878a2ac20efd1e0b231978e7b150f92a948"}, + {file = "watchdog-6.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c7c15dda13c4eb00d6fb6fc508b3c0ed88b9d5d374056b239c4ad1611125c860"}, + {file = "watchdog-6.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f10cb2d5902447c7d0da897e2c6768bca89174d0c6e1e30abec5421af97a5b0"}, + {file = "watchdog-6.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:490ab2ef84f11129844c23fb14ecf30ef3d8a6abafd3754a6f75ca1e6654136c"}, + {file = "watchdog-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:76aae96b00ae814b181bb25b1b98076d5fc84e8a53cd8885a318b42b6d3a5134"}, + {file = "watchdog-6.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a175f755fc2279e0b7312c0035d52e27211a5bc39719dd529625b1930917345b"}, + {file = "watchdog-6.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:c7ac31a19f4545dd92fc25d200694098f42c9a8e391bc00bdd362c5736dbf881"}, + {file = "watchdog-6.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9513f27a1a582d9808cf21a07dae516f0fab1cf2d7683a742c498b93eedabb11"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2"}, + {file = "watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a"}, + {file = "watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680"}, + {file = "watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f"}, + {file = "watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282"}, +] + +[[package]] +name = "wrapt" +version = "1.17.2" +requires_python = ">=3.8" +summary = "Module for decorators, wrappers and monkey patching." +files = [ + {file = "wrapt-1.17.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3d57c572081fed831ad2d26fd430d565b76aa277ed1d30ff4d40670b1c0dd984"}, + {file = "wrapt-1.17.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5e251054542ae57ac7f3fba5d10bfff615b6c2fb09abeb37d2f1463f841ae22"}, + {file = "wrapt-1.17.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:80dd7db6a7cb57ffbc279c4394246414ec99537ae81ffd702443335a61dbf3a7"}, + {file = "wrapt-1.17.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a6e821770cf99cc586d33833b2ff32faebdbe886bd6322395606cf55153246c"}, + {file = "wrapt-1.17.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b60fb58b90c6d63779cb0c0c54eeb38941bae3ecf7a73c764c52c88c2dcb9d72"}, + {file = "wrapt-1.17.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b870b5df5b71d8c3359d21be8f0d6c485fa0ebdb6477dda51a1ea54a9b558061"}, + {file = "wrapt-1.17.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4011d137b9955791f9084749cba9a367c68d50ab8d11d64c50ba1688c9b457f2"}, + {file = "wrapt-1.17.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:1473400e5b2733e58b396a04eb7f35f541e1fb976d0c0724d0223dd607e0f74c"}, + {file = "wrapt-1.17.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3cedbfa9c940fdad3e6e941db7138e26ce8aad38ab5fe9dcfadfed9db7a54e62"}, + {file = "wrapt-1.17.2-cp310-cp310-win32.whl", hash = "sha256:582530701bff1dec6779efa00c516496968edd851fba224fbd86e46cc6b73563"}, + {file = "wrapt-1.17.2-cp310-cp310-win_amd64.whl", hash = "sha256:58705da316756681ad3c9c73fd15499aa4d8c69f9fd38dc8a35e06c12468582f"}, + {file = "wrapt-1.17.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ff04ef6eec3eee8a5efef2401495967a916feaa353643defcc03fc74fe213b58"}, + {file = "wrapt-1.17.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4db983e7bca53819efdbd64590ee96c9213894272c776966ca6306b73e4affda"}, + {file = "wrapt-1.17.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9abc77a4ce4c6f2a3168ff34b1da9b0f311a8f1cfd694ec96b0603dff1c79438"}, + {file = "wrapt-1.17.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b929ac182f5ace000d459c59c2c9c33047e20e935f8e39371fa6e3b85d56f4a"}, + {file = "wrapt-1.17.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f09b286faeff3c750a879d336fb6d8713206fc97af3adc14def0cdd349df6000"}, + {file = "wrapt-1.17.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a7ed2d9d039bd41e889f6fb9364554052ca21ce823580f6a07c4ec245c1f5d6"}, + {file = "wrapt-1.17.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:129a150f5c445165ff941fc02ee27df65940fcb8a22a61828b1853c98763a64b"}, + {file = "wrapt-1.17.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1fb5699e4464afe5c7e65fa51d4f99e0b2eadcc176e4aa33600a3df7801d6662"}, + {file = "wrapt-1.17.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9a2bce789a5ea90e51a02dfcc39e31b7f1e662bc3317979aa7e5538e3a034f72"}, + {file = "wrapt-1.17.2-cp311-cp311-win32.whl", hash = "sha256:4afd5814270fdf6380616b321fd31435a462019d834f83c8611a0ce7484c7317"}, + {file = "wrapt-1.17.2-cp311-cp311-win_amd64.whl", hash = "sha256:acc130bc0375999da18e3d19e5a86403667ac0c4042a094fefb7eec8ebac7cf3"}, + {file = "wrapt-1.17.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d5e2439eecc762cd85e7bd37161d4714aa03a33c5ba884e26c81559817ca0925"}, + {file = "wrapt-1.17.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fc7cb4c1c744f8c05cd5f9438a3caa6ab94ce8344e952d7c45a8ed59dd88392"}, + {file = "wrapt-1.17.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8fdbdb757d5390f7c675e558fd3186d590973244fab0c5fe63d373ade3e99d40"}, + {file = "wrapt-1.17.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5bb1d0dbf99411f3d871deb6faa9aabb9d4e744d67dcaaa05399af89d847a91d"}, + {file = "wrapt-1.17.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d18a4865f46b8579d44e4fe1e2bcbc6472ad83d98e22a26c963d46e4c125ef0b"}, + {file = "wrapt-1.17.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc570b5f14a79734437cb7b0500376b6b791153314986074486e0b0fa8d71d98"}, + {file = "wrapt-1.17.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6d9187b01bebc3875bac9b087948a2bccefe464a7d8f627cf6e48b1bbae30f82"}, + {file = "wrapt-1.17.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:9e8659775f1adf02eb1e6f109751268e493c73716ca5761f8acb695e52a756ae"}, + {file = "wrapt-1.17.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e8b2816ebef96d83657b56306152a93909a83f23994f4b30ad4573b00bd11bb9"}, + {file = "wrapt-1.17.2-cp312-cp312-win32.whl", hash = "sha256:468090021f391fe0056ad3e807e3d9034e0fd01adcd3bdfba977b6fdf4213ea9"}, + {file = "wrapt-1.17.2-cp312-cp312-win_amd64.whl", hash = "sha256:ec89ed91f2fa8e3f52ae53cd3cf640d6feff92ba90d62236a81e4e563ac0e991"}, + {file = "wrapt-1.17.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6ed6ffac43aecfe6d86ec5b74b06a5be33d5bb9243d055141e8cabb12aa08125"}, + {file = "wrapt-1.17.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:35621ae4c00e056adb0009f8e86e28eb4a41a4bfa8f9bfa9fca7d343fe94f998"}, + {file = "wrapt-1.17.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a604bf7a053f8362d27eb9fefd2097f82600b856d5abe996d623babd067b1ab5"}, + {file = "wrapt-1.17.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cbabee4f083b6b4cd282f5b817a867cf0b1028c54d445b7ec7cfe6505057cf8"}, + {file = "wrapt-1.17.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49703ce2ddc220df165bd2962f8e03b84c89fee2d65e1c24a7defff6f988f4d6"}, + {file = "wrapt-1.17.2-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8112e52c5822fc4253f3901b676c55ddf288614dc7011634e2719718eaa187dc"}, + {file = "wrapt-1.17.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9fee687dce376205d9a494e9c121e27183b2a3df18037f89d69bd7b35bcf59e2"}, + {file = "wrapt-1.17.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:18983c537e04d11cf027fbb60a1e8dfd5190e2b60cc27bc0808e653e7b218d1b"}, + {file = "wrapt-1.17.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:703919b1633412ab54bcf920ab388735832fdcb9f9a00ae49387f0fe67dad504"}, + {file = "wrapt-1.17.2-cp313-cp313-win32.whl", hash = "sha256:abbb9e76177c35d4e8568e58650aa6926040d6a9f6f03435b7a522bf1c487f9a"}, + {file = "wrapt-1.17.2-cp313-cp313-win_amd64.whl", hash = "sha256:69606d7bb691b50a4240ce6b22ebb319c1cfb164e5f6569835058196e0f3a845"}, + {file = "wrapt-1.17.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:4a721d3c943dae44f8e243b380cb645a709ba5bd35d3ad27bc2ed947e9c68192"}, + {file = "wrapt-1.17.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:766d8bbefcb9e00c3ac3b000d9acc51f1b399513f44d77dfe0eb026ad7c9a19b"}, + {file = "wrapt-1.17.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e496a8ce2c256da1eb98bd15803a79bee00fc351f5dfb9ea82594a3f058309e0"}, + {file = "wrapt-1.17.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d615e4fe22f4ad3528448c193b218e077656ca9ccb22ce2cb20db730f8d306"}, + {file = "wrapt-1.17.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a5aaeff38654462bc4b09023918b7f21790efb807f54c000a39d41d69cf552cb"}, + {file = "wrapt-1.17.2-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a7d15bbd2bc99e92e39f49a04653062ee6085c0e18b3b7512a4f2fe91f2d681"}, + {file = "wrapt-1.17.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e3890b508a23299083e065f435a492b5435eba6e304a7114d2f919d400888cc6"}, + {file = "wrapt-1.17.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:8c8b293cd65ad716d13d8dd3624e42e5a19cc2a2f1acc74b30c2c13f15cb61a6"}, + {file = "wrapt-1.17.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4c82b8785d98cdd9fed4cac84d765d234ed3251bd6afe34cb7ac523cb93e8b4f"}, + {file = "wrapt-1.17.2-cp313-cp313t-win32.whl", hash = "sha256:13e6afb7fe71fe7485a4550a8844cc9ffbe263c0f1a1eea569bc7091d4898555"}, + {file = "wrapt-1.17.2-cp313-cp313t-win_amd64.whl", hash = "sha256:eaf675418ed6b3b31c7a989fd007fa7c3be66ce14e5c3b27336383604c9da85c"}, + {file = "wrapt-1.17.2-py3-none-any.whl", hash = "sha256:b18f2d1533a71f069c7f82d524a52599053d4c7166e9dd374ae2136b7f40f7c8"}, + {file = "wrapt-1.17.2.tar.gz", hash = "sha256:41388e9d4d1522446fe79d3213196bd9e3b301a336965b9e27ca2788ebd122f3"}, +] + +[[package]] +name = "yapf" +version = "0.43.0" +requires_python = ">=3.7" +summary = "A formatter for Python code" +dependencies = [ + "platformdirs>=3.5.1", + "tomli>=2.0.1; python_version < \"3.11\"", +] +files = [ + {file = "yapf-0.43.0-py3-none-any.whl", hash = "sha256:224faffbc39c428cb095818cf6ef5511fdab6f7430a10783fdfb292ccf2852ca"}, + {file = "yapf-0.43.0.tar.gz", hash = "sha256:00d3aa24bfedff9420b2e0d5d9f5ab6d9d4268e72afbf59bb3fa542781d5218e"}, +] diff --git a/pyproject.toml b/pyproject.toml index 89d0839..4fd9d6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "pyyaml>=6.0.1", "rich>=13.7.1", "requests>=2.32.4", + "pydoc-markdown>=4.8.2", ] requires-python = ">=3.10" readme = "README.md" diff --git a/requirements.txt b/requirements.txt index 805ed63..7291302 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ lkml>=1.3.1 pyyaml>=6.0.1 rich>=13.7.1 requests>=2.32.4 +pydoc-markdown>=4.8.2 diff --git a/scripts/generate_docs.py b/scripts/generate_docs.py new file mode 100644 index 0000000..761509c --- /dev/null +++ b/scripts/generate_docs.py @@ -0,0 +1,217 @@ +# scripts/generate_docs.py +import os +import subprocess +import yaml +import sys +import importlib +import inspect +import pkgutil + + +def generate_module_docs(): + """Generate Markdown documentation for all Python modules.""" + + # Ensure docs directory exists + os.makedirs("docs", exist_ok=True) + + # Add current directory to Python path + current_dir = os.path.abspath(".") + if current_dir not in sys.path: + sys.path.insert(0, current_dir) + + # Try both pydoc-markdown and fallback to manual generation + modules = ["lkml2cube.parser", "lkml2cube.converter", "lkml2cube.main"] + + for module in modules: + output_file = f"docs/{module.replace('.', '_')}.md" + print(f"Generating documentation for {module}...") + + # First try pydoc-markdown + if try_pydoc_markdown(module, output_file): + print(f" ✓ Created {output_file} using pydoc-markdown") + continue + + # Fallback to manual generation + try: + generate_manual_docs(module, output_file) + print(f" ✓ Created {output_file} using manual generation") + except Exception as e: + print(f" ✗ Error generating docs for {module}: {e}") + + +def try_pydoc_markdown(module, output_file): + """Try to generate documentation using pydoc-markdown.""" + try: + # Create a temporary config file for this module + config = { + "loaders": [ + { + "type": "python", + "search_path": ["."], + "modules": [module] + } + ], + "renderer": { + "type": "markdown", + "render_toc": True, + "render_module_header": True, + "markdown": { + "filename": output_file + } + } + } + + config_file = f".pydoc-markdown-{module.replace('.', '_')}.yml" + with open(config_file, "w") as f: + yaml.dump(config, f) + + # Run pydoc-markdown with the config file + env = os.environ.copy() + env["PYTHONPATH"] = os.path.abspath(".") + ":" + env.get("PYTHONPATH", "") + + result = subprocess.run( + ["pydoc-markdown", config_file], + capture_output=True, + text=True, + env=env + ) + + # Clean up temporary config file + if os.path.exists(config_file): + os.remove(config_file) + + if result.returncode == 0 and os.path.exists(output_file): + optimize_for_llm(output_file) + return True + else: + return False + except Exception: + return False + + +def generate_manual_docs(module_name, output_file): + """Generate documentation manually using Python introspection.""" + try: + # Import the module + module = importlib.import_module(module_name) + + # Start building the documentation + lines = [] + lines.append(f"# {module_name}") + lines.append("") + + # Add module docstring if available + if module.__doc__: + lines.append(module.__doc__.strip()) + lines.append("") + + # Handle different module types + if hasattr(module, "__path__"): # It's a package + # Get all submodules + submodules = [] + for importer, modname, ispkg in pkgutil.iter_modules(module.__path__, module_name + "."): + try: + submodule = importlib.import_module(modname) + submodules.append((modname, submodule)) + except ImportError: + continue + + if submodules: + lines.append("## Submodules") + lines.append("") + for modname, submodule in submodules: + lines.append(f"### {modname}") + if submodule.__doc__: + lines.append(submodule.__doc__.strip()) + lines.append("") + + # Add classes and functions from submodule + add_module_content(submodule, lines) + lines.append("") + else: + # It's a regular module + add_module_content(module, lines) + + # Write to file + with open(output_file, "w") as f: + f.write("\n".join(lines)) + + # Optimize for LLM consumption + optimize_for_llm(output_file) + + except Exception as e: + raise Exception(f"Failed to generate manual docs for {module_name}: {e}") + + +def add_module_content(module, lines): + """Add classes and functions from a module to the documentation.""" + # Get all classes + classes = [] + functions = [] + + for name, obj in inspect.getmembers(module): + if name.startswith("_"): + continue + if inspect.isclass(obj) and obj.__module__ == module.__name__: + classes.append((name, obj)) + elif inspect.isfunction(obj) and obj.__module__ == module.__name__: + functions.append((name, obj)) + + # Document classes + if classes: + lines.append("#### Classes") + lines.append("") + for name, cls in classes: + lines.append(f"##### {name}") + if cls.__doc__: + lines.append(cls.__doc__.strip()) + lines.append("") + + # Add methods + methods = [] + for method_name, method in inspect.getmembers(cls): + if (method_name.startswith("_") and method_name != "__init__") or not inspect.isfunction(method): + continue + methods.append((method_name, method)) + + if methods: + lines.append("**Methods:**") + lines.append("") + for method_name, method in methods: + lines.append(f"- `{method_name}()`: {method.__doc__.strip().split('.')[0] if method.__doc__ else 'No description'}") + lines.append("") + + # Document functions + if functions: + lines.append("#### Functions") + lines.append("") + for name, func in functions: + lines.append(f"##### {name}()") + if func.__doc__: + lines.append(func.__doc__.strip()) + lines.append("") + + +def optimize_for_llm(filepath): + """Compress documentation for optimal LLM consumption.""" + with open(filepath, "r") as f: + content = f.read() + + # Remove redundant phrases + optimizations = [ + ("This function ", ""), + ("This method ", ""), + ("is used to ", ""), + ("is responsible for ", ""), + ("The purpose of this ", ""), + ] + + for old, new in optimizations: + content = content.replace(old, new) + + with open(filepath, "w") as f: + f.write(content) + + +if __name__ == "__main__": + generate_module_docs() diff --git a/tests/test_converter.py b/tests/test_converter.py new file mode 100644 index 0000000..02fd3e5 --- /dev/null +++ b/tests/test_converter.py @@ -0,0 +1,535 @@ +""" +Unit tests for the LookMLConverter class. + +This module contains comprehensive tests for the LookMLConverter class, +including initialization, configuration management, and all core conversion methods. +""" + +import pytest +import yaml +import tempfile +from pathlib import Path +from unittest.mock import patch, Mock, mock_open +from os.path import join, dirname + +from lkml2cube.converter import LookMLConverter + + +class TestLookMLConverterInitialization: + """Test LookMLConverter initialization and configuration.""" + + def test_default_initialization(self): + """Test LookMLConverter with default parameters.""" + converter = LookMLConverter() + + assert converter.outputdir == "." + assert converter.rootdir is None + assert converter.parseonly is False + assert converter.printonly is False + assert converter.use_explores_name is False + + def test_custom_initialization(self): + """Test LookMLConverter with custom parameters.""" + converter = LookMLConverter( + outputdir="/tmp/output", + rootdir="/tmp/root", + parseonly=True, + printonly=True, + use_explores_name=True + ) + + assert converter.outputdir == "/tmp/output" + assert converter.rootdir == "/tmp/root" + assert converter.parseonly is True + assert converter.printonly is True + assert converter.use_explores_name is True + + def test_string_representation(self): + """Test __repr__ method.""" + converter = LookMLConverter(outputdir="/tmp") + repr_str = repr(converter) + + assert "LookMLConverter" in repr_str + assert "outputdir='/tmp'" in repr_str + assert "rootdir='None'" in repr_str + assert "parseonly=False" in repr_str + + +class TestLookMLConverterConfiguration: + """Test configuration management methods.""" + + def test_get_config(self): + """Test get_config method.""" + converter = LookMLConverter(outputdir="/tmp", parseonly=True) + config = converter.get_config() + + expected_config = { + 'outputdir': '/tmp', + 'rootdir': None, + 'parseonly': True, + 'printonly': False, + 'use_explores_name': False + } + + assert config == expected_config + + def test_set_config_partial(self): + """Test set_config with partial updates.""" + converter = LookMLConverter() + + converter.set_config(outputdir="/new/path", parseonly=True) + + assert converter.outputdir == "/new/path" + assert converter.parseonly is True + assert converter.printonly is False # Should remain unchanged + + def test_set_config_all_params(self): + """Test set_config with all parameters.""" + converter = LookMLConverter() + + converter.set_config( + outputdir="/new/path", + rootdir="/new/root", + parseonly=True, + printonly=True, + use_explores_name=True + ) + + assert converter.outputdir == "/new/path" + assert converter.rootdir == "/new/root" + assert converter.parseonly is True + assert converter.printonly is True + assert converter.use_explores_name is True + + def test_set_config_none_values(self): + """Test set_config with None values (should not change).""" + converter = LookMLConverter(outputdir="/original", parseonly=True) + + converter.set_config(outputdir=None, parseonly=None) + + assert converter.outputdir == "/original" + assert converter.parseonly is True + + +class TestLookMLConverterCubes: + """Test the cubes() method.""" + + @patch('lkml2cube.converter.file_loader') + def test_cubes_file_not_found(self, mock_file_loader): + """Test cubes method when file is not found.""" + mock_file_loader.return_value = None + converter = LookMLConverter() + + with pytest.raises(ValueError, match="No files were found on path"): + converter.cubes("nonexistent.lkml") + + @patch('lkml2cube.converter.file_loader') + def test_cubes_parseonly(self, mock_file_loader): + """Test cubes method with parseonly=True.""" + sample_lookml = { + 'views': [{'name': 'orders', 'sql_table_name': 'orders'}] + } + mock_file_loader.return_value = sample_lookml + + converter = LookMLConverter(parseonly=True) + result = converter.cubes("test.lkml") + + assert 'lookml_model' in result + assert 'parsed_model' in result + assert result['lookml_model'] == sample_lookml + assert isinstance(result['parsed_model'], str) + + @patch('lkml2cube.converter.console') + @patch('lkml2cube.converter.yaml.dump') + @patch('lkml2cube.converter.generate_cube_joins') + @patch('lkml2cube.converter.parse_view') + @patch('lkml2cube.converter.file_loader') + def test_cubes_printonly(self, mock_file_loader, mock_parse_view, + mock_generate_joins, mock_yaml_dump, mock_console): + """Test cubes method with printonly=True.""" + sample_lookml = {'views': [{'name': 'orders'}]} + sample_cube_def = {'cubes': [{'name': 'orders'}]} + yaml_output = "cubes:\n- name: orders\n" + + mock_file_loader.return_value = sample_lookml + mock_parse_view.return_value = sample_cube_def + mock_generate_joins.return_value = sample_cube_def + mock_yaml_dump.return_value = yaml_output + + converter = LookMLConverter(printonly=True) + result = converter.cubes("test.lkml") + + assert 'lookml_model' in result + assert 'cube_def' in result + assert 'yaml_output' in result + assert result['yaml_output'] == yaml_output + mock_console.print.assert_called_once_with(yaml_output) + + @patch('lkml2cube.converter.print_summary') + @patch('lkml2cube.converter.write_files') + @patch('lkml2cube.converter.generate_cube_joins') + @patch('lkml2cube.converter.parse_view') + @patch('lkml2cube.converter.file_loader') + def test_cubes_write_files(self, mock_file_loader, mock_parse_view, + mock_generate_joins, mock_write_files, mock_print_summary): + """Test cubes method with file writing.""" + sample_lookml = {'views': [{'name': 'orders'}]} + sample_cube_def = {'cubes': [{'name': 'orders'}]} + sample_summary = {'cubes': [{'name': 'orders', 'path': '/tmp/orders.yml'}]} + + mock_file_loader.return_value = sample_lookml + mock_parse_view.return_value = sample_cube_def + mock_generate_joins.return_value = sample_cube_def + mock_write_files.return_value = sample_summary + + converter = LookMLConverter(outputdir="/tmp") + result = converter.cubes("test.lkml") + + assert 'lookml_model' in result + assert 'cube_def' in result + assert 'summary' in result + assert result['summary'] == sample_summary + + mock_write_files.assert_called_once_with(sample_cube_def, outputdir="/tmp") + mock_print_summary.assert_called_once_with(sample_summary) + + +class TestLookMLConverterViews: + """Test the views() method.""" + + @patch('lkml2cube.converter.file_loader') + def test_views_file_not_found(self, mock_file_loader): + """Test views method when file is not found.""" + mock_file_loader.return_value = None + converter = LookMLConverter() + + with pytest.raises(ValueError, match="No files were found on path"): + converter.views("nonexistent.lkml") + + @patch('lkml2cube.converter.file_loader') + def test_views_parseonly(self, mock_file_loader): + """Test views method with parseonly=True.""" + sample_lookml = { + 'views': [{'name': 'orders'}], + 'explores': [{'name': 'orders_explore'}] + } + mock_file_loader.return_value = sample_lookml + + converter = LookMLConverter(parseonly=True) + result = converter.views("test.lkml") + + assert 'lookml_model' in result + assert 'parsed_model' in result + assert result['lookml_model'] == sample_lookml + + @patch('lkml2cube.converter.console') + @patch('lkml2cube.converter.yaml.dump') + @patch('lkml2cube.converter.parse_explores') + @patch('lkml2cube.converter.file_loader') + def test_views_printonly(self, mock_file_loader, mock_parse_explores, + mock_yaml_dump, mock_console): + """Test views method with printonly=True.""" + sample_lookml = {'views': [{'name': 'orders'}], 'explores': []} + sample_cube_def = {'cubes': [{'name': 'orders'}], 'views': []} + yaml_output = "cubes:\n- name: orders\nviews: []\n" + + mock_file_loader.return_value = sample_lookml + mock_parse_explores.return_value = sample_cube_def + mock_yaml_dump.return_value = yaml_output + + converter = LookMLConverter(printonly=True, use_explores_name=True) + result = converter.views("test.lkml") + + assert 'yaml_output' in result + assert result['yaml_output'] == yaml_output + mock_parse_explores.assert_called_once_with(sample_lookml, True) + mock_console.print.assert_called_once_with(yaml_output) + + @patch('lkml2cube.converter.print_summary') + @patch('lkml2cube.converter.write_files') + @patch('lkml2cube.converter.parse_explores') + @patch('lkml2cube.converter.file_loader') + def test_views_write_files(self, mock_file_loader, mock_parse_explores, + mock_write_files, mock_print_summary): + """Test views method with file writing.""" + sample_lookml = {'views': [{'name': 'orders'}], 'explores': []} + sample_cube_def = {'cubes': [{'name': 'orders'}], 'views': []} + sample_summary = {'cubes': [{'name': 'orders', 'path': '/tmp/orders.yml'}], 'views': []} + + mock_file_loader.return_value = sample_lookml + mock_parse_explores.return_value = sample_cube_def + mock_write_files.return_value = sample_summary + + converter = LookMLConverter(outputdir="/tmp", use_explores_name=False) + result = converter.views("test.lkml") + + assert 'summary' in result + assert result['summary'] == sample_summary + + mock_parse_explores.assert_called_once_with(sample_lookml, False) + mock_write_files.assert_called_once_with(sample_cube_def, outputdir="/tmp") + + +class TestLookMLConverterExplores: + """Test the explores() method.""" + + @patch('lkml2cube.converter.meta_loader') + def test_explores_no_response(self, mock_meta_loader): + """Test explores method when no response is received.""" + mock_meta_loader.return_value = None + converter = LookMLConverter() + + with pytest.raises(ValueError, match="No response received from"): + converter.explores("http://test.com/meta", "token") + + @patch('lkml2cube.converter.meta_loader') + def test_explores_parseonly(self, mock_meta_loader): + """Test explores method with parseonly=True.""" + sample_cube_model = { + 'cubes': [{'name': 'orders', 'sql_table': 'orders'}] + } + mock_meta_loader.return_value = sample_cube_model + + converter = LookMLConverter(parseonly=True) + result = converter.explores("http://test.com/meta", "token") + + assert 'cube_model' in result + assert 'parsed_model' in result + assert result['cube_model'] == sample_cube_model + + @patch('lkml2cube.converter.console') + @patch('lkml2cube.converter.yaml.dump') + @patch('lkml2cube.converter.parse_meta') + @patch('lkml2cube.converter.meta_loader') + def test_explores_printonly(self, mock_meta_loader, mock_parse_meta, + mock_yaml_dump, mock_console): + """Test explores method with printonly=True.""" + sample_cube_model = {'cubes': [{'name': 'orders'}]} + sample_lookml_model = {'views': [{'name': 'orders'}], 'explores': []} + yaml_output = "views:\n- name: orders\nexplores: []\n" + + mock_meta_loader.return_value = sample_cube_model + mock_parse_meta.return_value = sample_lookml_model + mock_yaml_dump.return_value = yaml_output + + converter = LookMLConverter(printonly=True) + result = converter.explores("http://test.com/meta", "token") + + assert 'lookml_model' in result + assert 'yaml_output' in result + assert result['yaml_output'] == yaml_output + mock_console.print.assert_called_once_with(yaml_output) + + @patch('lkml2cube.converter.print_summary') + @patch('lkml2cube.converter.write_lookml_files') + @patch('lkml2cube.converter.parse_meta') + @patch('lkml2cube.converter.meta_loader') + def test_explores_write_files(self, mock_meta_loader, mock_parse_meta, + mock_write_lookml_files, mock_print_summary): + """Test explores method with file writing.""" + sample_cube_model = {'cubes': [{'name': 'orders'}]} + sample_lookml_model = {'views': [{'name': 'orders'}], 'explores': []} + sample_summary = {'views': [{'name': 'orders', 'path': '/tmp/orders.lkml'}], 'explores': []} + + mock_meta_loader.return_value = sample_cube_model + mock_parse_meta.return_value = sample_lookml_model + mock_write_lookml_files.return_value = sample_summary + + converter = LookMLConverter(outputdir="/tmp") + result = converter.explores("http://test.com/meta", "token") + + assert 'lookml_model' in result + assert 'summary' in result + assert result['summary'] == sample_summary + + mock_write_lookml_files.assert_called_once_with(sample_lookml_model, outputdir="/tmp") + mock_print_summary.assert_called_once_with(sample_summary) + + +class TestLookMLConverterUtilities: + """Test utility methods.""" + + @patch('lkml2cube.converter.file_loader') + def test_validate_files_all_valid(self, mock_file_loader): + """Test validate_files with all valid files.""" + mock_file_loader.side_effect = [ + {'views': [{'name': 'orders'}]}, + {'views': [{'name': 'customers'}]} + ] + + converter = LookMLConverter() + result = converter.validate_files(["orders.lkml", "customers.lkml"]) + + assert result == {"orders.lkml": True, "customers.lkml": True} + assert mock_file_loader.call_count == 2 + + @patch('lkml2cube.converter.file_loader') + def test_validate_files_mixed_results(self, mock_file_loader): + """Test validate_files with mixed valid/invalid files.""" + mock_file_loader.side_effect = [ + {'views': [{'name': 'orders'}]}, + None, # Invalid file + Exception("Parse error") # Exception + ] + + converter = LookMLConverter() + result = converter.validate_files(["orders.lkml", "invalid.lkml", "error.lkml"]) + + assert result == { + "orders.lkml": True, + "invalid.lkml": False, + "error.lkml": False + } + + @patch('lkml2cube.converter.file_loader') + def test_validate_files_empty_list(self, mock_file_loader): + """Test validate_files with empty file list.""" + converter = LookMLConverter() + result = converter.validate_files([]) + + assert result == {} + mock_file_loader.assert_not_called() + + +class TestLookMLConverterIntegration: + """Integration tests using real sample files.""" + + def setup_method(self): + """Set up test fixtures.""" + self.samples_dir = join(dirname(__file__), "samples") + # Clear the global visited_path cache to prevent interference between tests + from lkml2cube.parser import loader + loader.visited_path.clear() + + def test_cubes_integration_with_sample_files(self): + """Test cubes method with actual sample files.""" + converter = LookMLConverter(parseonly=True, rootdir=self.samples_dir) + + # Use the sample orders view file (full path as expected by file_loader) + file_path = join(self.samples_dir, "lkml/views/orders.view.lkml") + result = converter.cubes(file_path) + + assert 'lookml_model' in result + assert 'parsed_model' in result + assert result['lookml_model'] is not None + assert 'views' in result['lookml_model'] + assert len(result['lookml_model']['views']) > 0 + + def test_views_integration_with_sample_files(self): + """Test views method with actual sample files.""" + converter = LookMLConverter(parseonly=True, rootdir=self.samples_dir) + + # Use the sample explores file (full path as expected by file_loader) + file_path = join(self.samples_dir, "lkml/explores/orders_summary.model.lkml") + result = converter.views(file_path) + + assert 'lookml_model' in result + assert 'parsed_model' in result + assert result['lookml_model'] is not None + + def test_validate_files_with_sample_files(self): + """Test validate_files with actual sample files.""" + converter = LookMLConverter(rootdir=self.samples_dir) + + # Use full paths as expected by file_loader + file_paths = [ + join(self.samples_dir, "lkml/views/orders.view.lkml"), + join(self.samples_dir, "lkml/views/nonexistent.view.lkml") + ] + + result = converter.validate_files(file_paths) + + # First file should be valid, second should be invalid + assert len(result) == 2 + assert result[file_paths[0]] == True + assert result[file_paths[1]] == False + + +class TestLookMLConverterErrorHandling: + """Test error handling and edge cases.""" + + def test_cubes_with_invalid_file_path(self): + """Test cubes method with invalid file path.""" + converter = LookMLConverter() + + with pytest.raises(ValueError, match="No files were found on path"): + converter.cubes("nonexistent_file.lkml") + + def test_views_with_invalid_file_path(self): + """Test views method with invalid file path.""" + converter = LookMLConverter() + + with pytest.raises(ValueError, match="No files were found on path"): + converter.views("nonexistent_file.lkml") + + def test_explores_with_invalid_url(self): + """Test explores method with invalid URL.""" + converter = LookMLConverter() + + with pytest.raises(ValueError, match="A valid token must be provided"): + converter.explores("http://invalid.com/meta", "") + + @patch('lkml2cube.converter.meta_loader') + def test_explores_with_api_error(self, mock_meta_loader): + """Test explores method with API error.""" + mock_meta_loader.side_effect = Exception("API Error") + converter = LookMLConverter() + + with pytest.raises(Exception, match="API Error"): + converter.explores("http://test.com/meta", "token") + + @patch('lkml2cube.converter.file_loader') + def test_cubes_with_parse_error(self, mock_file_loader): + """Test cubes method with parsing error.""" + mock_file_loader.side_effect = Exception("Parse error") + converter = LookMLConverter() + + with pytest.raises(Exception, match="Parse error"): + converter.cubes("test.lkml") + + def test_set_config_with_invalid_types(self): + """Test set_config with various parameter types.""" + converter = LookMLConverter() + + # These should work without error + converter.set_config(outputdir="string") + converter.set_config(parseonly=True) + converter.set_config(parseonly=False) + converter.set_config(rootdir=None) + + # Verify the values were set correctly + assert converter.outputdir == "string" + assert converter.parseonly is False + assert converter.rootdir is None + + +class TestLookMLConverterYAMLConfiguration: + """Test YAML configuration setup.""" + + def test_yaml_representers_configured(self): + """Test that YAML representers are properly configured.""" + from lkml2cube.parser.types import folded_unicode, literal_unicode + + # Creating a converter should configure YAML representers + converter = LookMLConverter() + + # Test that the representers work + folded_text = folded_unicode("This is folded text") + literal_text = literal_unicode("This is literal text") + + # These should not raise errors + yaml_output = yaml.dump({"folded": folded_text, "literal": literal_text}) + assert isinstance(yaml_output, str) + + def test_multiple_converter_instances(self): + """Test that multiple converter instances work correctly.""" + converter1 = LookMLConverter(outputdir="/tmp1") + converter2 = LookMLConverter(outputdir="/tmp2") + + assert converter1.outputdir == "/tmp1" + assert converter2.outputdir == "/tmp2" + + # Configuration should be independent + converter1.set_config(parseonly=True) + assert converter1.parseonly is True + assert converter2.parseonly is False \ No newline at end of file From 31e7382fe0ac8cd03224a1778aefb3643392da6c Mon Sep 17 00:00:00 2001 From: Paco Valdez Date: Wed, 16 Jul 2025 13:18:18 -0700 Subject: [PATCH 2/2] Fix cache clearing on converter --- CLAUDE.md | 20 +- docs/lkml2cube_converter.md | 316 ++++++++++++++++++++-- docs/lkml2cube_main.md | 304 ++++++++++++++------- docs/lkml2cube_parser.md | 430 ------------------------------ docs/lkml2cube_parser_cube_api.md | 101 +++++++ docs/lkml2cube_parser_explores.md | 255 ++++++++++++++++++ docs/lkml2cube_parser_loader.md | 200 ++++++++++++++ docs/lkml2cube_parser_types.md | 149 +++++++++++ docs/lkml2cube_parser_views.md | 55 ++++ lkml2cube/converter.py | 16 ++ scripts/generate_docs.py | 156 ++--------- tests/test_converter.py | 82 +++++- tests/test_e2e.py | 6 + 13 files changed, 1402 insertions(+), 688 deletions(-) delete mode 100644 docs/lkml2cube_parser.md create mode 100644 docs/lkml2cube_parser_cube_api.md create mode 100644 docs/lkml2cube_parser_explores.md create mode 100644 docs/lkml2cube_parser_loader.md create mode 100644 docs/lkml2cube_parser_types.md create mode 100644 docs/lkml2cube_parser_views.md diff --git a/CLAUDE.md b/CLAUDE.md index 2791f6a..41e599c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,8 +18,12 @@ lkml2cube is a Python CLI tool that converts LookML models into Cube data models 📖 **Documentation Files**: - `docs/lkml2cube_main.md` - CLI commands and usage -- `docs/lkml2cube_converter.md` - LookMLConverter Python API -- `docs/lkml2cube_parser.md` - Parser module functions and classes +- `docs/lkml2cube_converter.md` - LookMLConverter Python API +- `docs/lkml2cube_parser_cube_api.md` - Module to interact with Cube meta API +- `docs/lkml2cube_parser_loader.md` - File loading and writing utilities +- `docs/lkml2cube_parser_explores.md` - Module to convert LookML explores to Cube `view` definitions +- `docs/lkml2cube_parser_types.md` - Custom YAML types for proper formatting +- `docs/lkml2cube_parser_views.md` - Module to convert LookML views to Cube `cube` definitions ⚠️ **Required Workflow**: When modifying code → Update docstrings → Run `python scripts/generate_docs.py` @@ -162,7 +166,11 @@ All methods return a dictionary with relevant data: The project maintains auto-generated documentation: - `docs/lkml2cube_main.md` - CLI commands and usage - `docs/lkml2cube_converter.md` - LookMLConverter Python API -- `docs/lkml2cube_parser.md` - Parser module functions and classes +- `docs/lkml2cube_parser_cube_api.md` - Module to interact with Cube meta API +- `docs/lkml2cube_parser_loader.md` - File loading and writing utilities +- `docs/lkml2cube_parser_explores.md` - Module to convert LookML explores to Cube `view` definitions +- `docs/lkml2cube_parser_types.md` - Custom YAML types for proper formatting +- `docs/lkml2cube_parser_views.md` - Module to convert LookML views to Cube `cube` definitions ### Documentation Maintenance Workflow @@ -271,7 +279,11 @@ Before committing changes: **Key Files**: - `docs/lkml2cube_main.md` - CLI documentation - `docs/lkml2cube_converter.md` - Python API documentation -- `docs/lkml2cube_parser.md` - Parser module documentation +- `docs/lkml2cube_parser_cube_api.md` - Module to interact with Cube meta API +- `docs/lkml2cube_parser_loader.md` - File loading and writing utilities +- `docs/lkml2cube_parser_explores.md` - Module to convert LookML explores to Cube `view` definitions +- `docs/lkml2cube_parser_types.md` - Custom YAML types for proper formatting +- `docs/lkml2cube_parser_views.md` - Module to convert LookML views to Cube `cube` definitions - `scripts/generate_docs.py` - Documentation generation script **Remember**: Documentation first, code second. Always maintain docstrings! \ No newline at end of file diff --git a/docs/lkml2cube_converter.md b/docs/lkml2cube_converter.md index d942955..d3e05ac 100644 --- a/docs/lkml2cube_converter.md +++ b/docs/lkml2cube_converter.md @@ -1,3 +1,19 @@ +# Table of Contents + +* [lkml2cube.converter](#lkml2cube.converter) + * [LookMLConverter](#lkml2cube.converter.LookMLConverter) + * [\_\_init\_\_](#lkml2cube.converter.LookMLConverter.__init__) + * [cubes](#lkml2cube.converter.LookMLConverter.cubes) + * [views](#lkml2cube.converter.LookMLConverter.views) + * [explores](#lkml2cube.converter.LookMLConverter.explores) + * [set\_config](#lkml2cube.converter.LookMLConverter.set_config) + * [get\_config](#lkml2cube.converter.LookMLConverter.get_config) + * [validate\_files](#lkml2cube.converter.LookMLConverter.validate_files) + * [clear\_cache](#lkml2cube.converter.LookMLConverter.clear_cache) + * [\_\_repr\_\_](#lkml2cube.converter.LookMLConverter.__repr__) + + + # lkml2cube.converter Main converter class for lkml2cube providing a Python API for LookML to Cube conversion. @@ -6,27 +22,283 @@ This module provides a high-level interface for converting LookML models to Cube without requiring CLI usage. It maintains configuration state and provides the same functionality as the CLI commands. -#### Classes + + +## LookMLConverter Objects + +```python +class LookMLConverter() +``` -##### LookMLConverter Main converter class for LookML to Cube conversion operations. - - This class provides a Python API for converting LookML models to Cube definitions, - maintaining configuration state and providing the same functionality as the CLI commands. - - Attributes: - outputdir (str): Directory where output files will be written. - rootdir (str | None): Root directory for resolving LookML includes. - parseonly (bool): If True, only parse and return Python dict representation. - printonly (bool): If True, print YAML output to stdout instead of writing files. - use_explores_name (bool): Whether to use explore names for cube view names. - -**Methods:** - -- `__init__()`: Initialize the LookML converter with configuration options -- `cubes()`: Generate cube definitions from LookML views -- `explores()`: Generate LookML explores from Cube meta API -- `get_config()`: Get current converter configuration -- `set_config()`: Update converter configuration options -- `validate_files()`: Validate that LookML files exist and can be loaded -- `views()`: Generate cube definitions with views from LookML explores + +This class provides a Python API for converting LookML models to Cube definitions, +maintaining configuration state and providing the same functionality as the CLI commands. + +**Attributes**: + +- `outputdir` _str_ - Directory where output files will be written. +- `rootdir` _str | None_ - Root directory for resolving LookML includes. +- `parseonly` _bool_ - If True, only parse and return Python dict representation. +- `printonly` _bool_ - If True, print YAML output to stdout instead of writing files. +- `use_explores_name` _bool_ - Whether to use explore names for cube view names. + + + +#### \_\_init\_\_ + +```python +def __init__(outputdir: str = ".", + rootdir: Optional[str] = None, + parseonly: bool = False, + printonly: bool = False, + use_explores_name: bool = False) +``` + +Initialize the LookML converter with configuration options. + +**Arguments**: + +- `outputdir` _str, optional_ - Directory where output files will be written. Defaults to ".". +- `rootdir` _str | None, optional_ - Root directory for resolving LookML includes. Defaults to None. +- `parseonly` _bool, optional_ - If True, only parse and return Python dict representation. Defaults to False. +- `printonly` _bool, optional_ - If True, print YAML output to stdout instead of writing files. Defaults to False. +- `use_explores_name` _bool, optional_ - Whether to use explore names for cube view names. Defaults to False. + + +**Example**: + + >>> converter = LookMLConverter(outputdir="/tmp/output", rootdir="/lookml/models") + >>> result = converter.cubes("models/*.lkml") + >>> print(result['summary']['cubes'][0]['name']) + 'orders' + + + +#### cubes + +```python +def cubes(file_path: str) -> Dict[str, Any] +``` + +Generate cube definitions from LookML views. + +Converts LookML views into Cube cube definitions, handling dimensions, measures, +and basic join relationships. + +**Arguments**: + +- `file_path` _str_ - Path to LookML file(s) to process (supports glob patterns). + + +**Returns**: + +- `dict` - Result dictionary containing: + - 'lookml_model': Parsed LookML model (if parseonly=True) + - 'cube_def': Generated cube definitions + - 'yaml_output': YAML string representation (if printonly=True) + - 'summary': File generation summary (if files written) + + +**Raises**: + +- `ValueError` - If no files are found at the specified path. + + +**Example**: + + >>> converter = LookMLConverter() + >>> result = converter.cubes("models/orders.lkml") + >>> print(result['cube_def']['cubes'][0]['name']) + 'orders' + + + +#### views + +```python +def views(file_path: str) -> Dict[str, Any] +``` + +Generate cube definitions with views from LookML explores. + +Converts LookML explores into Cube definitions including both cubes and views +with join relationships. + +**Arguments**: + +- `file_path` _str_ - Path to LookML file(s) to process (supports glob patterns). + + +**Returns**: + +- `dict` - Result dictionary containing: + - 'lookml_model': Parsed LookML model (if parseonly=True) + - 'cube_def': Generated cube definitions with views + - 'yaml_output': YAML string representation (if printonly=True) + - 'summary': File generation summary (if files written) + + +**Raises**: + +- `ValueError` - If no files are found at the specified path. + + +**Example**: + + >>> converter = LookMLConverter(use_explores_name=True) + >>> result = converter.views("models/explores.lkml") + >>> print(len(result['cube_def']['views'])) + 2 + + + +#### explores + +```python +def explores(metaurl: str, token: str) -> Dict[str, Any] +``` + +Generate LookML explores from Cube meta API. + +Fetches Cube model from meta API and converts it to LookML explores, +correctly mapping Cube cubes to LookML views and Cube views to LookML explores. + +**Arguments**: + +- `metaurl` _str_ - URL to the Cube meta API endpoint. +- `token` _str_ - JWT token for Cube meta API authentication. + + +**Returns**: + +- `dict` - Result dictionary containing: + - 'cube_model': Raw Cube model from meta API (if parseonly=True) + - 'lookml_model': Converted LookML model + - 'yaml_output': YAML string representation (if printonly=True) + - 'summary': File generation summary (if files written) + + +**Raises**: + +- `ValueError` - If no response is received from the meta API. +- `Exception` - If API request fails or token is invalid. + + +**Example**: + + >>> converter = LookMLConverter(outputdir="/tmp/lookml") + >>> result = converter.explores("https://api.cube.dev/v1/meta", "jwt-token") + >>> print(len(result['lookml_model']['explores'])) + 3 + + + +#### set\_config + +```python +def set_config(outputdir: Optional[str] = None, + rootdir: Optional[str] = None, + parseonly: Optional[bool] = None, + printonly: Optional[bool] = None, + use_explores_name: Optional[bool] = None) -> None +``` + +Update converter configuration options. + +**Arguments**: + +- `outputdir` _str | None, optional_ - Directory where output files will be written. +- `rootdir` _str | None, optional_ - Root directory for resolving LookML includes. +- `parseonly` _bool | None, optional_ - If True, only parse and return Python dict representation. +- `printonly` _bool | None, optional_ - If True, print YAML output to stdout instead of writing files. +- `use_explores_name` _bool | None, optional_ - Whether to use explore names for cube view names. + + +**Example**: + + >>> converter = LookMLConverter() + >>> converter.set_config(outputdir="/new/path", parseonly=True) + >>> result = converter.cubes("models/*.lkml") + # Will now parse only and use the new output directory + + + +#### get\_config + +```python +def get_config() -> Dict[str, Any] +``` + +Get current converter configuration. + +**Returns**: + +- `dict` - Current configuration settings. + + +**Example**: + + >>> converter = LookMLConverter(outputdir="/tmp") + >>> config = converter.get_config() + >>> print(config['outputdir']) + '/tmp' + + + +#### validate\_files + +```python +def validate_files(file_paths: List[str]) -> Dict[str, bool] +``` + +Validate that LookML files exist and can be loaded. + +**Arguments**: + +- `file_paths` _list[str]_ - List of file paths to validate. + + +**Returns**: + +- `dict` - Dictionary mapping file paths to validation results. + + +**Example**: + + >>> converter = LookMLConverter() + >>> results = converter.validate_files(["models/orders.lkml", "models/missing.lkml"]) + >>> print(results["models/orders.lkml"]) + True + + + +#### clear\_cache + +```python +def clear_cache() -> None +``` + +Clear the global file loader cache. + +clears the visited_path cache used by the file_loader to prevent +circular includes. Useful for ensuring clean state between operations or +in testing scenarios. + +**Example**: + + >>> converter = LookMLConverter() + >>> converter.cubes("models/orders.lkml") # Populates cache + >>> converter.clear_cache() # Clears cache + >>> converter.cubes("models/orders.lkml") # Loads fresh from disk + + + +#### \_\_repr\_\_ + +```python +def __repr__() -> str +``` + +Return string representation of the converter. + diff --git a/docs/lkml2cube_main.md b/docs/lkml2cube_main.md index 08ae850..4ac4079 100644 --- a/docs/lkml2cube_main.md +++ b/docs/lkml2cube_main.md @@ -1,3 +1,13 @@ +# Table of Contents + +* [lkml2cube.main](#lkml2cube.main) + * [callback](#lkml2cube.main.callback) + * [cubes](#lkml2cube.main.cubes) + * [views](#lkml2cube.main.views) + * [explores](#lkml2cube.main.explores) + + + # lkml2cube.main Main CLI module for lkml2cube - LookML to Cube bidirectional converter. @@ -10,104 +20,204 @@ This module provides the command-line interface for lkml2cube, offering three ma The CLI is built using Typer and provides rich console output with proper error handling. Each command supports various options for parsing, output formatting, and file generation. -#### Functions + + +#### callback + +```python +@app.callback() +def callback() +``` -##### callback() Main callback function for the lkml2cube CLI application. - - serves as the entry point for the CLI and provides - general information about the tool. It sets up the global context - for all subcommands. - - Note: - is called automatically by Typer when the CLI is invoked. - It doesn't perform any specific actions but serves as a placeholder - for global CLI configuration. - - Example: - $ lkml2cube --help - # Shows help information for the entire CLI - -##### cubes() + +serves as the entry point for the CLI and provides +general information about the tool. It sets up the global context +for all subcommands. + +**Notes**: + + is called automatically by Typer when the CLI is invoked. + It doesn't perform any specific actions but serves as a placeholder + for global CLI configuration. + + +**Example**: + + $ lkml2cube --help + # Shows help information for the entire CLI + + + +#### cubes + +```python +@app.command() +def cubes( + file_path: Annotated[str, + typer.Argument(help="The path for the file to read")], + parseonly: Annotated[ + bool, + typer.Option(help=("When present it will only show the python" + " dict read from the lookml file")), + ] = False, + outputdir: Annotated[ + str, + typer.Option( + help="The path for the output files to be generated")] = ".", + printonly: Annotated[bool, + typer.Option( + help="Print to stdout the parsed files")] = False, + rootdir: Annotated[ + str, + typer.Option(help="The path to prepend to include paths")] = None) +``` + Generate Cube model definitions from LookML view files. - - Converts LookML view files into Cube YAML definitions, handling dimensions, - measures, and basic join relationships. This command focuses on generating - cube definitions only (no views). - - Args: - file_path (str): Path to the LookML file to process (supports glob patterns). - parseonly (bool, optional): If True, only displays the parsed LookML as Python dict. Defaults to False. - outputdir (str, optional): Directory where output files will be written. Defaults to ".". - printonly (bool, optional): If True, prints YAML to stdout instead of writing files. Defaults to False. - rootdir (str | None, optional): Root directory for resolving LookML includes. Defaults to None. - - Raises: - typer.Exit: If no files are found at the specified path. - - Example: - $ lkml2cube cubes models/orders.view.lkml --outputdir output/ - # Generates cube definitions in output/cubes/ - - $ lkml2cube cubes models/orders.view.lkml --parseonly - # Shows parsed LookML structure - - $ lkml2cube cubes models/orders.view.lkml --printonly - # Prints YAML to console - -##### explores() -Generate LookML explores and views from Cube meta API. - - Fetches Cube model definitions from the meta API and converts them to - production-ready LookML files. This command correctly maps: - - Cube cubes (with sql_table/sql) → LookML views - - Cube views (with aliasMember joins) → LookML explores - - Args: - metaurl (str): URL to the Cube meta API endpoint (e.g., https://api.cube.dev/v1/meta). - token (str): JWT authentication token for the Cube meta API. - parseonly (bool, optional): If True, only displays the parsed Cube model as Python dict. Defaults to False. - outputdir (str, optional): Directory where output files will be written. Defaults to ".". - printonly (bool, optional): If True, prints YAML to stdout instead of writing files. Defaults to False. - - Raises: - typer.Exit: If no response is received from the meta API. - ValueError: If the token is invalid or API request fails. - - Example: - $ lkml2cube explores "https://api.cube.dev/v1/meta" --token "jwt-token" --outputdir lookml/ - # Generates LookML views and explores in lookml/views/ and lookml/explores/ - - $ lkml2cube explores "https://api.cube.dev/v1/meta" --token "jwt-token" --parseonly - # Shows parsed Cube model structure - - $ lkml2cube explores "https://api.cube.dev/v1/meta" --token "jwt-token" --printonly - # Prints generated LookML to console - -##### views() + +Converts LookML view files into Cube YAML definitions, handling dimensions, +measures, and basic join relationships. This command focuses on generating +cube definitions only (no views). + +**Arguments**: + +- `file_path` _str_ - Path to the LookML file to process (supports glob patterns). +- `parseonly` _bool, optional_ - If True, only displays the parsed LookML as Python dict. Defaults to False. +- `outputdir` _str, optional_ - Directory where output files will be written. Defaults to ".". +- `printonly` _bool, optional_ - If True, prints YAML to stdout instead of writing files. Defaults to False. +- `rootdir` _str | None, optional_ - Root directory for resolving LookML includes. Defaults to None. + + +**Raises**: + +- `typer.Exit` - If no files are found at the specified path. + + +**Example**: + + $ lkml2cube cubes models/orders.view.lkml --outputdir output/ + # Generates cube definitions in output/cubes/ + + $ lkml2cube cubes models/orders.view.lkml --parseonly + # Shows parsed LookML structure + + $ lkml2cube cubes models/orders.view.lkml --printonly + # Prints YAML to console + + + +#### views + +```python +@app.command() +def views( + file_path: Annotated[str, + typer.Argument( + help="The path for the explore to read")], + parseonly: Annotated[ + bool, + typer.Option(help=("When present it will only show the python" + " dict read from the lookml file")), + ] = False, + outputdir: Annotated[ + str, + typer.Option( + help="The path for the output files to be generated")] = ".", + printonly: Annotated[bool, + typer.Option( + help="Print to stdout the parsed files")] = False, + rootdir: Annotated[ + str, typer.Option(help="The path to prepend to include paths")] = None, + use_explores_name: Annotated[ + bool, + typer.Option(help="Use explore names for cube view names")] = False) +``` + Generate Cube model definitions with views from LookML explore files. - - Converts LookML explore files into Cube YAML definitions, creating both - cube definitions and view definitions with join relationships. This command - generates a complete Cube model with views that define how cubes relate to each other. - - Args: - file_path (str): Path to the LookML explore file to process (supports glob patterns). - parseonly (bool, optional): If True, only displays the parsed LookML as Python dict. Defaults to False. - outputdir (str, optional): Directory where output files will be written. Defaults to ".". - printonly (bool, optional): If True, prints YAML to stdout instead of writing files. Defaults to False. - rootdir (str | None, optional): Root directory for resolving LookML includes. Defaults to None. - use_explores_name (bool, optional): If True, uses explore names for cube view names. Defaults to False. - - Raises: - typer.Exit: If no files are found at the specified path. - - Example: - $ lkml2cube views models/explores.lkml --outputdir output/ - # Generates cubes and views in output/cubes/ and output/views/ - - $ lkml2cube views models/explores.lkml --use-explores-name - # Uses explore names for view naming - - $ lkml2cube views models/explores.lkml --parseonly - # Shows parsed LookML structure + +Converts LookML explore files into Cube YAML definitions, creating both +cube definitions and view definitions with join relationships. This command +generates a complete Cube model with views that define how cubes relate to each other. + +**Arguments**: + +- `file_path` _str_ - Path to the LookML explore file to process (supports glob patterns). +- `parseonly` _bool, optional_ - If True, only displays the parsed LookML as Python dict. Defaults to False. +- `outputdir` _str, optional_ - Directory where output files will be written. Defaults to ".". +- `printonly` _bool, optional_ - If True, prints YAML to stdout instead of writing files. Defaults to False. +- `rootdir` _str | None, optional_ - Root directory for resolving LookML includes. Defaults to None. +- `use_explores_name` _bool, optional_ - If True, uses explore names for cube view names. Defaults to False. + + +**Raises**: + +- `typer.Exit` - If no files are found at the specified path. + + +**Example**: + + $ lkml2cube views models/explores.lkml --outputdir output/ + # Generates cubes and views in output/cubes/ and output/views/ + + $ lkml2cube views models/explores.lkml --use-explores-name + # Uses explore names for view naming + + $ lkml2cube views models/explores.lkml --parseonly + # Shows parsed LookML structure + + + +#### explores + +```python +@app.command() +def explores( + metaurl: Annotated[str, + typer.Argument(help="The url for cube meta endpoint")], + token: Annotated[str, typer.Option(help="JWT token for Cube meta")], + parseonly: Annotated[ + bool, + typer.Option(help=("When present it will only show the python" + " dict read from the lookml file")), + ] = False, + outputdir: Annotated[ + str, + typer.Option( + help="The path for the output files to be generated")] = ".", + printonly: Annotated[ + bool, typer.Option(help="Print to stdout the parsed files")] = False) +``` + +Generate LookML explores and views from Cube meta API. + +Fetches Cube model definitions from the meta API and converts them to +production-ready LookML files. This command correctly maps: +- Cube cubes (with sql_table/sql) → LookML views +- Cube views (with aliasMember joins) → LookML explores + +**Arguments**: + +- `metaurl` _str_ - URL to the Cube meta API endpoint (e.g., https://api.cube.dev/v1/meta). +- `token` _str_ - JWT authentication token for the Cube meta API. +- `parseonly` _bool, optional_ - If True, only displays the parsed Cube model as Python dict. Defaults to False. +- `outputdir` _str, optional_ - Directory where output files will be written. Defaults to ".". +- `printonly` _bool, optional_ - If True, prints YAML to stdout instead of writing files. Defaults to False. + + +**Raises**: + +- `typer.Exit` - If no response is received from the meta API. +- `ValueError` - If the token is invalid or API request fails. + + +**Example**: + + $ lkml2cube explores "https://api.cube.dev/v1/meta" --token "jwt-token" --outputdir lookml/ + # Generates LookML views and explores in lookml/views/ and lookml/explores/ + + $ lkml2cube explores "https://api.cube.dev/v1/meta" --token "jwt-token" --parseonly + # Shows parsed Cube model structure + + $ lkml2cube explores "https://api.cube.dev/v1/meta" --token "jwt-token" --printonly + # Prints generated LookML to console + diff --git a/docs/lkml2cube_parser.md b/docs/lkml2cube_parser.md deleted file mode 100644 index 4002fac..0000000 --- a/docs/lkml2cube_parser.md +++ /dev/null @@ -1,430 +0,0 @@ -# lkml2cube.parser - -## Submodules - -### lkml2cube.parser.cube_api - -#### Functions - -##### meta_loader() -Load the Cube meta API and return the model as a dictionary. - - Args: - meta_url (str): URL to the Cube meta API endpoint. - token (str): Authentication token for the API. - - Returns: - dict: Cube model data from the meta API. - - Raises: - ValueError: If no valid token is provided. - Exception: If the API request fails or returns non-200 status. - - Example: - >>> model = meta_loader('https://api.cube.dev/v1/meta', 'my-token') - >>> print(model['cubes'][0]['name']) - 'orders' - -##### parse_members() -Parse measures and dimensions from the Cube meta model. - - Args: - members (list): List of dimension or measure definitions from Cube meta. - - Returns: - list: List of parsed members in LookML format. - - Example: - >>> members = [{'name': 'total_sales', 'type': 'sum', 'sql': 'amount'}] - >>> parsed = parse_members(members) - >>> print(parsed[0]['name']) - 'total_sales' - -##### parse_meta() -Parse the Cube meta model and return a simplified version. - - Separates Cube cubes (-> LookML views) from Cube views (-> LookML explores). - - Args: - cube_model (dict): Complete Cube model from meta API. - - Returns: - dict: LookML model with structure: - {'views': list, 'explores': list} - - Example: - >>> cube_model = {'cubes': [{'name': 'orders', 'sql_table': 'orders'}]} - >>> lookml_model = parse_meta(cube_model) - >>> print(lookml_model['views'][0]['name']) - 'orders' - - -### lkml2cube.parser.explores - -#### Functions - -##### build_cube_name_look_up() -Build a lookup dictionary for cube names in the cube definition. - - Args: - cube_def (dict): Cube definition containing 'cubes' list. - - Note: - modifies the cube_def dictionary in place by adding - a 'cube_name_look_up' key if it doesn't exist. - - Example: - >>> cube_def = {'cubes': [{'name': 'orders'}, {'name': 'customers'}]} - >>> build_cube_name_look_up(cube_def) - >>> print('orders' in cube_def['cube_name_look_up']) - True - -##### generate_cube_joins() -Generate cube join definitions from LookML explores. - - Args: - cube_def (dict): Existing cube definition to modify. - lookml_model (dict): LookML model containing explores with joins. - - Returns: - dict: Updated cube definition with join information added to cubes. - - Raises: - Exception: If cube referenced in explores is not found. - - Example: - >>> cube_def = {'cubes': [{'name': 'orders'}, {'name': 'customers'}]} - >>> lookml_model = {'explores': [{'joins': [{'name': 'customers', 'sql_on': '${orders.customer_id} = ${customers.id}', 'relationship': 'many_to_one'}]}]} - >>> updated_def = generate_cube_joins(cube_def, lookml_model) - >>> print(updated_def['cubes'][1]['joins'][0]['name']) - 'orders' - -##### generate_cube_views() -Generate Cube view definitions from LookML explores. - - Args: - cube_def (dict): Cube definition to add views to. - lookml_model (dict): LookML model containing explores. - use_explores_name (bool, optional): Whether to use explore names as view names. Defaults to False. - - Returns: - dict: Updated cube definition with view definitions added. - - Example: - >>> cube_def = {'cubes': [{'name': 'orders'}]} - >>> lookml_model = {'explores': [{'name': 'orders_explore', 'label': 'Orders Analysis'}]} - >>> updated_def = generate_cube_views(cube_def, lookml_model) - >>> print(updated_def['views'][0]['name']) - 'orders_analysis' - -##### get_cube_from_cube_def() -Get a cube definition by name from the cube definition. - - Args: - cube_def (dict): Cube definition containing 'cubes' list. - cube_name (str): Name of the cube to retrieve. - - Returns: - dict | None: Cube definition if found, None otherwise. - - Example: - >>> cube_def = {'cubes': [{'name': 'orders', 'sql_table': 'orders'}]} - >>> cube = get_cube_from_cube_def(cube_def, 'orders') - >>> print(cube['sql_table']) - 'orders' - -##### get_cube_names_from_join_condition() -Extract cube names from a join condition SQL string. - - Args: - join_condition (str): SQL join condition containing cube references. - - Returns: - list[str]: List of cube names found in the join condition. - - Example: - >>> join_condition = '${orders.customer_id} = ${customers.id}' - >>> get_cube_names_from_join_condition(join_condition) - ['orders', 'customers'] - -##### parse_explores() -Parse LookML explores into Cube definitions with joins and views. - - Args: - lookml_model (dict): LookML model containing views and explores. - use_explores_name (bool, optional): Whether to use explore names as view names. Defaults to False. - - Returns: - dict: Complete cube definition with cubes, joins, and views. - - Raises: - Exception: If no explores are found in the LookML model. - - Example: - >>> lookml_model = { - ... 'views': [{'name': 'orders', 'sql_table_name': 'orders'}], - ... 'explores': [{'name': 'orders_explore', 'joins': [{'name': 'customers', 'sql_on': '${orders.customer_id} = ${customers.id}', 'relationship': 'many_to_one'}]}] - ... } - >>> cube_def = parse_explores(lookml_model) - >>> print(len(cube_def['cubes'])) - 1 - >>> print(len(cube_def['views'])) - 1 - -##### snakify() -Convert a string to snake_case format. - - Args: - s (str): String to convert to snake_case. - - Returns: - str: Snake_case version of the input string. - - Example: - >>> snakify('MyViewName') - 'my_view_name' - >>> snakify('Order-Details') - 'order_details' - -##### traverse_graph() -Find the shortest path between two cubes using BFS traversal. - - Args: - join_paths (dict): Dictionary mapping cube names to their connected cubes. - cube_left (str): Starting cube name. - cube_right (str): Target cube name. - - Returns: - str: Dot-separated path from cube_left to cube_right. - - Example: - >>> join_paths = {'orders': ['customers'], 'customers': ['addresses']} - >>> traverse_graph(join_paths, 'orders', 'addresses') - 'orders.customers.addresses' - - -### lkml2cube.parser.loader - -#### Functions - -##### file_loader() -Load LookML files and resolve includes recursively. - - Args: - file_path_input (str): File path pattern to load (supports glob patterns). - rootdir_param (str | None): Root directory for resolving includes. - namespace (dict | None): Existing namespace to merge content into. - - Returns: - dict: Loaded LookML model with resolved includes. - - Raises: - FileNotFoundError: If specified file path cannot be found. - ValueError: If LookML file cannot be parsed. - - Example: - >>> namespace = file_loader('models/*.lkml', '/path/to/root') - >>> print(namespace['views'][0]['name']) - 'my_view' - -##### print_summary() -Print a formatted summary of generated files using Rich tables. - - Args: - summary (dict): Summary dictionary containing file information with keys - 'cubes', 'views', and/or 'explores', each containing lists of - {'name': str, 'path': str} dictionaries. - - Example: - >>> summary = {'cubes': [{'name': 'orders', 'path': '/output/cubes/orders.yml'}]} - >>> print_summary(summary) - # Displays a formatted table showing the generated files - -##### update_namespace() -Update namespace with new file content, merging lists and handling conflicts. - - Args: - namespace (dict | None): Existing namespace dictionary or None. - new_file (dict): New file content to merge into namespace. - - Returns: - dict: Updated namespace with merged content. - - Example: - >>> namespace = {'views': [{'name': 'view1'}]} - >>> new_file = {'views': [{'name': 'view2'}]} - >>> update_namespace(namespace, new_file) - {'views': [{'name': 'view1'}, {'name': 'view2'}]} - -##### write_files() -Write cube definitions to separate files organized by type. - - Args: - cube_def (dict): Cube definitions containing 'cubes' and/or 'views' keys. - outputdir (str): Output directory path. - - Returns: - dict: Summary of written files with structure: - {'cubes': [{'name': str, 'path': str}], 'views': [{'name': str, 'path': str}]} - - Raises: - Exception: If no cube definition is provided. - OSError: If output directory cannot be created or files cannot be written. - - Example: - >>> cube_def = {'cubes': [{'name': 'orders'}], 'views': [{'name': 'orders_view'}]} - >>> summary = write_files(cube_def, '/output') - >>> print(summary['cubes'][0]['name']) - 'orders' - -##### write_lookml_files() -Write LookML model to files in the output directory. - - Args: - lookml_model (dict): LookML model containing 'views' and/or 'explores' keys. - outputdir (str): Output directory path. - - Returns: - dict: Summary of written files with structure: - {'views': [{'name': str, 'path': str}], 'explores': [{'name': str, 'path': str}]} - - Raises: - Exception: If no LookML model is provided. - OSError: If output directory cannot be created or files cannot be written. - - Example: - >>> lookml_model = {'views': [{'name': 'orders'}], 'explores': [{'name': 'orders_explore'}]} - >>> summary = write_lookml_files(lookml_model, '/output') - >>> print(summary['views'][0]['name']) - 'orders' - -##### write_single_file() -Write a single cube definition to a YAML file. - - Args: - cube_def (dict): Cube definition to write. - outputdir (str): Output directory path. - subdir (str, optional): Subdirectory within output directory. Defaults to "cubes". - file_name (str, optional): Name of the output file. Defaults to "my_cubes.yml". - - Raises: - OSError: If output directory cannot be created or file cannot be written. - - Example: - >>> cube_def = {'cubes': [{'name': 'orders', 'sql_table': 'orders'}]} - >>> write_single_file(cube_def, '/output', 'cubes', 'orders.yml') - - -### lkml2cube.parser.types - -#### Classes - -##### Console -Simple console wrapper for printing messages. - - This class provides a basic print interface compatible with Rich console - while falling back to standard print functionality. - -**Methods:** - -- `print()`: Print a message to the console - -##### folded_unicode -String subclass for YAML folded scalar representation. - - This class marks strings that should be represented as folded scalars - in YAML output (using the '>' style). - - Example: - >>> text = folded_unicode("This is a long -string that will be folded") - >>> # When dumped to YAML, will use '>' style - -##### literal_unicode -String subclass for YAML literal scalar representation. - - This class marks strings that should be represented as literal scalars - in YAML output (using the '|' style). - - Example: - >>> sql = literal_unicode("SELECT * -FROM table -WHERE id = 1") - >>> # When dumped to YAML, will use '|' style preserving line breaks - -#### Functions - -##### folded_unicode_representer() -YAML representer for folded_unicode strings. - - Args: - dumper: YAML dumper instance. - data (folded_unicode): String data to represent. - - Returns: - Scalar representation with folded style. - - Example: - >>> import yaml - >>> yaml.add_representer(folded_unicode, folded_unicode_representer) - >>> yaml.dump(folded_unicode("long text")) - '> long text -' - -##### literal_unicode_representer() -YAML representer for literal_unicode strings. - - Args: - dumper: YAML dumper instance. - data (literal_unicode): String data to represent. - - Returns: - Scalar representation with literal style. - - Example: - >>> import yaml - >>> yaml.add_representer(literal_unicode, literal_unicode_representer) - >>> yaml.dump(literal_unicode("SELECT * -FROM table")) - '| - SELECT * - FROM table -' - - -### lkml2cube.parser.views - -#### Functions - -##### parse_view() -Parse LookML views into Cube definitions. - - Converts LookML view definitions into Cube format, handling dimensions, measures, - view inheritance, and various LookML-specific features like tiers and drill fields. - - Args: - lookml_model (dict): LookML model containing views to parse. - raise_when_views_not_present (bool, optional): Whether to raise an exception - when no views are found. Defaults to True. - - Returns: - dict: Cube definitions with structure: - {'cubes': [{'name': str, 'description': str, 'dimensions': list, 'measures': list, 'joins': list}]} - - Raises: - Exception: If raise_when_views_not_present is True and no views are found, - or if required dimension properties are missing. - - Example: - >>> lookml_model = { - ... 'views': [{ - ... 'name': 'orders', - ... 'sql_table_name': 'public.orders', - ... 'dimensions': [{'name': 'id', 'type': 'number', 'sql': '${TABLE}.id'}], - ... 'measures': [{'name': 'count', 'type': 'count'}] - ... }] - ... } - >>> cube_def = parse_view(lookml_model) - >>> print(cube_def['cubes'][0]['name']) - 'orders' - diff --git a/docs/lkml2cube_parser_cube_api.md b/docs/lkml2cube_parser_cube_api.md new file mode 100644 index 0000000..7e8b9f1 --- /dev/null +++ b/docs/lkml2cube_parser_cube_api.md @@ -0,0 +1,101 @@ +# Table of Contents + +* [lkml2cube.parser.cube\_api](#lkml2cube.parser.cube_api) + * [meta\_loader](#lkml2cube.parser.cube_api.meta_loader) + * [parse\_members](#lkml2cube.parser.cube_api.parse_members) + * [parse\_meta](#lkml2cube.parser.cube_api.parse_meta) + + + +# lkml2cube.parser.cube\_api + + + +#### meta\_loader + +```python +def meta_loader(meta_url: str, token: str) -> dict +``` + +Load the Cube meta API and return the model as a dictionary. + +**Arguments**: + +- `meta_url` _str_ - URL to the Cube meta API endpoint. +- `token` _str_ - Authentication token for the API. + + +**Returns**: + +- `dict` - Cube model data from the meta API. + + +**Raises**: + +- `ValueError` - If no valid token is provided. +- `Exception` - If the API request fails or returns non-200 status. + + +**Example**: + + >>> model = meta_loader('https://api.cube.dev/v1/meta', 'my-token') + >>> print(model['cubes'][0]['name']) + 'orders' + + + +#### parse\_members + +```python +def parse_members(members: list) -> list +``` + +Parse measures and dimensions from the Cube meta model. + +**Arguments**: + +- `members` _list_ - List of dimension or measure definitions from Cube meta. + + +**Returns**: + +- `list` - List of parsed members in LookML format. + + +**Example**: + + >>> members = [{'name': 'total_sales', 'type': 'sum', 'sql': 'amount'}] + >>> parsed = parse_members(members) + >>> print(parsed[0]['name']) + 'total_sales' + + + +#### parse\_meta + +```python +def parse_meta(cube_model: dict) -> dict +``` + +Parse the Cube meta model and return a simplified version. + +Separates Cube cubes (-> LookML views) from Cube views (-> LookML explores). + +**Arguments**: + +- `cube_model` _dict_ - Complete Cube model from meta API. + + +**Returns**: + +- `dict` - LookML model with structure: +- `{'views'` - list, 'explores': list} + + +**Example**: + + >>> cube_model = {'cubes': [{'name': 'orders', 'sql_table': 'orders'}]} + >>> lookml_model = parse_meta(cube_model) + >>> print(lookml_model['views'][0]['name']) + 'orders' + diff --git a/docs/lkml2cube_parser_explores.md b/docs/lkml2cube_parser_explores.md new file mode 100644 index 0000000..bb17e62 --- /dev/null +++ b/docs/lkml2cube_parser_explores.md @@ -0,0 +1,255 @@ +# Table of Contents + +* [lkml2cube.parser.explores](#lkml2cube.parser.explores) + * [snakify](#lkml2cube.parser.explores.snakify) + * [build\_cube\_name\_look\_up](#lkml2cube.parser.explores.build_cube_name_look_up) + * [get\_cube\_from\_cube\_def](#lkml2cube.parser.explores.get_cube_from_cube_def) + * [get\_cube\_names\_from\_join\_condition](#lkml2cube.parser.explores.get_cube_names_from_join_condition) + * [traverse\_graph](#lkml2cube.parser.explores.traverse_graph) + * [generate\_cube\_joins](#lkml2cube.parser.explores.generate_cube_joins) + * [generate\_cube\_views](#lkml2cube.parser.explores.generate_cube_views) + * [parse\_explores](#lkml2cube.parser.explores.parse_explores) + + + +# lkml2cube.parser.explores + + + +#### snakify + +```python +def snakify(s) +``` + +Convert a string to snake_case format. + +**Arguments**: + +- `s` _str_ - String to convert to snake_case. + + +**Returns**: + +- `str` - Snake_case version of the input string. + + +**Example**: + + >>> snakify('MyViewName') + 'my_view_name' + >>> snakify('Order-Details') + 'order_details' + + + +#### build\_cube\_name\_look\_up + +```python +def build_cube_name_look_up(cube_def) +``` + +Build a lookup dictionary for cube names in the cube definition. + +**Arguments**: + +- `cube_def` _dict_ - Cube definition containing 'cubes' list. + + +**Notes**: + + modifies the cube_def dictionary in place by adding + a 'cube_name_look_up' key if it doesn't exist. + + +**Example**: + + >>> cube_def = {'cubes': [{'name': 'orders'}, {'name': 'customers'}]} + >>> build_cube_name_look_up(cube_def) + >>> print('orders' in cube_def['cube_name_look_up']) + True + + + +#### get\_cube\_from\_cube\_def + +```python +def get_cube_from_cube_def(cube_def, cube_name) +``` + +Get a cube definition by name from the cube definition. + +**Arguments**: + +- `cube_def` _dict_ - Cube definition containing 'cubes' list. +- `cube_name` _str_ - Name of the cube to retrieve. + + +**Returns**: + + dict | None: Cube definition if found, None otherwise. + + +**Example**: + + >>> cube_def = {'cubes': [{'name': 'orders', 'sql_table': 'orders'}]} + >>> cube = get_cube_from_cube_def(cube_def, 'orders') + >>> print(cube['sql_table']) + 'orders' + + + +#### get\_cube\_names\_from\_join\_condition + +```python +def get_cube_names_from_join_condition(join_condition) +``` + +Extract cube names from a join condition SQL string. + +**Arguments**: + +- `join_condition` _str_ - SQL join condition containing cube references. + + +**Returns**: + +- `list[str]` - List of cube names found in the join condition. + + +**Example**: + + >>> join_condition = '${orders.customer_id} = ${customers.id}' + >>> get_cube_names_from_join_condition(join_condition) + ['orders', 'customers'] + + + +#### traverse\_graph + +```python +def traverse_graph(join_paths, cube_left, cube_right) +``` + +Find the shortest path between two cubes using BFS traversal. + +**Arguments**: + +- `join_paths` _dict_ - Dictionary mapping cube names to their connected cubes. +- `cube_left` _str_ - Starting cube name. +- `cube_right` _str_ - Target cube name. + + +**Returns**: + +- `str` - Dot-separated path from cube_left to cube_right. + + +**Example**: + + >>> join_paths = {'orders': ['customers'], 'customers': ['addresses']} + >>> traverse_graph(join_paths, 'orders', 'addresses') + 'orders.customers.addresses' + + + +#### generate\_cube\_joins + +```python +def generate_cube_joins(cube_def, lookml_model) +``` + +Generate cube join definitions from LookML explores. + +**Arguments**: + +- `cube_def` _dict_ - Existing cube definition to modify. +- `lookml_model` _dict_ - LookML model containing explores with joins. + + +**Returns**: + +- `dict` - Updated cube definition with join information added to cubes. + + +**Raises**: + +- `Exception` - If cube referenced in explores is not found. + + +**Example**: + + >>> cube_def = {'cubes': [{'name': 'orders'}, {'name': 'customers'}]} + >>> lookml_model = {'explores': [{'joins': [{'name': 'customers', 'sql_on': '${orders.customer_id} = ${customers.id}', 'relationship': 'many_to_one'}]}]} + >>> updated_def = generate_cube_joins(cube_def, lookml_model) + >>> print(updated_def['cubes'][1]['joins'][0]['name']) + 'orders' + + + +#### generate\_cube\_views + +```python +def generate_cube_views(cube_def, lookml_model, use_explores_name=False) +``` + +Generate Cube view definitions from LookML explores. + +**Arguments**: + +- `cube_def` _dict_ - Cube definition to add views to. +- `lookml_model` _dict_ - LookML model containing explores. +- `use_explores_name` _bool, optional_ - Whether to use explore names as view names. Defaults to False. + + +**Returns**: + +- `dict` - Updated cube definition with view definitions added. + + +**Example**: + + >>> cube_def = {'cubes': [{'name': 'orders'}]} + >>> lookml_model = {'explores': [{'name': 'orders_explore', 'label': 'Orders Analysis'}]} + >>> updated_def = generate_cube_views(cube_def, lookml_model) + >>> print(updated_def['views'][0]['name']) + 'orders_analysis' + + + +#### parse\_explores + +```python +def parse_explores(lookml_model, use_explores_name=False) +``` + +Parse LookML explores into Cube definitions with joins and views. + +**Arguments**: + +- `lookml_model` _dict_ - LookML model containing views and explores. +- `use_explores_name` _bool, optional_ - Whether to use explore names as view names. Defaults to False. + + +**Returns**: + +- `dict` - Complete cube definition with cubes, joins, and views. + + +**Raises**: + +- `Exception` - If no explores are found in the LookML model. + + +**Example**: + + >>> lookml_model = { + ... 'views': [{'name': 'orders', 'sql_table_name': 'orders'}], + ... 'explores': [{'name': 'orders_explore', 'joins': [{'name': 'customers', 'sql_on': '${orders.customer_id} = ${customers.id}', 'relationship': 'many_to_one'}]}] + ... } + >>> cube_def = parse_explores(lookml_model) + >>> print(len(cube_def['cubes'])) + 1 + >>> print(len(cube_def['views'])) + 1 + diff --git a/docs/lkml2cube_parser_loader.md b/docs/lkml2cube_parser_loader.md new file mode 100644 index 0000000..06c88f1 --- /dev/null +++ b/docs/lkml2cube_parser_loader.md @@ -0,0 +1,200 @@ +# Table of Contents + +* [lkml2cube.parser.loader](#lkml2cube.parser.loader) + * [update\_namespace](#lkml2cube.parser.loader.update_namespace) + * [file\_loader](#lkml2cube.parser.loader.file_loader) + * [write\_single\_file](#lkml2cube.parser.loader.write_single_file) + * [write\_files](#lkml2cube.parser.loader.write_files) + * [write\_lookml\_files](#lkml2cube.parser.loader.write_lookml_files) + * [print\_summary](#lkml2cube.parser.loader.print_summary) + + + +# lkml2cube.parser.loader + + + +#### update\_namespace + +```python +def update_namespace(namespace, new_file) +``` + +Update namespace with new file content, merging lists and handling conflicts. + +**Arguments**: + +- `namespace` _dict | None_ - Existing namespace dictionary or None. +- `new_file` _dict_ - New file content to merge into namespace. + + +**Returns**: + +- `dict` - Updated namespace with merged content. + + +**Example**: + + >>> namespace = {'views': [{'name': 'view1'}]} + >>> new_file = {'views': [{'name': 'view2'}]} + >>> update_namespace(namespace, new_file) +- `{'views'` - [{'name': 'view1'}, {'name': 'view2'}]} + + + +#### file\_loader + +```python +def file_loader(file_path_input, rootdir_param, namespace=None) +``` + +Load LookML files and resolve includes recursively. + +**Arguments**: + +- `file_path_input` _str_ - File path pattern to load (supports glob patterns). +- `rootdir_param` _str | None_ - Root directory for resolving includes. +- `namespace` _dict | None_ - Existing namespace to merge content into. + + +**Returns**: + +- `dict` - Loaded LookML model with resolved includes. + + +**Raises**: + +- `FileNotFoundError` - If specified file path cannot be found. +- `ValueError` - If LookML file cannot be parsed. + + +**Example**: + + >>> namespace = file_loader('models/*.lkml', '/path/to/root') + >>> print(namespace['views'][0]['name']) + 'my_view' + + + +#### write\_single\_file + +```python +def write_single_file(cube_def: dict, + outputdir: str, + subdir: str = "cubes", + file_name: str = "my_cubes.yml") +``` + +Write a single cube definition to a YAML file. + +**Arguments**: + +- `cube_def` _dict_ - Cube definition to write. +- `outputdir` _str_ - Output directory path. +- `subdir` _str, optional_ - Subdirectory within output directory. Defaults to "cubes". +- `file_name` _str, optional_ - Name of the output file. Defaults to "my_cubes.yml". + + +**Raises**: + +- `OSError` - If output directory cannot be created or file cannot be written. + + +**Example**: + + >>> cube_def = {'cubes': [{'name': 'orders', 'sql_table': 'orders'}]} + >>> write_single_file(cube_def, '/output', 'cubes', 'orders.yml') + + + +#### write\_files + +```python +def write_files(cube_def, outputdir) +``` + +Write cube definitions to separate files organized by type. + +**Arguments**: + +- `cube_def` _dict_ - Cube definitions containing 'cubes' and/or 'views' keys. +- `outputdir` _str_ - Output directory path. + + +**Returns**: + +- `dict` - Summary of written files with structure: +- `{'cubes'` - [{'name': str, 'path': str}], 'views': [{'name': str, 'path': str}]} + + +**Raises**: + +- `Exception` - If no cube definition is provided. +- `OSError` - If output directory cannot be created or files cannot be written. + + +**Example**: + + >>> cube_def = {'cubes': [{'name': 'orders'}], 'views': [{'name': 'orders_view'}]} + >>> summary = write_files(cube_def, '/output') + >>> print(summary['cubes'][0]['name']) + 'orders' + + + +#### write\_lookml\_files + +```python +def write_lookml_files(lookml_model, outputdir) +``` + +Write LookML model to files in the output directory. + +**Arguments**: + +- `lookml_model` _dict_ - LookML model containing 'views' and/or 'explores' keys. +- `outputdir` _str_ - Output directory path. + + +**Returns**: + +- `dict` - Summary of written files with structure: +- `{'views'` - [{'name': str, 'path': str}], 'explores': [{'name': str, 'path': str}]} + + +**Raises**: + +- `Exception` - If no LookML model is provided. +- `OSError` - If output directory cannot be created or files cannot be written. + + +**Example**: + + >>> lookml_model = {'views': [{'name': 'orders'}], 'explores': [{'name': 'orders_explore'}]} + >>> summary = write_lookml_files(lookml_model, '/output') + >>> print(summary['views'][0]['name']) + 'orders' + + + +#### print\_summary + +```python +def print_summary(summary) +``` + +Print a formatted summary of generated files using Rich tables. + +**Arguments**: + +- `summary` _dict_ - Summary dictionary containing file information with keys + 'cubes', 'views', and/or 'explores', each containing lists of +- `{'name'` - str, 'path': str} dictionaries. + + +**Example**: + + >>> summary = {'cubes': [{'name': 'orders', 'path': '/output/cubes/orders.yml'}]} + >>> print_summary(summary) + # Displays a formatted table showing the generated files + diff --git a/docs/lkml2cube_parser_types.md b/docs/lkml2cube_parser_types.md new file mode 100644 index 0000000..f2b8f38 --- /dev/null +++ b/docs/lkml2cube_parser_types.md @@ -0,0 +1,149 @@ +# Table of Contents + +* [lkml2cube.parser.types](#lkml2cube.parser.types) + * [Console](#lkml2cube.parser.types.Console) + * [print](#lkml2cube.parser.types.Console.print) + * [folded\_unicode](#lkml2cube.parser.types.folded_unicode) + * [literal\_unicode](#lkml2cube.parser.types.literal_unicode) + * [folded\_unicode\_representer](#lkml2cube.parser.types.folded_unicode_representer) + * [literal\_unicode\_representer](#lkml2cube.parser.types.literal_unicode_representer) + + + +# lkml2cube.parser.types + + + +## Console Objects + +```python +class Console() +``` + +Simple console wrapper for printing messages. + +This class provides a basic print interface compatible with Rich console +while falling back to standard print functionality. + + + +#### print + +```python +def print(s, *args) +``` + +Print a message to the console. + +**Arguments**: + +- `s` _str_ - Message to print. +- `*args` - Additional arguments (currently ignored). + + +**Example**: + + >>> console = Console() + >>> console.print("Hello world", style="bold") + Hello world + + + +## folded\_unicode Objects + +```python +class folded_unicode(str) +``` + +String subclass for YAML folded scalar representation. + +This class marks strings that should be represented as folded scalars +in YAML output (using the '>' style). + +**Example**: + + >>> text = folded_unicode("This is a long + string that will be folded") + >>> # When dumped to YAML, will use '>' style + + + +## literal\_unicode Objects + +```python +class literal_unicode(str) +``` + +String subclass for YAML literal scalar representation. + +This class marks strings that should be represented as literal scalars +in YAML output (using the '|' style). + +**Example**: + + >>> sql = literal_unicode("SELECT * + FROM table + WHERE id = 1") + >>> # When dumped to YAML, will use '|' style preserving line breaks + + + +#### folded\_unicode\_representer + +```python +def folded_unicode_representer(dumper, data) +``` + +YAML representer for folded_unicode strings. + +**Arguments**: + +- `dumper` - YAML dumper instance. +- `data` _folded_unicode_ - String data to represent. + + +**Returns**: + + Scalar representation with folded style. + + +**Example**: + + >>> import yaml + >>> yaml.add_representer(folded_unicode, folded_unicode_representer) + >>> yaml.dump(folded_unicode("long text")) + '> long text + ' + + + +#### literal\_unicode\_representer + +```python +def literal_unicode_representer(dumper, data) +``` + +YAML representer for literal_unicode strings. + +**Arguments**: + +- `dumper` - YAML dumper instance. +- `data` _literal_unicode_ - String data to represent. + + +**Returns**: + + Scalar representation with literal style. + + +**Example**: + + >>> import yaml + >>> yaml.add_representer(literal_unicode, literal_unicode_representer) + >>> yaml.dump(literal_unicode("SELECT * + FROM table")) + '| + SELECT * + FROM table + ' + diff --git a/docs/lkml2cube_parser_views.md b/docs/lkml2cube_parser_views.md new file mode 100644 index 0000000..870f403 --- /dev/null +++ b/docs/lkml2cube_parser_views.md @@ -0,0 +1,55 @@ +# Table of Contents + +* [lkml2cube.parser.views](#lkml2cube.parser.views) + * [parse\_view](#lkml2cube.parser.views.parse_view) + + + +# lkml2cube.parser.views + + + +#### parse\_view + +```python +def parse_view(lookml_model, raise_when_views_not_present=True) +``` + +Parse LookML views into Cube definitions. + +Converts LookML view definitions into Cube format, handling dimensions, measures, +view inheritance, and various LookML-specific features like tiers and drill fields. + +**Arguments**: + +- `lookml_model` _dict_ - LookML model containing views to parse. +- `raise_when_views_not_present` _bool, optional_ - Whether to raise an exception + when no views are found. Defaults to True. + + +**Returns**: + +- `dict` - Cube definitions with structure: +- `{'cubes'` - [{'name': str, 'description': str, 'dimensions': list, 'measures': list, 'joins': list}]} + + +**Raises**: + +- `Exception` - If raise_when_views_not_present is True and no views are found, + or if required dimension properties are missing. + + +**Example**: + + >>> lookml_model = { + ... 'views': [{ + ... 'name': 'orders', + ... 'sql_table_name': 'public.orders', + ... 'dimensions': [{'name': 'id', 'type': 'number', 'sql': '${TABLE}.id'}], + ... 'measures': [{'name': 'count', 'type': 'count'}] + ... }] + ... } + >>> cube_def = parse_view(lookml_model) + >>> print(cube_def['cubes'][0]['name']) + 'orders' + diff --git a/lkml2cube/converter.py b/lkml2cube/converter.py index 51d9362..4175e89 100644 --- a/lkml2cube/converter.py +++ b/lkml2cube/converter.py @@ -304,6 +304,22 @@ def validate_files(self, file_paths: List[str]) -> Dict[str, bool]: results[file_path] = False return results + def clear_cache(self) -> None: + """Clear the global file loader cache. + + This method clears the visited_path cache used by the file_loader to prevent + circular includes. Useful for ensuring clean state between operations or + in testing scenarios. + + Example: + >>> converter = LookMLConverter() + >>> converter.cubes("models/orders.lkml") # Populates cache + >>> converter.clear_cache() # Clears cache + >>> converter.cubes("models/orders.lkml") # Loads fresh from disk + """ + from lkml2cube.parser import loader + loader.visited_path.clear() + def __repr__(self) -> str: """Return string representation of the converter.""" return ( diff --git a/scripts/generate_docs.py b/scripts/generate_docs.py index 761509c..e49b98d 100644 --- a/scripts/generate_docs.py +++ b/scripts/generate_docs.py @@ -13,30 +13,32 @@ def generate_module_docs(): # Ensure docs directory exists os.makedirs("docs", exist_ok=True) - + # Add current directory to Python path current_dir = os.path.abspath(".") if current_dir not in sys.path: sys.path.insert(0, current_dir) # Try both pydoc-markdown and fallback to manual generation - modules = ["lkml2cube.parser", "lkml2cube.converter", "lkml2cube.main"] - + modules = [ + "lkml2cube.parser.cube_api", + "lkml2cube.parser.explores", + "lkml2cube.parser.loader", + "lkml2cube.parser.types", + "lkml2cube.parser.views", + "lkml2cube.converter", + "lkml2cube.main", + ] + for module in modules: output_file = f"docs/{module.replace('.', '_')}.md" print(f"Generating documentation for {module}...") - + # First try pydoc-markdown if try_pydoc_markdown(module, output_file): print(f" ✓ Created {output_file} using pydoc-markdown") - continue - - # Fallback to manual generation - try: - generate_manual_docs(module, output_file) - print(f" ✓ Created {output_file} using manual generation") - except Exception as e: - print(f" ✗ Error generating docs for {module}: {e}") + else: + print(f" ✗ Error generating docs for {module}") def try_pydoc_markdown(module, output_file): @@ -44,23 +46,15 @@ def try_pydoc_markdown(module, output_file): try: # Create a temporary config file for this module config = { - "loaders": [ - { - "type": "python", - "search_path": ["."], - "modules": [module] - } - ], + "loaders": [{"type": "python", "search_path": ["."], "modules": [module]}], "renderer": { "type": "markdown", "render_toc": True, "render_module_header": True, - "markdown": { - "filename": output_file - } - } + "filename": output_file, + }, } - + config_file = f".pydoc-markdown-{module.replace('.', '_')}.yml" with open(config_file, "w") as f: yaml.dump(config, f) @@ -68,18 +62,15 @@ def try_pydoc_markdown(module, output_file): # Run pydoc-markdown with the config file env = os.environ.copy() env["PYTHONPATH"] = os.path.abspath(".") + ":" + env.get("PYTHONPATH", "") - + result = subprocess.run( - ["pydoc-markdown", config_file], - capture_output=True, - text=True, - env=env + ["pydoc-markdown", config_file], capture_output=True, text=True, env=env ) - + # Clean up temporary config file if os.path.exists(config_file): os.remove(config_file) - + if result.returncode == 0 and os.path.exists(output_file): optimize_for_llm(output_file) return True @@ -89,109 +80,6 @@ def try_pydoc_markdown(module, output_file): return False -def generate_manual_docs(module_name, output_file): - """Generate documentation manually using Python introspection.""" - try: - # Import the module - module = importlib.import_module(module_name) - - # Start building the documentation - lines = [] - lines.append(f"# {module_name}") - lines.append("") - - # Add module docstring if available - if module.__doc__: - lines.append(module.__doc__.strip()) - lines.append("") - - # Handle different module types - if hasattr(module, "__path__"): # It's a package - # Get all submodules - submodules = [] - for importer, modname, ispkg in pkgutil.iter_modules(module.__path__, module_name + "."): - try: - submodule = importlib.import_module(modname) - submodules.append((modname, submodule)) - except ImportError: - continue - - if submodules: - lines.append("## Submodules") - lines.append("") - for modname, submodule in submodules: - lines.append(f"### {modname}") - if submodule.__doc__: - lines.append(submodule.__doc__.strip()) - lines.append("") - - # Add classes and functions from submodule - add_module_content(submodule, lines) - lines.append("") - else: - # It's a regular module - add_module_content(module, lines) - - # Write to file - with open(output_file, "w") as f: - f.write("\n".join(lines)) - - # Optimize for LLM consumption - optimize_for_llm(output_file) - - except Exception as e: - raise Exception(f"Failed to generate manual docs for {module_name}: {e}") - - -def add_module_content(module, lines): - """Add classes and functions from a module to the documentation.""" - # Get all classes - classes = [] - functions = [] - - for name, obj in inspect.getmembers(module): - if name.startswith("_"): - continue - if inspect.isclass(obj) and obj.__module__ == module.__name__: - classes.append((name, obj)) - elif inspect.isfunction(obj) and obj.__module__ == module.__name__: - functions.append((name, obj)) - - # Document classes - if classes: - lines.append("#### Classes") - lines.append("") - for name, cls in classes: - lines.append(f"##### {name}") - if cls.__doc__: - lines.append(cls.__doc__.strip()) - lines.append("") - - # Add methods - methods = [] - for method_name, method in inspect.getmembers(cls): - if (method_name.startswith("_") and method_name != "__init__") or not inspect.isfunction(method): - continue - methods.append((method_name, method)) - - if methods: - lines.append("**Methods:**") - lines.append("") - for method_name, method in methods: - lines.append(f"- `{method_name}()`: {method.__doc__.strip().split('.')[0] if method.__doc__ else 'No description'}") - lines.append("") - - # Document functions - if functions: - lines.append("#### Functions") - lines.append("") - for name, func in functions: - lines.append(f"##### {name}()") - if func.__doc__: - lines.append(func.__doc__.strip()) - lines.append("") - - def optimize_for_llm(filepath): """Compress documentation for optimal LLM consumption.""" with open(filepath, "r") as f: diff --git a/tests/test_converter.py b/tests/test_converter.py index 02fd3e5..df739d5 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -532,4 +532,84 @@ def test_multiple_converter_instances(self): # Configuration should be independent converter1.set_config(parseonly=True) assert converter1.parseonly is True - assert converter2.parseonly is False \ No newline at end of file + assert converter2.parseonly is False + + +class TestLookMLConverterCacheManagement: + """Test cache management functionality.""" + + def test_clear_cache_method_exists(self): + """Test that clear_cache method exists and can be called.""" + converter = LookMLConverter() + + # Method should exist and be callable + assert hasattr(converter, 'clear_cache') + assert callable(converter.clear_cache) + + # Should not raise any errors + converter.clear_cache() + + def test_clear_cache_actually_clears_visited_path(self): + """Test that clear_cache actually clears the visited_path cache.""" + from lkml2cube.parser import loader + + # Set up some dummy cache data + loader.visited_path['test_file.lkml'] = True + loader.visited_path['another_file.lkml'] = True + + # Verify cache has data + assert len(loader.visited_path) == 2 + assert 'test_file.lkml' in loader.visited_path + assert 'another_file.lkml' in loader.visited_path + + # Clear cache using converter method + converter = LookMLConverter() + converter.clear_cache() + + # Verify cache is cleared + assert len(loader.visited_path) == 0 + assert 'test_file.lkml' not in loader.visited_path + assert 'another_file.lkml' not in loader.visited_path + + def test_clear_cache_with_empty_cache(self): + """Test that clear_cache works when cache is already empty.""" + from lkml2cube.parser import loader + + # Ensure cache starts empty + loader.visited_path.clear() + assert len(loader.visited_path) == 0 + + # Clear cache - should not raise any errors + converter = LookMLConverter() + converter.clear_cache() + + # Cache should still be empty + assert len(loader.visited_path) == 0 + + @patch('lkml2cube.converter.file_loader') + def test_clear_cache_integration_with_file_loading(self, mock_file_loader): + """Test that clear_cache works in integration with file loading operations.""" + from lkml2cube.parser import loader + + # Mock the file loader to return a simple model + sample_lookml = {'views': [{'name': 'test_view'}]} + mock_file_loader.return_value = sample_lookml + + converter = LookMLConverter(parseonly=True) + + # First call should populate cache (through file_loader) + converter.cubes("test.lkml") + + # Manually add something to cache to simulate file_loader behavior + loader.visited_path['test.lkml'] = True + assert len(loader.visited_path) == 1 + + # Clear cache + converter.clear_cache() + + # Cache should be empty + assert len(loader.visited_path) == 0 + + # Another call should work normally + result = converter.cubes("test.lkml") + assert result['lookml_model'] == sample_lookml \ No newline at end of file diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 93f080e..b4914a3 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -12,6 +12,12 @@ class TestExamples: + def setup_method(self): + """Set up test fixtures.""" + # Clear the global visited_path cache to prevent interference between tests + from lkml2cube.parser import loader + loader.visited_path.clear() + def test_simple_view(self): file_path = "lkml/views/orders.view.lkml" # print(join(rootdir, file_path))