Merge pull request #202 from databrickslabs/feature/v0.0.10

Feature/v0.0.10

Author: pohlposition

.coveragerc

@@ -1,6 +1,6 @@
 [run]
 branch = True
-command_line = -m unittest
+command_line = -m pytest tests/
 include = src/*.py
 omit =
     */site-packages/*

.github/workflows/onpush.yml

@@ -18,6 +18,12 @@ jobs:
     steps:
       - uses: actions/checkout@v1
 
+      - name: Set up Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '11'
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
@@ -40,25 +46,20 @@ jobs:
       - name: Lint
         run: flake8        
 
-      - name: set spark local
-        run: export SPARK_LOCAL_IP=127.0.0.1
-
-      - name: set spark executor memory
-        run: export SPARK_EXECUTOR_MEMORY=8g  
-      
-      - name: set spark driver memory
-        run: export SPARK_DRIVER_MEMORY=8g
-
-      - name: set javaopts
-        run: export JAVA_OPTS="-Xmx10g -XX:+UseG1GC"
+      - name: Set environment variables
+        run: |
+          echo "SPARK_LOCAL_IP=127.0.0.1" >> $GITHUB_ENV
+          echo "SPARK_EXECUTOR_MEMORY=8g" >> $GITHUB_ENV
+          echo "SPARK_DRIVER_MEMORY=8g" >> $GITHUB_ENV
+          echo "JAVA_OPTS=-Xmx10g -XX:+UseG1GC" >> $GITHUB_ENV
 
       - name: Print System Information
         run: | 
                python -c "import psutil; import os;
                print(f'Physical Memory: {psutil.virtual_memory().total / 1e9:.2f} GB'); print(f'CPU Cores: {os.cpu_count()}')"
  
       - name: Run Unit Tests
-        run: python -m coverage run
+        run: python -m coverage run -m pytest tests/ -v
 
       - name: Publish test coverage
         if: startsWith(matrix.os,'ubuntu')

.gitignore

@@ -156,6 +156,4 @@ demo/conf/onboarding.json
 integration_tests/conf/onboarding*.json
 demo/conf/onboarding*.json
 integration_test_output*.csv
-databricks.yml
-oboarding_job_details.json
-
+onboarding_job_details.json

CHANGELOG.md

@@ -1,4 +1,25 @@
 # Changelog
+
+## [v0.0.10]
+### Added
+- Added apply_changes_from_snapshot support in silver layer [PR](https://github.com/databrickslabs/dlt-meta/pull/187)
+- Added UI using databricks lakehouse app for onboarding/deploy commands [PR](https://github.com/databrickslabs/dlt-meta/pull/168)
+- Added support for non-Delta as sinks(delta, kafka) [PR](https://github.com/databrickslabs/dlt-meta/pull/157)
+- Added quarantine support in silver layer for data quality rules [PR](https://github.com/databrickslabs/dlt-meta/pull/191)
+- Added support for table comments, column comments, and cluster_by [PR](https://github.com/databrickslabs/dlt-meta/pull/91)
+- Added catalog support for sourceDetails and targetDetails [PR](https://github.com/databrickslabs/dlt-meta/issues/173)
+- Added DBDemos for dlt-meta [PR](https://github.com/databrickslabs/dlt-meta/issues/183)
+- Added YAML support for onboarding [PR](https://github.com/databrickslabs/dlt-meta/issues/184)
+- Fixed issue cluster by not working with bronze append only table [PR](https://github.com/databrickslabs/dlt-meta/issues/197)
+- Fixed issue view name containing period when using DPM [PR](https://github.com/databrickslabs/dlt-meta/issues/169)
+- Fixed issue CLI onboarding overwrite option always set to True [PR](https://github.com/databrickslabs/dlt-meta/issues/163)
+- Fixed issue Silver DLT not creating based on passed database [PR](https://github.com/databrickslabs/dlt-meta/issues/160)
+- Fixed issue PyPI download stats display [PR](https://github.com/databrickslabs/dlt-meta/issues/200)
+- Fixed issue Silver Data Quality not working [PR](https://github.com/databrickslabs/dlt-meta/issues/156)
+- Fixed issue Removed DPM flag check inside dataflowpipeline [PR](https://github.com/databrickslabs/dlt-meta/issues/177)
+- Fixed issue Updated dlt-meta demos into Delta Live Tables Notebook github [PR](https://github.com/databrickslabs/dlt-meta/issues/158)
+
+
 ## [v.0.0.9] 
 - Added  apply_changes_from_snapshot api support in bronze layer: [PR](https://github.com/databrickslabs/dlt-meta/pull/124)
 - Added dlt append_flow api support for silver layer: [PR](https://github.com/databrickslabs/dlt-meta/pull/63)

README.md

@@ -11,38 +11,15 @@
 
 ---
 
-<p align="left">
-    <a href="https://databrickslabs.github.io/dlt-meta/">
-        <img src="https://img.shields.io/badge/DOCS-PASSING-green?style=for-the-badge" alt="Documentation Status"/>
-    </a>
-    <a href="https://pypi.org/project/dlt-meta/">
-        <img src="https://img.shields.io/badge/PYPI-v%200.0.9-green?style=for-the-badge" alt="Latest Python Release"/>
-    </a>
-    <a href="https://github.com/databrickslabs/dlt-meta/actions/workflows/onpush.yml">
-        <img src="https://img.shields.io/github/workflow/status/databrickslabs/dlt-meta/build/main?style=for-the-badge"
-             alt="GitHub Workflow Status (branch)"/>
-    </a>
-    <a href="https://codecov.io/gh/databrickslabs/dlt-meta">
-        <img src="https://img.shields.io/codecov/c/github/databrickslabs/dlt-meta?style=for-the-badge&amp;token=2CxLj3YBam"
-             alt="codecov"/>
-    </a>
-    <a href="https://pypistats.org/packages/dl-meta">
-        <img src="https://img.shields.io/pypi/dm/dlt-meta?style=for-the-badge" alt="downloads"/>
-    </a>
-    <a href="https://github.com/PyCQA/flake8">
-        <img src="https://img.shields.io/badge/FLAKE8-FLAKE8-lightgrey?style=for-the-badge"
-             alt="We use flake8 for formatting"/>
-    </a>
-</p>
-
-[![lines of code](https://tokei.rs/b1/github/databrickslabs/dlt-meta)](<[https://codecov.io/github/databrickslabs/dlt-meta](https://github.com/databrickslabs/dlt-meta)>)
+[![Documentation](https://img.shields.io/badge/docs-passing-green)](https://databrickslabs.github.io/dlt-meta/) [![PyPI](https://img.shields.io/badge/pypi-v0.0.9-green)](https://pypi.org/project/dlt-meta/) [![Build](https://img.shields.io/github/workflow/status/databrickslabs/dlt-meta/build/main)](https://github.com/databrickslabs/dlt-meta/actions/workflows/onpush.yml) [![Coverage](https://img.shields.io/codecov/c/github/databrickslabs/dlt-meta)](https://codecov.io/gh/databrickslabs/dlt-meta) [![Style](https://img.shields.io/badge/code%20style-flake8-blue)](https://github.com/PyCQA/flake8) [![PyPI Downloads](https://static.pepy.tech/badge/dlt-meta/month)](https://pepy.tech/projects/dlt-meta)
 
 ---
 
+
 # Project Overview
-`DLT-META` is a metadata-driven framework designed to work with [Delta Live Tables](https://www.databricks.com/product/delta-live-tables). This framework enables the automation of bronze and silver data pipelines by leveraging metadata recorded in an onboarding JSON file. This file, known as the Dataflowspec, serves as the data flow specification, detailing the source and target metadata required for the pipelines.
+`DLT-META` is a metadata-driven framework designed to work with [Lakeflow Declarative Pipelines](https://www.databricks.com/product/data-engineering/lakeflow-declarative-pipelines). This framework enables the automation of bronze and silver data pipelines by leveraging metadata recorded in an onboarding JSON file. This file, known as the Dataflowspec, serves as the data flow specification, detailing the source and target metadata required for the pipelines.
 
-In practice, a single generic DLT pipeline reads the Dataflowspec and uses it to orchestrate and run the necessary data processing workloads. This approach streamlines the development and management of data pipelines, allowing for a more efficient and scalable data processing workflow
+In practice, a single generic pipeline reads the Dataflowspec and uses it to orchestrate and run the necessary data processing workloads. This approach streamlines the development and management of data pipelines, allowing for a more efficient and scalable data processing workflow
 
 ### Components:
 
@@ -82,6 +59,8 @@ In practice, a single generic DLT pipeline reads the Dataflowspec and uses it to
 | Liquid cluster support | Bronze, Bronze Quarantine, Silver tables|
 | [DLT-META CLI](https://databrickslabs.github.io/dlt-meta/getting_started/dltmeta_cli/) |  ```databricks labs dlt-meta onboard```, ```databricks labs dlt-meta deploy``` |
 | Bronze and Silver pipeline chaining | Deploy dlt-meta pipeline with ```layer=bronze_silver``` option using Direct publishing mode |
+| [DLT Sinks](https://docs.databricks.com/aws/en/delta-live-tables/dlt-sinks) |Supported formats:external ```delta table```, ```kafka```.Bronze, Silver layers|
+| [Databricks Asset Bundles](https://docs.databricks.com/aws/en/dev-tools/bundles/) | Supported
 
 ## Getting Started
 
@@ -121,36 +100,47 @@ databricks auth login --host WORKSPACE_HOST
 
 If you want to run existing demo files please follow these steps before running onboard command:
 
-```commandline
+1. Clone dlt-meta:
+    ```commandline
     git clone https://github.com/databrickslabs/dlt-meta.git
-```
+    ```
 
-```commandline
+2. Navigate to project directory:
+    ```commandline
     cd dlt-meta
-```
+    ```
 
-```commandline
+3. Create Python virtual environment:
+    ```commandline
     python -m venv .venv
-```
+    ```
 
-```commandline
+4. Activate virtual environment:
+    ```commandline
     source .venv/bin/activate
-```
+    ```
 
-```commandline
-    pip install databricks-sdk
-```
+5. Install required packages:
+    ```commandline
+    # Core requirements
+    pip install "PyYAML>=6.0" setuptools databricks-sdk
+    
+    # Development requirements
+    pip install delta-spark==3.0.0 pyspark==3.5.5 pytest>=7.0.0 coverage>=7.0.0
+    
+    # Integration test requirements
+    pip install "typer[all]==0.6.1"
+    ```
 
-```commandline
+6. Set environment variables:
+    ```commandline
     dlt_meta_home=$(pwd)
-```
-
-```commandline
     export PYTHONPATH=$dlt_meta_home
-```
-```commandline
+    ```
+7. Run onboarding command:
+    ```commandline
     databricks labs dlt-meta onboard
-```
+    ```
 ![onboardingDLTMeta.gif](docs/static/images/onboardingDLTMeta.gif)