From 701363cccfbf56476b38b37863e96dfa7ed2f487 Mon Sep 17 00:00:00 2001 From: Michele Lamarca Date: Mon, 1 Sep 2025 21:27:04 +0200 Subject: [PATCH 1/2] add checks for parameter uc_catalog_name --- demo/README.md | 20 +++++++++++++++----- demo/generate_dabs_resources.py | 3 +++ demo/launch_acfs_demo.py | 3 +++ demo/launch_af_cloudfiles_demo.py | 4 ++++ demo/launch_af_eventhub_demo.py | 3 +++ demo/launch_dais_demo.py | 3 +++ demo/launch_dlt_sink_demo.py | 4 ++++ demo/launch_silver_fanout_demo.py | 3 +++ demo/launch_techsummit_demo.py | 4 ++++ 9 files changed, 42 insertions(+), 5 deletions(-) diff --git a/demo/README.md b/demo/README.md index e6be559d..f68d6814 100644 --- a/demo/README.md +++ b/demo/README.md @@ -48,7 +48,7 @@ This Demo launches Bronze and Silver pipelines with following activities: 7. ```commandline python demo/launch_dais_demo.py --uc_catalog_name=<> --profile=<> ``` - - uc_catalog_name : Unity catalog name + - uc_catalog_name : UC catalog name. Can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. - you can provide `--profile=databricks_profile name` in case you already have databricks cli otherwise command prompt will ask host and token. ![dais_demo.png](../docs/static/images/dais_demo.png) @@ -86,7 +86,7 @@ This demo will launch auto generated tables(100s) inside single bronze and silve 7. ```commandline python demo/launch_techsummit_demo.py --uc_catalog_name=<> --profile=<> ``` - - uc_catalog_name : Unity catalog name + - uc_catalog_name : UC catalog name. Can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. - you can provide `--profile=databricks_profile name` in case you already have databricks cli otherwise command prompt will ask host and token ![tech_summit_demo.png](../docs/static/images/tech_summit_demo.png) @@ -128,7 +128,7 @@ This demo will perform following tasks: 7. ```commandline python demo/launch_af_cloudfiles_demo.py --uc_catalog_name=<> --source=cloudfiles --profile=<> ``` - - uc_catalog_name : Unity Catalog name + - uc_catalog_name : UC catalog name. Can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. - you can provide `--profile=databricks_profile name` in case you already have databricks cli otherwise command prompt will ask host and token ![af_am_demo.png](../docs/static/images/af_am_demo.png) @@ -178,7 +178,7 @@ This demo will perform following tasks: - Create databricks secrets to store producer and consumer keys using the scope created in step 2 - Following are the mandatory arguments for running EventHubs demo - - uc_catalog_name : unity catalog name e.g. ravi_dlt_meta_uc + - uc_catalog_name : UC catalog name. Can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. - eventhub_namespace: Eventhub namespace e.g. dltmeta - eventhub_name : Primary Eventhubname e.g. dltmeta_demo - eventhub_name_append_flow: Secondary eventhub name for appendflow feed e.g. dltmeta_demo_af @@ -231,7 +231,8 @@ This demo will perform following tasks: ```commandline python demo/launch_silver_fanout_demo.py --source=cloudfiles --uc_catalog_name=<> --profile=<> ``` - + + - uc_catalog_name : UC catalog name. Can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. - you can provide `--profile=databricks_profile name` in case you already have databricks cli otherwise command prompt will ask host and token. a. Databricks Workspace URL: @@ -296,6 +297,9 @@ This demo will perform following tasks: ```commandline python demo/launch_acfs_demo.py --uc_catalog_name=<> --profile=<> ``` + - uc_catalog_name : UC catalog name. Can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. + - you can provide `--profile=databricks_profile name` in case you already have databricks cli otherwise command prompt will ask host and token. + ![acfs.png](../docs/static/images/acfs.png) # Lakeflow Declarative Pipelines Sink Demo @@ -350,6 +354,10 @@ This demo will perform following tasks: ```commandline python demo/launch_dlt_sink_demo.py --uc_catalog_name=<> --source=kafka --kafka_source_topic=<>>> --kafka_sink_topic=<> --kafka_source_servers_secrets_scope_name=<> --kafka_source_servers_secrets_scope_key=<> --kafka_sink_servers_secret_scope_name=<> --kafka_sink_servers_secret_scope_key=<> --profile=<> ``` + + - uc_catalog_name : UC catalog name. Can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. + - you can provide `--profile=databricks_profile name` in case you already have databricks cli otherwise command prompt will ask host and token. + ![dlt_demo_sink.png](../docs/static/images/dlt_demo_sink.png) ![dlt_delta_sink.png](../docs/static/images/dlt_delta_sink.png) ![dlt_kafka_sink.png](../docs/static/images/dlt_kafka_sink.png) @@ -406,6 +414,8 @@ This demo will perform following steps: ```commandline python demo/generate_dabs_resources.py --source=cloudfiles --uc_catalog_name= --profile= ``` + > Note: uc_catalog_name : UC catalog name. Can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. + > Note: If you don't specify `--profile`, you'll be prompted for your Databricks workspace URL and access token. 7. Deploy and run the DAB bundle: diff --git a/demo/generate_dabs_resources.py b/demo/generate_dabs_resources.py index f18e4f83..f9f52be1 100644 --- a/demo/generate_dabs_resources.py +++ b/demo/generate_dabs_resources.py @@ -69,6 +69,9 @@ def init_runner_conf(self) -> DLTMetaRunnerConf: f"{run_id}/demo-output.csv" ), ) + if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): + print("\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting.") + exit(1) return runner_conf diff --git a/demo/launch_acfs_demo.py b/demo/launch_acfs_demo.py index bd28d5e7..880d3d6e 100644 --- a/demo/launch_acfs_demo.py +++ b/demo/launch_acfs_demo.py @@ -55,6 +55,9 @@ def init_runner_conf(self) -> DLTMetaRunnerConf: env="demo" ) runner_conf.uc_catalog_name = self.args['uc_catalog_name'] + if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): + print("\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting.") + exit(1) return runner_conf def launch_workflow(self, runner_conf: DLTMetaRunnerConf): diff --git a/demo/launch_af_cloudfiles_demo.py b/demo/launch_af_cloudfiles_demo.py index 98fdcbf0..67c5c40b 100644 --- a/demo/launch_af_cloudfiles_demo.py +++ b/demo/launch_af_cloudfiles_demo.py @@ -67,6 +67,10 @@ def init_runner_conf(self) -> DLTMetaRunnerConf: ), ) + if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): + print("\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting.") + exit(1) + return runner_conf def launch_workflow(self, runner_conf: DLTMetaRunnerConf): diff --git a/demo/launch_af_eventhub_demo.py b/demo/launch_af_eventhub_demo.py index 9b16f243..8c1e91e5 100644 --- a/demo/launch_af_eventhub_demo.py +++ b/demo/launch_af_eventhub_demo.py @@ -71,6 +71,9 @@ def init_runner_conf(self) -> DLTMetaRunnerConf: eventhub_port=self.args["eventhub_port"] ) runner_conf.uc_catalog_name = self.args['uc_catalog_name'] + if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): + print("\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting.") + exit(1) runner_conf.runners_full_local_path = 'demo/notebooks/afam_eventhub_runners' return runner_conf diff --git a/demo/launch_dais_demo.py b/demo/launch_dais_demo.py index ad5c9846..690680bd 100644 --- a/demo/launch_dais_demo.py +++ b/demo/launch_dais_demo.py @@ -52,6 +52,9 @@ def init_runner_conf(self) -> DLTMetaRunnerConf: ) if self.args['uc_catalog_name']: runner_conf.uc_catalog_name = self.args['uc_catalog_name'] + if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): + print("\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting.") + exit(1) runner_conf.uc_volume_name = f"{runner_conf.uc_catalog_name}_dais_demo_{run_id}" return runner_conf diff --git a/demo/launch_dlt_sink_demo.py b/demo/launch_dlt_sink_demo.py index 2fb53083..3e60eaa5 100644 --- a/demo/launch_dlt_sink_demo.py +++ b/demo/launch_dlt_sink_demo.py @@ -70,6 +70,10 @@ def init_runner_conf(self) -> DLTMetaRunnerConf: ), ) + if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): + print("\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting.") + exit(1) + return runner_conf def launch_workflow(self, runner_conf: DLTMetaRunnerConf): diff --git a/demo/launch_silver_fanout_demo.py b/demo/launch_silver_fanout_demo.py index d075f271..30da8ddb 100644 --- a/demo/launch_silver_fanout_demo.py +++ b/demo/launch_silver_fanout_demo.py @@ -83,6 +83,9 @@ def init_runner_conf(self) -> DLTMetaRunnerConf: env="demo" ) runner_conf.uc_catalog_name = self.args['uc_catalog_name'] + if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): + print("\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting.") + exit(1) runner_conf.uc_volume_name = f"{runner_conf.uc_catalog_name}_dlt_meta_fout_demo_{run_id}" return runner_conf diff --git a/demo/launch_techsummit_demo.py b/demo/launch_techsummit_demo.py index 94d76e29..3eeefb5c 100644 --- a/demo/launch_techsummit_demo.py +++ b/demo/launch_techsummit_demo.py @@ -102,7 +102,11 @@ def init_runner_conf(self) -> TechsummitRunnerConf: ) if self.args['uc_catalog_name']: runner_conf.uc_catalog_name = self.args['uc_catalog_name'] + if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): + print("\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting.") + exit(1) runner_conf.uc_volume_name = f"{self.args['uc_catalog_name']}_volume_{run_id}" + return runner_conf def create_bronze_silver_dlt(self, runner_conf: DLTMetaRunnerConf): From 3fa02c4aba05e6484a216d68cae3901e03824f29 Mon Sep 17 00:00:00 2001 From: Michele Lamarca Date: Mon, 8 Sep 2025 18:30:49 +0200 Subject: [PATCH 2/2] fix linting --- demo/generate_dabs_resources.py | 5 ++++- demo/launch_acfs_demo.py | 5 ++++- demo/launch_af_cloudfiles_demo.py | 5 ++++- demo/launch_af_eventhub_demo.py | 5 ++++- demo/launch_dais_demo.py | 5 ++++- demo/launch_dlt_sink_demo.py | 5 ++++- demo/launch_silver_fanout_demo.py | 5 ++++- demo/launch_techsummit_demo.py | 5 ++++- 8 files changed, 32 insertions(+), 8 deletions(-) diff --git a/demo/generate_dabs_resources.py b/demo/generate_dabs_resources.py index f9f52be1..2b49be5a 100644 --- a/demo/generate_dabs_resources.py +++ b/demo/generate_dabs_resources.py @@ -70,7 +70,10 @@ def init_runner_conf(self) -> DLTMetaRunnerConf: ), ) if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): - print("\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting.") + print( + "\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z')," + " digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting." + ) exit(1) return runner_conf diff --git a/demo/launch_acfs_demo.py b/demo/launch_acfs_demo.py index 880d3d6e..5473c131 100644 --- a/demo/launch_acfs_demo.py +++ b/demo/launch_acfs_demo.py @@ -56,7 +56,10 @@ def init_runner_conf(self) -> DLTMetaRunnerConf: ) runner_conf.uc_catalog_name = self.args['uc_catalog_name'] if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): - print("\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting.") + print( + "\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z')," + " digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting." + ) exit(1) return runner_conf diff --git a/demo/launch_af_cloudfiles_demo.py b/demo/launch_af_cloudfiles_demo.py index 67c5c40b..9ce0196a 100644 --- a/demo/launch_af_cloudfiles_demo.py +++ b/demo/launch_af_cloudfiles_demo.py @@ -68,7 +68,10 @@ def init_runner_conf(self) -> DLTMetaRunnerConf: ) if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): - print("\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting.") + print( + "\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z')," + " digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting." + ) exit(1) return runner_conf diff --git a/demo/launch_af_eventhub_demo.py b/demo/launch_af_eventhub_demo.py index 8c1e91e5..1bced1bc 100644 --- a/demo/launch_af_eventhub_demo.py +++ b/demo/launch_af_eventhub_demo.py @@ -72,7 +72,10 @@ def init_runner_conf(self) -> DLTMetaRunnerConf: ) runner_conf.uc_catalog_name = self.args['uc_catalog_name'] if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): - print("\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting.") + print( + "\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z')," + " digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting." + ) exit(1) runner_conf.runners_full_local_path = 'demo/notebooks/afam_eventhub_runners' return runner_conf diff --git a/demo/launch_dais_demo.py b/demo/launch_dais_demo.py index 690680bd..e6d99cf1 100644 --- a/demo/launch_dais_demo.py +++ b/demo/launch_dais_demo.py @@ -53,7 +53,10 @@ def init_runner_conf(self) -> DLTMetaRunnerConf: if self.args['uc_catalog_name']: runner_conf.uc_catalog_name = self.args['uc_catalog_name'] if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): - print("\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting.") + print( + "\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z')," + " digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting." + ) exit(1) runner_conf.uc_volume_name = f"{runner_conf.uc_catalog_name}_dais_demo_{run_id}" diff --git a/demo/launch_dlt_sink_demo.py b/demo/launch_dlt_sink_demo.py index 3e60eaa5..f3899d49 100644 --- a/demo/launch_dlt_sink_demo.py +++ b/demo/launch_dlt_sink_demo.py @@ -71,7 +71,10 @@ def init_runner_conf(self) -> DLTMetaRunnerConf: ) if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): - print("\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting.") + print( + "\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z')," + " digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting." + ) exit(1) return runner_conf diff --git a/demo/launch_silver_fanout_demo.py b/demo/launch_silver_fanout_demo.py index 30da8ddb..639235b3 100644 --- a/demo/launch_silver_fanout_demo.py +++ b/demo/launch_silver_fanout_demo.py @@ -84,7 +84,10 @@ def init_runner_conf(self) -> DLTMetaRunnerConf: ) runner_conf.uc_catalog_name = self.args['uc_catalog_name'] if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): - print("\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting.") + print( + "\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z')," + " digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting." + ) exit(1) runner_conf.uc_volume_name = f"{runner_conf.uc_catalog_name}_dlt_meta_fout_demo_{run_id}" return runner_conf diff --git a/demo/launch_techsummit_demo.py b/demo/launch_techsummit_demo.py index 3eeefb5c..d680ecd1 100644 --- a/demo/launch_techsummit_demo.py +++ b/demo/launch_techsummit_demo.py @@ -103,7 +103,10 @@ def init_runner_conf(self) -> TechsummitRunnerConf: if self.args['uc_catalog_name']: runner_conf.uc_catalog_name = self.args['uc_catalog_name'] if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): - print("\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting.") + print( + "\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z')," + " digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting." + ) exit(1) runner_conf.uc_volume_name = f"{self.args['uc_catalog_name']}_volume_{run_id}"