diff --git a/demo/README.md b/demo/README.md index f39f35d..463b3ae 100644 --- a/demo/README.md +++ b/demo/README.md @@ -48,7 +48,7 @@ This Demo launches Bronze and Silver pipelines with following activities: 7. ```commandline python demo/launch_dais_demo.py --uc_catalog_name=<> --profile=<> ``` - - uc_catalog_name : Unity catalog name + - uc_catalog_name : UC catalog name. Can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. - you can provide `--profile=databricks_profile name` in case you already have databricks cli otherwise command prompt will ask host and token. ![dais_demo.png](../docs/static/images/dais_demo.png) @@ -86,7 +86,7 @@ This demo will launch auto generated tables(100s) inside single bronze and silve 7. ```commandline python demo/launch_techsummit_demo.py --uc_catalog_name=<> --profile=<> ``` - - uc_catalog_name : Unity catalog name + - uc_catalog_name : UC catalog name. Can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. - you can provide `--profile=databricks_profile name` in case you already have databricks cli otherwise command prompt will ask host and token ![tech_summit_demo.png](../docs/static/images/tech_summit_demo.png) @@ -128,7 +128,7 @@ This demo will perform following tasks: 7. ```commandline python demo/launch_af_cloudfiles_demo.py --uc_catalog_name=<> --source=cloudfiles --profile=<> ``` - - uc_catalog_name : Unity Catalog name + - uc_catalog_name : UC catalog name. Can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. - you can provide `--profile=databricks_profile name` in case you already have databricks cli otherwise command prompt will ask host and token ![af_am_demo.png](../docs/static/images/af_am_demo.png) @@ -178,7 +178,7 @@ This demo will perform following tasks: - Create databricks secrets to store producer and consumer keys using the scope created in step 2 - Following are the mandatory arguments for running EventHubs demo - - uc_catalog_name : unity catalog name e.g. ravi_dlt_meta_uc + - uc_catalog_name : UC catalog name. Can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. - eventhub_namespace: Eventhub namespace e.g. dltmeta - eventhub_name : Primary Eventhubname e.g. dltmeta_demo - eventhub_name_append_flow: Secondary eventhub name for appendflow feed e.g. dltmeta_demo_af @@ -231,7 +231,8 @@ This demo will perform following tasks: ```commandline python demo/launch_silver_fanout_demo.py --source=cloudfiles --uc_catalog_name=<> --profile=<> ``` - + + - uc_catalog_name : UC catalog name. Can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. - you can provide `--profile=databricks_profile name` in case you already have databricks cli otherwise command prompt will ask host and token. a. Databricks Workspace URL: @@ -296,6 +297,9 @@ This demo will perform following tasks: ```commandline python demo/launch_acfs_demo.py --uc_catalog_name=<> --profile=<> ``` + - uc_catalog_name : UC catalog name. Can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. + - you can provide `--profile=databricks_profile name` in case you already have databricks cli otherwise command prompt will ask host and token. + ![acfs.png](../docs/static/images/acfs.png) # Lakeflow Declarative Pipelines Sink Demo @@ -350,6 +354,10 @@ This demo will perform following tasks: ```commandline python demo/launch_dlt_sink_demo.py --uc_catalog_name=<> --source=kafka --kafka_source_topic=<>>> --kafka_sink_topic=<> --kafka_source_servers_secrets_scope_name=<> --kafka_source_servers_secrets_scope_key=<> --kafka_sink_servers_secret_scope_name=<> --kafka_sink_servers_secret_scope_key=<> --profile=<> ``` + + - uc_catalog_name : UC catalog name. Can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. + - you can provide `--profile=databricks_profile name` in case you already have databricks cli otherwise command prompt will ask host and token. + ![dlt_demo_sink.png](../docs/static/images/dlt_demo_sink.png) ![dlt_delta_sink.png](../docs/static/images/dlt_delta_sink.png) ![dlt_kafka_sink.png](../docs/static/images/dlt_kafka_sink.png) @@ -406,6 +414,8 @@ This demo will perform following steps: ```commandline python demo/generate_dabs_resources.py --source=cloudfiles --uc_catalog_name= --profile= ``` + > Note: uc_catalog_name : UC catalog name. Can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. + > Note: If you don't specify `--profile`, you'll be prompted for your Databricks workspace URL and access token. 7. Deploy and run the DAB bundle: diff --git a/demo/generate_dabs_resources.py b/demo/generate_dabs_resources.py index f18e4f8..2b49be5 100644 --- a/demo/generate_dabs_resources.py +++ b/demo/generate_dabs_resources.py @@ -69,6 +69,12 @@ def init_runner_conf(self) -> DLTMetaRunnerConf: f"{run_id}/demo-output.csv" ), ) + if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): + print( + "\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z')," + " digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting." + ) + exit(1) return runner_conf diff --git a/demo/launch_acfs_demo.py b/demo/launch_acfs_demo.py index bd28d5e..5473c13 100644 --- a/demo/launch_acfs_demo.py +++ b/demo/launch_acfs_demo.py @@ -55,6 +55,12 @@ def init_runner_conf(self) -> DLTMetaRunnerConf: env="demo" ) runner_conf.uc_catalog_name = self.args['uc_catalog_name'] + if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): + print( + "\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z')," + " digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting." + ) + exit(1) return runner_conf def launch_workflow(self, runner_conf: DLTMetaRunnerConf): diff --git a/demo/launch_af_cloudfiles_demo.py b/demo/launch_af_cloudfiles_demo.py index 98fdcbf..9ce0196 100644 --- a/demo/launch_af_cloudfiles_demo.py +++ b/demo/launch_af_cloudfiles_demo.py @@ -67,6 +67,13 @@ def init_runner_conf(self) -> DLTMetaRunnerConf: ), ) + if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): + print( + "\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z')," + " digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting." + ) + exit(1) + return runner_conf def launch_workflow(self, runner_conf: DLTMetaRunnerConf): diff --git a/demo/launch_af_eventhub_demo.py b/demo/launch_af_eventhub_demo.py index 9b16f24..1bced1b 100644 --- a/demo/launch_af_eventhub_demo.py +++ b/demo/launch_af_eventhub_demo.py @@ -71,6 +71,12 @@ def init_runner_conf(self) -> DLTMetaRunnerConf: eventhub_port=self.args["eventhub_port"] ) runner_conf.uc_catalog_name = self.args['uc_catalog_name'] + if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): + print( + "\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z')," + " digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting." + ) + exit(1) runner_conf.runners_full_local_path = 'demo/notebooks/afam_eventhub_runners' return runner_conf diff --git a/demo/launch_dais_demo.py b/demo/launch_dais_demo.py index ad5c984..e6d99cf 100644 --- a/demo/launch_dais_demo.py +++ b/demo/launch_dais_demo.py @@ -52,6 +52,12 @@ def init_runner_conf(self) -> DLTMetaRunnerConf: ) if self.args['uc_catalog_name']: runner_conf.uc_catalog_name = self.args['uc_catalog_name'] + if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): + print( + "\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z')," + " digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting." + ) + exit(1) runner_conf.uc_volume_name = f"{runner_conf.uc_catalog_name}_dais_demo_{run_id}" return runner_conf diff --git a/demo/launch_dlt_sink_demo.py b/demo/launch_dlt_sink_demo.py index 2fb5308..f3899d4 100644 --- a/demo/launch_dlt_sink_demo.py +++ b/demo/launch_dlt_sink_demo.py @@ -70,6 +70,13 @@ def init_runner_conf(self) -> DLTMetaRunnerConf: ), ) + if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): + print( + "\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z')," + " digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting." + ) + exit(1) + return runner_conf def launch_workflow(self, runner_conf: DLTMetaRunnerConf): diff --git a/demo/launch_silver_fanout_demo.py b/demo/launch_silver_fanout_demo.py index d075f27..639235b 100644 --- a/demo/launch_silver_fanout_demo.py +++ b/demo/launch_silver_fanout_demo.py @@ -83,6 +83,12 @@ def init_runner_conf(self) -> DLTMetaRunnerConf: env="demo" ) runner_conf.uc_catalog_name = self.args['uc_catalog_name'] + if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): + print( + "\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z')," + " digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting." + ) + exit(1) runner_conf.uc_volume_name = f"{runner_conf.uc_catalog_name}_dlt_meta_fout_demo_{run_id}" return runner_conf diff --git a/demo/launch_techsummit_demo.py b/demo/launch_techsummit_demo.py index 94d76e2..d680ecd 100644 --- a/demo/launch_techsummit_demo.py +++ b/demo/launch_techsummit_demo.py @@ -102,7 +102,14 @@ def init_runner_conf(self) -> TechsummitRunnerConf: ) if self.args['uc_catalog_name']: runner_conf.uc_catalog_name = self.args['uc_catalog_name'] + if '-' in runner_conf.uc_catalog_name or runner_conf.uc_catalog_name[0].isdigit(): + print( + "\nERROR: 'uc_catalog_name' can only contain ASCII letters ('a' - 'z', 'A' - 'Z')," + " digits ('0' - '9'), and underbar ('_'). Must also not start with a digit. Exiting." + ) + exit(1) runner_conf.uc_volume_name = f"{self.args['uc_catalog_name']}_volume_{run_id}" + return runner_conf def create_bronze_silver_dlt(self, runner_conf: DLTMetaRunnerConf):