[CIVIS-11461] Add example with environment variables compatible with civis studios

dbt/README.md

@@ -5,7 +5,7 @@ This directory contains the configuration files, models, and seeds for the dbt (
 ## Structure
 
 - [`dbt_project.yml`]( dbt_project.yml ): Main configuration file for the dbt project.
-- [`profiles.yml`]( profiles.yml  ): Profiles configuration for dbt, specifying different environments and their settings.
+- [`profiles.yml`]( profiles.yml ): Profiles configuration for dbt, specifying different environments and their settings. This contains examples for both Civis dbt scripts and Civis studio environment configurations. Civis Studios follow a slightly different environment variable naming convention, so this file includes examples for use directly in dbt scripts (`dbt-civis`), use directly in a studio (`civis-studio`), as well as an example of a configuration that works both in a studio and in dbt scripts (`studio-or-dbt-script`).
 - `models/`: Directory containing SQL models for dbt.
   - `docs.md`: Documentation for the models.
   - [`overview.md`](  ./models/overview.md )): Overview of the dbt project.

dbt/dbt_project.yml

@@ -3,7 +3,7 @@ name: 'civis_jaffle_shop_example'
 config-version: 2
 version: '0.1'
 
-profile: 'dbt-civis'
+profile: 'studio-or-dbt-script'
 
 model-paths: ["models"]
 seed-paths: ["seeds"]

dbt/profiles.yml

@@ -26,3 +26,121 @@ dbt-civis:
       project: "{{ env_var('GCP_PROJECT_ID') }}"
       schema: "{{ env_var('DBT_SCHEMA') }}"
       token: "{{ env_var('DBT_ENV_SECRET_GCP_ACCESS_TOKEN') }}"
+
+
+
+#############################################
+########## dbt in Civis Studios #############
+#############################################
+
+# Civis Studios have slightly different environment variables than
+# dbt scripts in Platform (see the dbt-civis profile above for comparison).
+#
+# The following profiles demonstrate the necessary changes to use the Civis Studio environment:
+# For a database parameter named "civis_consumers":
+#  "params": [
+#    {
+#        "name": "civis_consumers",
+#        "description": null,
+#        "type": "database",
+#        "value": "{\"credential\":1234,\"database\":5678}"
+#    }, {
+#        "name": "dbt_schema",
+#        "description": null,
+#        "type": "string",
+#        "value": "my_dbt_schema"
+#    }
+#  ]
+# , the corresponding environment variables in Civis Studios are:
+civis-studio:
+  target: redshift_adapter
+  outputs:
+    redshift_adapter:
+      type: "{{ env_var('CIVIS_CONSUMER_TYPE') }}"
+      host: "{{ env_var('CIVIS_CONSUMER_HOST') }}"
+      port: "{{ env_var('CIVIS_CONSUMER_PORT') | as_number }}"
+      user: "{{ env_var('CIVIS_CONSUMER_CREDENTIAL_USERNAME') }}" # note the _CREDENTIAL
+      pass: "{{ env_var('DBT_ENV_SECRET_CIVIS_CONSUMER_CREDENTIAL_PASSWORD') }}" # note the _CREDENTIAL
+      dbname: "{{ env_var('CIVIS_CONSUMER_DATABASE') }}" # _DATABASE instead of _NAME
+      schema: "{{ env_var('DBT_SCHEMA') }}"
+      threads: 4
+
+    # TODO
+    bigquery_service_account_adapter:
+      type: "{{ env_var('CIVIS_CONSUMER_TYPE') }}"
+      method: "{{ env_var('GCP_AUTH_METHOD') }}"
+      project: "{{ env_var('GCP_PROJECT_ID') }}"
+      schema: "{{ env_var('DBT_SCHEMA') }}"
+      keyfile: "{{ env_var('GCP_SERVICE_ACCOUNT_KEYFILE_JSON') }}"
+      threads: 4
+
+    bigquery_oauth_adapter:
+      threads: 4 # Must be a value of 1 or greater
+      type: "{{ env_var('CIVIS_CONSUMER_TYPE') }}"
+      method: "{{ env_var('GCP_AUTH_METHOD') }}"
+      project: "{{ env_var('GCP_PROJECT_ID') }}"
+      schema: "{{ env_var('DBT_SCHEMA') }}"
+      token: "{{ env_var('DBT_ENV_SECRET_GCP_ACCESS_TOKEN') }}"
+
+# The following can be used in both studios and Platform dbt scripts, assuming
+# parameters* like the following exist in the studio:
+#  "params": [
+#    {
+#        "name": "civis_consumers", // a redshift database parameter
+#        "description": null,
+#        "type": "database",
+#        "value": "{\"credential\":1234,\"database\":5678}"
+#    }, {
+#        "name": "dbt_schema",
+#        "description": null,
+#        "type": "string",
+#        "value": "my_dbt_schema"
+#    }, {
+#        "name": "studio_redshift_prefix",
+#        "description": null,
+#        "type": "string",
+#        "value": "civis_consumers" // matches the parameter name for the redshift database
+#    }, {
+#        "name": "civis_international_consumers", // a bigquery database parameter
+#        "description": null,
+#        "type": "database",
+#        "value": "{\"credential\":333,\"database\":999}"
+#    }, {
+#        "name": "studio_bigquery_prefix",
+#        "description": null,
+#        "type": "string",
+#        "value": "civis_international_consumers" // matches the parameter name for the BQ database
+#    }
+#  ]
+# *Note: this example includes parameters for both redshift and bigquery, but you'll likely only need
+# one or the other
+studio-or-dbt-script:
+  target: redshift_adapter # sets the default target
+  outputs:
+    # Works for redshift or postgres
+    redshift_adapter:
+      type: "{{ env_var(env_var('STUDIO_REDSHIFT_PREFIX', 'DATABASE') ~ '_TYPE') }}"
+      host: "{{ env_var(env_var('STUDIO_REDSHIFT_PREFIX', 'DATABASE') ~ '_HOST') }}"
+      port: "{{ env_var(env_var('STUDIO_REDSHIFT_PREFIX', 'DATABASE') ~ '_PORT') | as_number }}"
+      user: "{% if env_var('STUDIO_REDSHIFT_PREFIX', '') %}{{ env_var(env_var('STUDIO_REDSHIFT_PREFIX') ~ '_CREDENTIAL_USERNAME') }}{% else %}{{ env_var('DATABASE_USERNAME') }}{% endif %}"
+      pass: "{% if env_var('STUDIO_REDSHIFT_PREFIX', '') %}{{ env_var('DBT_ENV_SECRET_' ~ env_var('STUDIO_REDSHIFT_PREFIX') ~ '_CREDENTIAL_PASSWORD') }}{% else %}{{ env_var('DBT_ENV_SECRET_DATABASE_PASSWORD') }}{% endif %}"
+      dbname: "{% if env_var('STUDIO_REDSHIFT_PREFIX', '') %}{{ env_var(env_var('STUDIO_REDSHIFT_PREFIX') ~ '_DATABASE') }}{% else %}{{ env_var('DATABASE_NAME') }}{% endif %}"
+      schema: "{{ env_var('DBT_SCHEMA') }}"
+      threads: 4
+
+    # TODO
+    bigquery_service_account_adapter:
+      type: "{{ env_var(env_var('STUDIO_BIGQUERY_PREFIX', 'DATABASE') ~ '_TYPE') }}"
+      method: "{{ env_var('GCP_AUTH_METHOD') }}"
+      project: "{{ env_var('GCP_PROJECT_ID') }}"
+      schema: "{{ env_var('DBT_SCHEMA') }}"
+      keyfile: "{{ env_var('GCP_SERVICE_ACCOUNT_KEYFILE_JSON') }}"
+      threads: 4
+
+    bigquery_oauth_adapter:
+      threads: 4
+      type: "{{ env_var(env_var('STUDIO_BIGQUERY_PREFIX', 'DATABASE') ~ '_TYPE') }}"
+      method: "{{ env_var('GCP_AUTH_METHOD') }}"
+      project: "{{ env_var('GCP_PROJECT_ID') }}"
+      schema: "{{ env_var('DBT_SCHEMA') }}"
+      token: "{{ env_var('DBT_ENV_SECRET_GCP_ACCESS_TOKEN') }}"