From a8a945d7a3e9e350a799dbcafacc9b2a9ffefbf6 Mon Sep 17 00:00:00 2001 From: Gabby Skifstad Date: Tue, 21 Oct 2025 15:15:13 -0700 Subject: [PATCH 1/8] add example with environment variables compatible with code clouds --- dbt/README.md | 1 + dbt/code_cloud_profiles.yml | 109 ++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 dbt/code_cloud_profiles.yml diff --git a/dbt/README.md b/dbt/README.md index 3323fa1..7354c94 100644 --- a/dbt/README.md +++ b/dbt/README.md @@ -6,6 +6,7 @@ This directory contains the configuration files, models, and seeds for the dbt ( - [`dbt_project.yml`]( dbt_project.yml ): Main configuration file for the dbt project. - [`profiles.yml`]( profiles.yml ): Profiles configuration for dbt, specifying different environments and their settings. +- [`code_cloud_profiles.yml`]( code_cloud_profiles.yml ): Same as `profiles.yml`, but configured for use in Civis Code Cloud. Code Clouds follow a slightly different environment variable naming convention. This file includes examples for use directly in a code cloud, as well as an example of a configuration that works both in a code cloud and in dbt scripts. - `models/`: Directory containing SQL models for dbt. - `docs.md`: Documentation for the models. - [`overview.md`]( ./models/overview.md )): Overview of the dbt project. diff --git a/dbt/code_cloud_profiles.yml b/dbt/code_cloud_profiles.yml new file mode 100644 index 0000000..733653d --- /dev/null +++ b/dbt/code_cloud_profiles.yml @@ -0,0 +1,109 @@ +# Code Clouds have slightly different environment variables than +# dbt scripts in Platform (see dbt/profiles.yml for comparison). +# +# This file demonstrates the necessary changes to use Code Cloud environment: +# For a database parameter named "civis_consumers": +# "params": [ +# { +# "name": "civis_consumers", +# "description": null, +# "type": "database", +# "value": "{\"credential\":1234,\"database\":5678}" +# }, { +# "name": "dbt_schema", +# "description": null, +# "type": "string", +# "value": "my_dbt_schema" +# } +# ] +# , the corresponding environment variables in Code Cloud are: +dbt-civis: + target: redshift_adapter + outputs: + redshift_adapter: + type: "{{ env_var('CIVIS_CONSUMER_TYPE') }}" + host: "{{ env_var('CIVIS_CONSUMER_HOST') }}" + port: "{{ env_var('CIVIS_CONSUMER_PORT') | as_number }}" + user: "{{ env_var('CIVIS_CONSUMER_CREDENTIAL_USERNAME') }}" # note the _CREDENTIAL + pass: "{{ env_var('DBT_ENV_SECRET_CIVIS_CONSUMER_CREDENTIAL_PASSWORD') }}" # note the _CREDENTIAL + dbname: "{{ env_var('CIVIS_CONSUMER_DATABASE') }}" # _DATABASE instead of _NAME + schema: "{{ env_var('DBT_SCHEMA') }}" + threads: 4 + + # TODO + bigquery_service_account_adapter: + type: "{{ env_var('CIVIS_CONSUMER_TYPE') }}" + method: "{{ env_var('GCP_AUTH_METHOD') }}" + project: "{{ env_var('GCP_PROJECT_ID') }}" + schema: "{{ env_var('DBT_SCHEMA') }}" + keyfile: "{{ env_var('GCP_SERVICE_ACCOUNT_KEYFILE_JSON') }}" + threads: 4 + + bigquery_oauth_adapter: + threads: 4 # Must be a value of 1 or greater + type: "{{ env_var('CIVIS_CONSUMER_TYPE') }}" + method: "{{ env_var('GCP_AUTH_METHOD') }}" + project: "{{ env_var('GCP_PROJECT_ID') }}" + schema: "{{ env_var('DBT_SCHEMA') }}" + token: "{{ env_var('DBT_ENV_SECRET_GCP_ACCESS_TOKEN') }}" + +# The following can be used in both code clouds and Platform dbt scripts, assuming +# parameters like the following exist in the code cloud: +# "params": [ +# { +# "name": "civis_consumers", +# "description": null, +# "type": "database", +# "value": "{\"credential\":1234,\"database\":5678}" +# }, { +# "name": "dbt_schema", +# "description": null, +# "type": "string", +# "value": "my_dbt_schema" +# }, { +# "name": "code_cloud_redshift_prefix", +# "description": null, +# "type": "string", +# "value": "civis_consumers" // matches the parameter name for the redshift database +# }, { +# "name": "civis_international_consumers", +# "description": null, +# "type": "database", +# "value": "{\"credential\":333,\"database\":999}" +# }, { +# "name": "code_cloud_bigquery_prefix", +# "description": null, +# "type": "string", +# "value": "civis_international_consumers" // matches the parameter name for the BQ database +# } +# ] +my-project-name: + target: redshift_adapter # sets the default target + outputs: + # Works for redshift or postgres + redshift_adapter: + type: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ '_TYPE') }}" + host: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ '_HOST') }}" + port: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ _PORT') | as_number }}" + user: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX') ~ _CREDENTIAL_USERNAME', 'DATABASE_USERNAME') }}" + pass: "{{ env_var('DBT_ENV_SECRET_' ~ env_var('CODE_CLOUD_REDSHIFT_PREFIX') ~ _CREDENTIAL_PASSWORD', 'DBT_ENV_SECRET_DATABASE_PASSWORD') }}" + dbname: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX') ~ _DATABASE', 'DATABASE_NAME') }}" + schema: "{{ env_var('DBT_SCHEMA') }}" + threads: 4 + + # TODO + bigquery_service_account_adapter: + type: "{{ env_var(env_var('CODE_CLOUD_BIGQUERY_PREFIX', 'DATABASE') ~ '_TYPE') }}" + method: "{{ env_var('GCP_AUTH_METHOD') }}" + project: "{{ env_var('GCP_PROJECT_ID') }}" + schema: "{{ env_var('DBT_SCHEMA') }}" + keyfile: "{{ env_var('GCP_SERVICE_ACCOUNT_KEYFILE_JSON') }}" + threads: 4 + + bigquery_oauth_adapter: + threads: 4 + type: "{{ env_var(env_var('CODE_CLOUD_BIGQUERY_PREFIX', 'DATABASE') ~ '_TYPE') }}" + method: "{{ env_var('GCP_AUTH_METHOD') }}" + project: "{{ env_var('GCP_PROJECT_ID') }}" + schema: "{{ env_var('DBT_SCHEMA') }}" + token: "{{ env_var('DBT_ENV_SECRET_GCP_ACCESS_TOKEN') }}" From 64fc888b315e40ed45b578d70122f8b6e4748968 Mon Sep 17 00:00:00 2001 From: Gabby Skifstad Date: Tue, 21 Oct 2025 15:36:34 -0700 Subject: [PATCH 2/8] add missing quotes --- dbt/code_cloud_profiles.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbt/code_cloud_profiles.yml b/dbt/code_cloud_profiles.yml index 733653d..5ea2308 100644 --- a/dbt/code_cloud_profiles.yml +++ b/dbt/code_cloud_profiles.yml @@ -84,10 +84,10 @@ my-project-name: redshift_adapter: type: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ '_TYPE') }}" host: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ '_HOST') }}" - port: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ _PORT') | as_number }}" - user: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX') ~ _CREDENTIAL_USERNAME', 'DATABASE_USERNAME') }}" - pass: "{{ env_var('DBT_ENV_SECRET_' ~ env_var('CODE_CLOUD_REDSHIFT_PREFIX') ~ _CREDENTIAL_PASSWORD', 'DBT_ENV_SECRET_DATABASE_PASSWORD') }}" - dbname: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX') ~ _DATABASE', 'DATABASE_NAME') }}" + port: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ '_PORT') | as_number }}" + user: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX') ~ '_CREDENTIAL_USERNAME', 'DATABASE_USERNAME') }}" + pass: "{{ env_var('DBT_ENV_SECRET_' ~ env_var('CODE_CLOUD_REDSHIFT_PREFIX') ~ '_CREDENTIAL_PASSWORD', 'DBT_ENV_SECRET_DATABASE_PASSWORD') }}" + dbname: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX') ~ '_DATABASE', 'DATABASE_NAME') }}" schema: "{{ env_var('DBT_SCHEMA') }}" threads: 4 From 80fca8efce9f7d289f9c250f2a30d8c3f40f0c0a Mon Sep 17 00:00:00 2001 From: Gabby Skifstad Date: Tue, 4 Nov 2025 18:42:59 +0000 Subject: [PATCH 3/8] merge code_cloud_profiles.yml with profiles.yml --- dbt/code_cloud_profiles.yml | 109 --------------------------------- dbt/dbt_project.yml | 2 +- dbt/profiles.yml | 118 ++++++++++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 110 deletions(-) delete mode 100644 dbt/code_cloud_profiles.yml diff --git a/dbt/code_cloud_profiles.yml b/dbt/code_cloud_profiles.yml deleted file mode 100644 index 5ea2308..0000000 --- a/dbt/code_cloud_profiles.yml +++ /dev/null @@ -1,109 +0,0 @@ -# Code Clouds have slightly different environment variables than -# dbt scripts in Platform (see dbt/profiles.yml for comparison). -# -# This file demonstrates the necessary changes to use Code Cloud environment: -# For a database parameter named "civis_consumers": -# "params": [ -# { -# "name": "civis_consumers", -# "description": null, -# "type": "database", -# "value": "{\"credential\":1234,\"database\":5678}" -# }, { -# "name": "dbt_schema", -# "description": null, -# "type": "string", -# "value": "my_dbt_schema" -# } -# ] -# , the corresponding environment variables in Code Cloud are: -dbt-civis: - target: redshift_adapter - outputs: - redshift_adapter: - type: "{{ env_var('CIVIS_CONSUMER_TYPE') }}" - host: "{{ env_var('CIVIS_CONSUMER_HOST') }}" - port: "{{ env_var('CIVIS_CONSUMER_PORT') | as_number }}" - user: "{{ env_var('CIVIS_CONSUMER_CREDENTIAL_USERNAME') }}" # note the _CREDENTIAL - pass: "{{ env_var('DBT_ENV_SECRET_CIVIS_CONSUMER_CREDENTIAL_PASSWORD') }}" # note the _CREDENTIAL - dbname: "{{ env_var('CIVIS_CONSUMER_DATABASE') }}" # _DATABASE instead of _NAME - schema: "{{ env_var('DBT_SCHEMA') }}" - threads: 4 - - # TODO - bigquery_service_account_adapter: - type: "{{ env_var('CIVIS_CONSUMER_TYPE') }}" - method: "{{ env_var('GCP_AUTH_METHOD') }}" - project: "{{ env_var('GCP_PROJECT_ID') }}" - schema: "{{ env_var('DBT_SCHEMA') }}" - keyfile: "{{ env_var('GCP_SERVICE_ACCOUNT_KEYFILE_JSON') }}" - threads: 4 - - bigquery_oauth_adapter: - threads: 4 # Must be a value of 1 or greater - type: "{{ env_var('CIVIS_CONSUMER_TYPE') }}" - method: "{{ env_var('GCP_AUTH_METHOD') }}" - project: "{{ env_var('GCP_PROJECT_ID') }}" - schema: "{{ env_var('DBT_SCHEMA') }}" - token: "{{ env_var('DBT_ENV_SECRET_GCP_ACCESS_TOKEN') }}" - -# The following can be used in both code clouds and Platform dbt scripts, assuming -# parameters like the following exist in the code cloud: -# "params": [ -# { -# "name": "civis_consumers", -# "description": null, -# "type": "database", -# "value": "{\"credential\":1234,\"database\":5678}" -# }, { -# "name": "dbt_schema", -# "description": null, -# "type": "string", -# "value": "my_dbt_schema" -# }, { -# "name": "code_cloud_redshift_prefix", -# "description": null, -# "type": "string", -# "value": "civis_consumers" // matches the parameter name for the redshift database -# }, { -# "name": "civis_international_consumers", -# "description": null, -# "type": "database", -# "value": "{\"credential\":333,\"database\":999}" -# }, { -# "name": "code_cloud_bigquery_prefix", -# "description": null, -# "type": "string", -# "value": "civis_international_consumers" // matches the parameter name for the BQ database -# } -# ] -my-project-name: - target: redshift_adapter # sets the default target - outputs: - # Works for redshift or postgres - redshift_adapter: - type: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ '_TYPE') }}" - host: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ '_HOST') }}" - port: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ '_PORT') | as_number }}" - user: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX') ~ '_CREDENTIAL_USERNAME', 'DATABASE_USERNAME') }}" - pass: "{{ env_var('DBT_ENV_SECRET_' ~ env_var('CODE_CLOUD_REDSHIFT_PREFIX') ~ '_CREDENTIAL_PASSWORD', 'DBT_ENV_SECRET_DATABASE_PASSWORD') }}" - dbname: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX') ~ '_DATABASE', 'DATABASE_NAME') }}" - schema: "{{ env_var('DBT_SCHEMA') }}" - threads: 4 - - # TODO - bigquery_service_account_adapter: - type: "{{ env_var(env_var('CODE_CLOUD_BIGQUERY_PREFIX', 'DATABASE') ~ '_TYPE') }}" - method: "{{ env_var('GCP_AUTH_METHOD') }}" - project: "{{ env_var('GCP_PROJECT_ID') }}" - schema: "{{ env_var('DBT_SCHEMA') }}" - keyfile: "{{ env_var('GCP_SERVICE_ACCOUNT_KEYFILE_JSON') }}" - threads: 4 - - bigquery_oauth_adapter: - threads: 4 - type: "{{ env_var(env_var('CODE_CLOUD_BIGQUERY_PREFIX', 'DATABASE') ~ '_TYPE') }}" - method: "{{ env_var('GCP_AUTH_METHOD') }}" - project: "{{ env_var('GCP_PROJECT_ID') }}" - schema: "{{ env_var('DBT_SCHEMA') }}" - token: "{{ env_var('DBT_ENV_SECRET_GCP_ACCESS_TOKEN') }}" diff --git a/dbt/dbt_project.yml b/dbt/dbt_project.yml index 72bfffd..f006f5b 100644 --- a/dbt/dbt_project.yml +++ b/dbt/dbt_project.yml @@ -3,7 +3,7 @@ name: 'civis_jaffle_shop_example' config-version: 2 version: '0.1' -profile: 'dbt-civis' +profile: 'studio-or-dbt-script' model-paths: ["models"] seed-paths: ["seeds"] diff --git a/dbt/profiles.yml b/dbt/profiles.yml index 157dcc0..15b61de 100644 --- a/dbt/profiles.yml +++ b/dbt/profiles.yml @@ -26,3 +26,121 @@ dbt-civis: project: "{{ env_var('GCP_PROJECT_ID') }}" schema: "{{ env_var('DBT_SCHEMA') }}" token: "{{ env_var('DBT_ENV_SECRET_GCP_ACCESS_TOKEN') }}" + + + +############################################# +########## dbt in Civis Studios ############# +############################################# + +# Civis Studios have slightly different environment variables than +# dbt scripts in Platform (see the dbt-civis profile above for comparison). +# +# The following profiles demonstrate the necessary changes to use the Civis Studio environment: +# For a database parameter named "civis_consumers": +# "params": [ +# { +# "name": "civis_consumers", +# "description": null, +# "type": "database", +# "value": "{\"credential\":1234,\"database\":5678}" +# }, { +# "name": "dbt_schema", +# "description": null, +# "type": "string", +# "value": "my_dbt_schema" +# } +# ] +# , the corresponding environment variables in Civis Studios are: +civis-studio: + target: redshift_adapter + outputs: + redshift_adapter: + type: "{{ env_var('CIVIS_CONSUMER_TYPE') }}" + host: "{{ env_var('CIVIS_CONSUMER_HOST') }}" + port: "{{ env_var('CIVIS_CONSUMER_PORT') | as_number }}" + user: "{{ env_var('CIVIS_CONSUMER_CREDENTIAL_USERNAME') }}" # note the _CREDENTIAL + pass: "{{ env_var('DBT_ENV_SECRET_CIVIS_CONSUMER_CREDENTIAL_PASSWORD') }}" # note the _CREDENTIAL + dbname: "{{ env_var('CIVIS_CONSUMER_DATABASE') }}" # _DATABASE instead of _NAME + schema: "{{ env_var('DBT_SCHEMA') }}" + threads: 4 + + # TODO + bigquery_service_account_adapter: + type: "{{ env_var('CIVIS_CONSUMER_TYPE') }}" + method: "{{ env_var('GCP_AUTH_METHOD') }}" + project: "{{ env_var('GCP_PROJECT_ID') }}" + schema: "{{ env_var('DBT_SCHEMA') }}" + keyfile: "{{ env_var('GCP_SERVICE_ACCOUNT_KEYFILE_JSON') }}" + threads: 4 + + bigquery_oauth_adapter: + threads: 4 # Must be a value of 1 or greater + type: "{{ env_var('CIVIS_CONSUMER_TYPE') }}" + method: "{{ env_var('GCP_AUTH_METHOD') }}" + project: "{{ env_var('GCP_PROJECT_ID') }}" + schema: "{{ env_var('DBT_SCHEMA') }}" + token: "{{ env_var('DBT_ENV_SECRET_GCP_ACCESS_TOKEN') }}" + +# The following can be used in both studios and Platform dbt scripts, assuming +# parameters* like the following exist in the studio: +# "params": [ +# { +# "name": "civis_consumers", // a redshift database parameter +# "description": null, +# "type": "database", +# "value": "{\"credential\":1234,\"database\":5678}" +# }, { +# "name": "dbt_schema", +# "description": null, +# "type": "string", +# "value": "my_dbt_schema" +# }, { +# "name": "code_cloud_redshift_prefix", +# "description": null, +# "type": "string", +# "value": "civis_consumers" // matches the parameter name for the redshift database +# }, { +# "name": "civis_international_consumers", // a bigquery database parameter +# "description": null, +# "type": "database", +# "value": "{\"credential\":333,\"database\":999}" +# }, { +# "name": "code_cloud_bigquery_prefix", +# "description": null, +# "type": "string", +# "value": "civis_international_consumers" // matches the parameter name for the BQ database +# } +# ] +# *Note: this example includes parameters for both redshift and bigquery, but you'll likely only need +# one or the other +studio-or-dbt-script: + target: redshift_adapter # sets the default target + outputs: + # Works for redshift or postgres + redshift_adapter: + type: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ '_TYPE') }}" + host: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ '_HOST') }}" + port: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ '_PORT') | as_number }}" + user: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX') ~ '_CREDENTIAL_USERNAME', 'DATABASE_USERNAME') }}" + pass: "{{ env_var('DBT_ENV_SECRET_' ~ env_var('CODE_CLOUD_REDSHIFT_PREFIX') ~ '_CREDENTIAL_PASSWORD', 'DBT_ENV_SECRET_DATABASE_PASSWORD') }}" + dbname: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX') ~ '_DATABASE', 'DATABASE_NAME') }}" + schema: "{{ env_var('DBT_SCHEMA') }}" + threads: 4 + + # TODO + bigquery_service_account_adapter: + type: "{{ env_var(env_var('CODE_CLOUD_BIGQUERY_PREFIX', 'DATABASE') ~ '_TYPE') }}" + method: "{{ env_var('GCP_AUTH_METHOD') }}" + project: "{{ env_var('GCP_PROJECT_ID') }}" + schema: "{{ env_var('DBT_SCHEMA') }}" + keyfile: "{{ env_var('GCP_SERVICE_ACCOUNT_KEYFILE_JSON') }}" + threads: 4 + + bigquery_oauth_adapter: + threads: 4 + type: "{{ env_var(env_var('CODE_CLOUD_BIGQUERY_PREFIX', 'DATABASE') ~ '_TYPE') }}" + method: "{{ env_var('GCP_AUTH_METHOD') }}" + project: "{{ env_var('GCP_PROJECT_ID') }}" + schema: "{{ env_var('DBT_SCHEMA') }}" + token: "{{ env_var('DBT_ENV_SECRET_GCP_ACCESS_TOKEN') }}" From 7272ba0a7038daaaa0313084ec88b7500a8ff97a Mon Sep 17 00:00:00 2001 From: Gabby Skifstad Date: Tue, 4 Nov 2025 18:46:33 +0000 Subject: [PATCH 4/8] update readme --- dbt/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dbt/README.md b/dbt/README.md index 7354c94..a6bb417 100644 --- a/dbt/README.md +++ b/dbt/README.md @@ -5,8 +5,7 @@ This directory contains the configuration files, models, and seeds for the dbt ( ## Structure - [`dbt_project.yml`]( dbt_project.yml ): Main configuration file for the dbt project. -- [`profiles.yml`]( profiles.yml ): Profiles configuration for dbt, specifying different environments and their settings. -- [`code_cloud_profiles.yml`]( code_cloud_profiles.yml ): Same as `profiles.yml`, but configured for use in Civis Code Cloud. Code Clouds follow a slightly different environment variable naming convention. This file includes examples for use directly in a code cloud, as well as an example of a configuration that works both in a code cloud and in dbt scripts. +- [`profiles.yml`]( profiles.yml ): Profiles configuration for dbt, specifying different environments and their settings. This contains examples for both Civis dbt scripts and Civis studio environment configurations. Civis Studios follow a slightly different environment variable naming convention, so this file includes examples for use directly in dbt scripts (`dbt-civis`), use directly in a studio (`civis-studio`), as well as an example of a configuration that works both in a studio and in dbt scripts (`studio-or-dbt-script`). - `models/`: Directory containing SQL models for dbt. - `docs.md`: Documentation for the models. - [`overview.md`]( ./models/overview.md )): Overview of the dbt project. From b3068dde37f65d1abe7c294acd4af816c1bce6ee Mon Sep 17 00:00:00 2001 From: Gabby Skifstad Date: Tue, 4 Nov 2025 19:34:25 +0000 Subject: [PATCH 5/8] fallback to none for cases where env var names are very different --- dbt/profiles.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbt/profiles.yml b/dbt/profiles.yml index 15b61de..3c35e59 100644 --- a/dbt/profiles.yml +++ b/dbt/profiles.yml @@ -122,9 +122,9 @@ studio-or-dbt-script: type: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ '_TYPE') }}" host: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ '_HOST') }}" port: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ '_PORT') | as_number }}" - user: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX') ~ '_CREDENTIAL_USERNAME', 'DATABASE_USERNAME') }}" - pass: "{{ env_var('DBT_ENV_SECRET_' ~ env_var('CODE_CLOUD_REDSHIFT_PREFIX') ~ '_CREDENTIAL_PASSWORD', 'DBT_ENV_SECRET_DATABASE_PASSWORD') }}" - dbname: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX') ~ '_DATABASE', 'DATABASE_NAME') }}" + user: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', none) ~ '_CREDENTIAL_USERNAME', 'DATABASE_USERNAME') }}" + pass: "{{ env_var('DBT_ENV_SECRET_' ~ env_var('CODE_CLOUD_REDSHIFT_PREFIX', none) ~ '_CREDENTIAL_PASSWORD', 'DBT_ENV_SECRET_DATABASE_PASSWORD') }}" + dbname: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', none) ~ '_DATABASE', 'DATABASE_NAME') }}" schema: "{{ env_var('DBT_SCHEMA') }}" threads: 4 From ccc773dc22f85b879c6310ef023a545bd8669875 Mon Sep 17 00:00:00 2001 From: Gabby Skifstad Date: Tue, 4 Nov 2025 15:08:03 -0800 Subject: [PATCH 6/8] try empty string instead of none --- dbt/profiles.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbt/profiles.yml b/dbt/profiles.yml index 3c35e59..a186bd9 100644 --- a/dbt/profiles.yml +++ b/dbt/profiles.yml @@ -122,9 +122,9 @@ studio-or-dbt-script: type: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ '_TYPE') }}" host: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ '_HOST') }}" port: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ '_PORT') | as_number }}" - user: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', none) ~ '_CREDENTIAL_USERNAME', 'DATABASE_USERNAME') }}" - pass: "{{ env_var('DBT_ENV_SECRET_' ~ env_var('CODE_CLOUD_REDSHIFT_PREFIX', none) ~ '_CREDENTIAL_PASSWORD', 'DBT_ENV_SECRET_DATABASE_PASSWORD') }}" - dbname: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', none) ~ '_DATABASE', 'DATABASE_NAME') }}" + user: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', '') ~ '_CREDENTIAL_USERNAME', 'DATABASE_USERNAME') }}" + pass: "{{ env_var('DBT_ENV_SECRET_' ~ env_var('CODE_CLOUD_REDSHIFT_PREFIX', '') ~ '_CREDENTIAL_PASSWORD', 'DBT_ENV_SECRET_DATABASE_PASSWORD') }}" + dbname: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', '') ~ '_DATABASE', 'DATABASE_NAME') }}" schema: "{{ env_var('DBT_SCHEMA') }}" threads: 4 From 70a238269e75240032f7145d19eb7919c90bb95b Mon Sep 17 00:00:00 2001 From: Gabby Skifstad Date: Wed, 12 Nov 2025 19:24:02 +0000 Subject: [PATCH 7/8] bring back service account example; respond to other code review feedback --- dbt/README.md | 2 +- dbt/dbt_project.yml | 2 +- dbt/profiles.yml | 83 +++++++-------------------------------------- 3 files changed, 14 insertions(+), 73 deletions(-) diff --git a/dbt/README.md b/dbt/README.md index a6bb417..b9a1f96 100644 --- a/dbt/README.md +++ b/dbt/README.md @@ -5,7 +5,7 @@ This directory contains the configuration files, models, and seeds for the dbt ( ## Structure - [`dbt_project.yml`]( dbt_project.yml ): Main configuration file for the dbt project. -- [`profiles.yml`]( profiles.yml ): Profiles configuration for dbt, specifying different environments and their settings. This contains examples for both Civis dbt scripts and Civis studio environment configurations. Civis Studios follow a slightly different environment variable naming convention, so this file includes examples for use directly in dbt scripts (`dbt-civis`), use directly in a studio (`civis-studio`), as well as an example of a configuration that works both in a studio and in dbt scripts (`studio-or-dbt-script`). +- [`profiles.yml`]( profiles.yml ): Profiles configuration for dbt, specifying different environments and their settings. This contains examples for both Civis dbt scripts and Civis studio environment configurations. Civis Studios follow a slightly different environment variable naming convention, so this file includes examples for use in dbt scripts (`dbt-civis`) and use in a studio (`civis-studio`). Note that Platform dbt scripts can autogenerate this file based on environment variables set on the job! - `models/`: Directory containing SQL models for dbt. - `docs.md`: Documentation for the models. - [`overview.md`]( ./models/overview.md )): Overview of the dbt project. diff --git a/dbt/dbt_project.yml b/dbt/dbt_project.yml index f006f5b..72bfffd 100644 --- a/dbt/dbt_project.yml +++ b/dbt/dbt_project.yml @@ -3,7 +3,7 @@ name: 'civis_jaffle_shop_example' config-version: 2 version: '0.1' -profile: 'studio-or-dbt-script' +profile: 'dbt-civis' model-paths: ["models"] seed-paths: ["seeds"] diff --git a/dbt/profiles.yml b/dbt/profiles.yml index a186bd9..3c71e92 100644 --- a/dbt/profiles.yml +++ b/dbt/profiles.yml @@ -36,7 +36,7 @@ dbt-civis: # Civis Studios have slightly different environment variables than # dbt scripts in Platform (see the dbt-civis profile above for comparison). # -# The following profiles demonstrate the necessary changes to use the Civis Studio environment: +# The following profiles demonstrate the necessary configuration to use dbt in the Civis Studio environment. # For a database parameter named "civis_consumers": # "params": [ # { @@ -65,82 +65,23 @@ civis-studio: schema: "{{ env_var('DBT_SCHEMA') }}" threads: 4 - # TODO - bigquery_service_account_adapter: - type: "{{ env_var('CIVIS_CONSUMER_TYPE') }}" - method: "{{ env_var('GCP_AUTH_METHOD') }}" - project: "{{ env_var('GCP_PROJECT_ID') }}" - schema: "{{ env_var('DBT_SCHEMA') }}" - keyfile: "{{ env_var('GCP_SERVICE_ACCOUNT_KEYFILE_JSON') }}" - threads: 4 - bigquery_oauth_adapter: threads: 4 # Must be a value of 1 or greater type: "{{ env_var('CIVIS_CONSUMER_TYPE') }}" - method: "{{ env_var('GCP_AUTH_METHOD') }}" - project: "{{ env_var('GCP_PROJECT_ID') }}" + method: "{{ env_var('CIVIS_CONSUMER_GCP_AUTH_METHOD') }}" + project: "{{ env_var('CIVIS_CONSUMER_GCP_PROJECT_ID') }}" schema: "{{ env_var('DBT_SCHEMA') }}" - token: "{{ env_var('DBT_ENV_SECRET_GCP_ACCESS_TOKEN') }}" + token: "{{ env_var('DBT_ENV_SECRET_CIVIS_CONSUMER_GCP_ACCESS_TOKEN') }}" -# The following can be used in both studios and Platform dbt scripts, assuming -# parameters* like the following exist in the studio: -# "params": [ -# { -# "name": "civis_consumers", // a redshift database parameter -# "description": null, -# "type": "database", -# "value": "{\"credential\":1234,\"database\":5678}" -# }, { -# "name": "dbt_schema", -# "description": null, -# "type": "string", -# "value": "my_dbt_schema" -# }, { -# "name": "code_cloud_redshift_prefix", -# "description": null, -# "type": "string", -# "value": "civis_consumers" // matches the parameter name for the redshift database -# }, { -# "name": "civis_international_consumers", // a bigquery database parameter -# "description": null, -# "type": "database", -# "value": "{\"credential\":333,\"database\":999}" -# }, { -# "name": "code_cloud_bigquery_prefix", -# "description": null, -# "type": "string", -# "value": "civis_international_consumers" // matches the parameter name for the BQ database -# } -# ] -# *Note: this example includes parameters for both redshift and bigquery, but you'll likely only need -# one or the other -studio-or-dbt-script: - target: redshift_adapter # sets the default target - outputs: - # Works for redshift or postgres - redshift_adapter: - type: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ '_TYPE') }}" - host: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ '_HOST') }}" - port: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', 'DATABASE') ~ '_PORT') | as_number }}" - user: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', '') ~ '_CREDENTIAL_USERNAME', 'DATABASE_USERNAME') }}" - pass: "{{ env_var('DBT_ENV_SECRET_' ~ env_var('CODE_CLOUD_REDSHIFT_PREFIX', '') ~ '_CREDENTIAL_PASSWORD', 'DBT_ENV_SECRET_DATABASE_PASSWORD') }}" - dbname: "{{ env_var(env_var('CODE_CLOUD_REDSHIFT_PREFIX', '') ~ '_DATABASE', 'DATABASE_NAME') }}" - schema: "{{ env_var('DBT_SCHEMA') }}" - threads: 4 - - # TODO + # If you'd like to connect to BigQuery using a service account, you can do so by + # writing the credential password (`CIVIS_CONSUMER_CREDENTIAL_PASSWORD` in this example) + # to a new file of your choice, then add a `keyfile` param with the file path. + # Be careful not to commit the keyfile to source control. bigquery_service_account_adapter: - type: "{{ env_var(env_var('CODE_CLOUD_BIGQUERY_PREFIX', 'DATABASE') ~ '_TYPE') }}" - method: "{{ env_var('GCP_AUTH_METHOD') }}" - project: "{{ env_var('GCP_PROJECT_ID') }}" + type: "{{ env_var('CIVIS_CONSUMER_TYPE') }}" + method: "{{ env_var('CIVIS_CONSUMER_GCP_AUTH_METHOD') }}" + project: "{{ env_var('CIVIS_CONSUMER_GCP_PROJECT_ID') }}" schema: "{{ env_var('DBT_SCHEMA') }}" - keyfile: "{{ env_var('GCP_SERVICE_ACCOUNT_KEYFILE_JSON') }}" + keyfile: "path/to/keyfile.json" # replace this with the path to your keyfile threads: 4 - bigquery_oauth_adapter: - threads: 4 - type: "{{ env_var(env_var('CODE_CLOUD_BIGQUERY_PREFIX', 'DATABASE') ~ '_TYPE') }}" - method: "{{ env_var('GCP_AUTH_METHOD') }}" - project: "{{ env_var('GCP_PROJECT_ID') }}" - schema: "{{ env_var('DBT_SCHEMA') }}" - token: "{{ env_var('DBT_ENV_SECRET_GCP_ACCESS_TOKEN') }}" From 4033fc70d51bb767ddc62a3e550fb3e1cda61a7b Mon Sep 17 00:00:00 2001 From: "Gabby Skifstad (she/her)" Date: Wed, 12 Nov 2025 11:30:07 -0800 Subject: [PATCH 8/8] change plural: one profile --- dbt/profiles.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/profiles.yml b/dbt/profiles.yml index 3c71e92..b9da011 100644 --- a/dbt/profiles.yml +++ b/dbt/profiles.yml @@ -36,7 +36,7 @@ dbt-civis: # Civis Studios have slightly different environment variables than # dbt scripts in Platform (see the dbt-civis profile above for comparison). # -# The following profiles demonstrate the necessary configuration to use dbt in the Civis Studio environment. +# The following profile demonstrates the necessary configuration to use dbt in the Civis Studio environment. # For a database parameter named "civis_consumers": # "params": [ # {