From 458adbbbcf3ab496e5a514809e3729085c22ef11 Mon Sep 17 00:00:00 2001 From: Gabe Mendoza Date: Tue, 22 Jul 2025 09:02:47 -0500 Subject: [PATCH 1/8] update env vars --- dbt/profiles.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dbt/profiles.yml b/dbt/profiles.yml index 157dcc0..e6a8af2 100644 --- a/dbt/profiles.yml +++ b/dbt/profiles.yml @@ -2,17 +2,17 @@ dbt-civis: target: redshift_adapter outputs: redshift_adapter: - type: "{{ env_var('DATABASE_TYPE') }}" - host: "{{ env_var('DATABASE_HOST') }}" - port: "{{ env_var('DATABASE_PORT') | as_number }}" - user: "{{ env_var('DATABASE_USERNAME') }}" - pass: "{{ env_var('DBT_ENV_SECRET_DATABASE_PASSWORD') }}" - dbname: "{{ env_var('DATABASE_NAME') }}" + type: "{{ env_var('DB_TYPE') }}" + host: "{{ env_var('DB_HOST') }}" + port: "{{ env_var('DB_PORT') | as_number }}" + user: "{{ env_var('DB_CREDENTIAL_USERNAME') }}" + pass: "{{ env_var('DB_CREDENTIAL_PASSWORD') }}" + dbname: "{{ env_var('DB_NAME') }}" schema: "{{ env_var('DBT_SCHEMA') }}" threads: 4 bigquery_service_account_adapter: - type: "{{ env_var('DATABASE_TYPE') }}" + type: "{{ env_var('DB_TYPE') }}" method: "{{ env_var('GCP_AUTH_METHOD') }}" project: "{{ env_var('GCP_PROJECT_ID') }}" schema: "{{ env_var('DBT_SCHEMA') }}" @@ -21,7 +21,7 @@ dbt-civis: bigquery_oauth_adapter: threads: 4 # Must be a value of 1 or greater - type: "{{ env_var('DATABASE_TYPE') }}" + type: "{{ env_var('DB_TYPE') }}" method: "{{ env_var('GCP_AUTH_METHOD') }}" project: "{{ env_var('GCP_PROJECT_ID') }}" schema: "{{ env_var('DBT_SCHEMA') }}" From 777612cc1a93bfb46fbde198134f995ce6e5df7a Mon Sep 17 00:00:00 2001 From: Gabe Mendoza Date: Tue, 22 Jul 2025 09:24:21 -0500 Subject: [PATCH 2/8] update dbtreadme --- dbt/README.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/dbt/README.md b/dbt/README.md index 3323fa1..c52c343 100644 --- a/dbt/README.md +++ b/dbt/README.md @@ -16,6 +16,45 @@ This directory contains the configuration files, models, and seeds for the dbt ( - `stg_payments.sql`: SQL model for staging payments. - `seeds/`: Directory containing seed files for dbt. +## Essential dbt Commands + +### Core Development Commands +- `dbt run` - Executes your models to create tables/views in the database +- `dbt test` - Runs data quality tests defined in your models +- `dbt build` - Runs models and tests together (recommended workflow) +- `dbt compile` - Compiles SQL without executing (useful for syntax checking) + +### Data Management Commands +- `dbt seed` - Loads CSV files from `seeds/` directory into your database +- `dbt snapshot` - Creates snapshots for slowly changing dimensions +- `dbt clean` - Removes compiled files and artifacts from your project + +### Project Management Commands +- `dbt deps` - Downloads package dependencies defined in `packages.yml` +- `dbt debug` - Tests database connections and validates project setup +- `dbt docs generate` - Creates documentation for your project +- `dbt docs serve` - Serves documentation locally in your browser + +### Selection and Targeting +- `dbt run --select model_name` - Run a specific model +- `dbt run --select +model_name` - Run model and all its upstream dependencies +- `dbt run --select model_name+` - Run model and all its downstream dependencies +- `dbt test --select source:*` - Test all source data freshness and quality +- `dbt run --select staging.*` - Run all models in the staging folder + +### Useful Flags +- `--full-refresh` - Rebuild incremental models completely from scratch +- `--fail-fast` - Stop execution on first failure +- `--target dev` - Use a specific profile target (dev, prod, etc.) +- `--vars '{key: value}'` - Pass variables to your dbt project + +### Typical Workflow for This Project +1. `dbt seed` - Load the CSV files from seeds/ into your database +2. `dbt run --select staging.*` - Run staging models (stg_customers, stg_orders, stg_payments) +3. `dbt run --select customers orders` - Run final transformation models +4. `dbt test` - Validate data quality with tests +5. `dbt docs generate && dbt docs serve` - Generate and view documentation + ## Configuration ### dbt_project.yml From a83e26c4d2b79825a2f5b2ca77b85475f76f1716 Mon Sep 17 00:00:00 2001 From: Gabe Mendoza Date: Tue, 22 Jul 2025 15:17:27 +0000 Subject: [PATCH 3/8] ignore --- dbt/.env.example | 7 +++++++ dbt/.gitignore | 1 + 2 files changed, 8 insertions(+) create mode 100644 dbt/.env.example diff --git a/dbt/.env.example b/dbt/.env.example new file mode 100644 index 0000000..8c7b5ca --- /dev/null +++ b/dbt/.env.example @@ -0,0 +1,7 @@ +DB_HOST=redshift-something.civis.orgs.civis.io +DB_NAME=dev +DB_PORT=5432 +DB_TYPE=redshift +DB_CREDENTIAL_PASSWORD=replace-with-actual-password +DB_CREDENTIAL_USERNAME=someuser +DBT_SCHEMA=someschema \ No newline at end of file diff --git a/dbt/.gitignore b/dbt/.gitignore index a6fdb0b..35f0cb9 100644 --- a/dbt/.gitignore +++ b/dbt/.gitignore @@ -2,3 +2,4 @@ target/ .user.yml logs/ target.tar.gz +.env \ No newline at end of file From 541322f5b81f9c6867be229475f24d86fc81af15 Mon Sep 17 00:00:00 2001 From: Gabe Mendoza Date: Tue, 22 Jul 2025 15:19:59 +0000 Subject: [PATCH 4/8] save --- dbt/.env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/.env.example b/dbt/.env.example index 8c7b5ca..a679065 100644 --- a/dbt/.env.example +++ b/dbt/.env.example @@ -1,4 +1,4 @@ -DB_HOST=redshift-something.civis.orgs.civis.io +DB_HOST=redshift-something.url.com DB_NAME=dev DB_PORT=5432 DB_TYPE=redshift From 56fd8ee252cfa746022bb1f292cb3e1d054c03c9 Mon Sep 17 00:00:00 2001 From: Gabe Mendoza Date: Tue, 22 Jul 2025 20:33:43 +0000 Subject: [PATCH 5/8] exmaples env --- dbt/.env.example | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dbt/.env.example b/dbt/.env.example index a679065..e527273 100644 --- a/dbt/.env.example +++ b/dbt/.env.example @@ -1,7 +1,7 @@ -DB_HOST=redshift-something.url.com -DB_NAME=dev -DB_PORT=5432 -DB_TYPE=redshift -DB_CREDENTIAL_PASSWORD=replace-with-actual-password -DB_CREDENTIAL_USERNAME=someuser -DBT_SCHEMA=someschema \ No newline at end of file +export DB_HOST=redshift-something.civis.orgs.civis.io +export DB_NAME=dev +export DB_PORT=5432 +export DB_TYPE=redshift +export DB_CREDENTIAL_PASSWORD=replace-with-actual-password +export DB_CREDENTIAL_USERNAME=someuser +export DBT_SCHEMA=someschema \ No newline at end of file From 53a429031819f309217bcf4329dbcbdedfd00e67 Mon Sep 17 00:00:00 2001 From: Gabe Mendoza Date: Mon, 28 Jul 2025 20:34:25 +0000 Subject: [PATCH 6/8] smol changes --- dbt/.python-version | 1 + dbt/do-thing.sh | 8 ++++++++ dbt/main.py | 6 ++++++ dbt/pyproject.toml | 7 +++++++ 4 files changed, 22 insertions(+) create mode 100644 dbt/.python-version create mode 100755 dbt/do-thing.sh create mode 100644 dbt/main.py create mode 100644 dbt/pyproject.toml diff --git a/dbt/.python-version b/dbt/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/dbt/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/dbt/do-thing.sh b/dbt/do-thing.sh new file mode 100755 index 0000000..a5e203a --- /dev/null +++ b/dbt/do-thing.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# dbt test --select dbt_project.yml --project-dir /workspace/platform-code-examples/dbt --profiles-dir /workspace/platform-code-examples/dbt --target redshift_adapter +# curl -LsSf https://astral.sh/uv/install.sh | sh +# sudo apt-get install build-essential procps curl git +# /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +# curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.3/install.sh | bash +# awk -F: '{printf "%s:%s\n",$1,$3}' /etc/passwd +curl https://mise.run | sh \ No newline at end of file diff --git a/dbt/main.py b/dbt/main.py new file mode 100644 index 0000000..ecc039b --- /dev/null +++ b/dbt/main.py @@ -0,0 +1,6 @@ +def main(): + print("Hello from dbt!") + + +if __name__ == "__main__": + main() diff --git a/dbt/pyproject.toml b/dbt/pyproject.toml new file mode 100644 index 0000000..3e19b7d --- /dev/null +++ b/dbt/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "dbt" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [] From e860a107ad2d6744ea48b7d3e42657dccf66f78c Mon Sep 17 00:00:00 2001 From: Gabe Mendoza Date: Wed, 30 Jul 2025 15:18:13 +0000 Subject: [PATCH 7/8] save file --- newfile.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newfile.txt diff --git a/newfile.txt b/newfile.txt new file mode 100644 index 0000000..e69de29 From 6a7b8d5c3b9a2d2b63ad71579b2fcb35f8a12ebc Mon Sep 17 00:00:00 2001 From: Gabe Mendoza Date: Tue, 12 Aug 2025 12:13:01 -0500 Subject: [PATCH 8/8] mise toml --- .gitignore | 1 + mise.toml | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 mise.toml diff --git a/.gitignore b/.gitignore index 4201825..c82284d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ target** logs/ manifest.json static_index.html +!mise.toml diff --git a/mise.toml b/mise.toml new file mode 100644 index 0000000..f77350e --- /dev/null +++ b/mise.toml @@ -0,0 +1,11 @@ +[env] +_.python.venv = { path = ".venv", create = true } + +[settings] +python.uv_venv_auto = true + +[tools] +jq = "latest" +python = "3.12" +uv = "latest" +