airflow_dbt_demo/docker-compose.yml at main · wyang10/airflow_dbt_demo · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
version: "3.8"

x-airflow-common: &airflow-common
  build: ./airflow
  image: local/airflow-dbt:py311
  restart: unless-stopped
  env_file:
    - ./airflow/.env              # 你的 Snowflake 变量
  environment:
    AIRFLOW__CORE__EXECUTOR: LocalExecutor
    AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres:5432/airflow
    AIRFLOW__CORE__LOAD_EXAMPLES: "False"
    AIRFLOW__WEBSERVER__RBAC: "True"
    AIRFLOW__CORE__FERNET_KEY: "${AIRFLOW__CORE__FERNET_KEY:-TN6KCTBau36eJ3SLKN08_K_u5_sNQs-kDKUEdSuSq4o=}"
    AIRFLOW__WEBSERVER__SECRET_KEY: "${AIRFLOW__WEBSERVER__SECRET_KEY:-dev-change-me}"  # override in airflow/.env for login/CSRF
    # SMTP via Mailpit (UI at http://localhost:8025)
    AIRFLOW__SMTP__SMTP_HOST: "${AIRFLOW__SMTP__SMTP_HOST:-mailpit}"
    AIRFLOW__SMTP__SMTP_PORT: "${AIRFLOW__SMTP__SMTP_PORT:-1025}"
    AIRFLOW__SMTP__SMTP_STARTTLS: "${AIRFLOW__SMTP__SMTP_STARTTLS:-False}"
    AIRFLOW__SMTP__SMTP_SSL: "${AIRFLOW__SMTP__SMTP_SSL:-False}"
    AIRFLOW__SMTP__SMTP_MAIL_FROM: "${AIRFLOW__SMTP__SMTP_MAIL_FROM:-airflow@example.com}"
    DBT_PROFILES_DIR: /opt/airflow/dbt
  user: "${AIRFLOW_UID:-50000}:0"
  volumes:
    - airflow_home:/opt/airflow
    - ./airflow/dags:/opt/airflow/dags
    - ./data_pipeline:/opt/airflow/dbt
    - ./great_expectations:/opt/airflow/great_expectations
    - ./secrets:/opt/airflow/secrets:ro
  healthcheck:
    test: ["CMD", "bash", "-c", "airflow db check"]
    interval: 30s
    timeout: 10s
    retries: 5

services:
  gx-docs:
    image: nginx:alpine
    restart: unless-stopped
    ports:
      - "8081:80"
    volumes:
      - ./great_expectations/uncommitted/data_docs/local_site:/usr/share/nginx/html:ro

  mailpit:
    image: axllent/mailpit:latest
    restart: unless-stopped
    ports:
      - "8025:8025"   # Web UI
    environment:
      MP_SMTP_BIND_ADDR: "0.0.0.0:1025"
      MP_UI_BIND_ADDR: "0.0.0.0:8025"
  postgres:
    image: postgres:15-alpine
    environment:
      POSTGRES_USER: airflow
      POSTGRES_PASSWORD: airflow
      POSTGRES_DB: airflow
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U airflow -d airflow"]
      interval: 10s
      timeout: 5s
      retries: 10
    volumes:
      - postgres_data:/var/lib/postgresql/data

  airflow-init:
    <<: *airflow-common
    command: >
      bash -lc "airflow db migrate \
      && (airflow users create --username airflow --password airflow --firstname Admin --lastname User --role Admin --email admin@example.com || true) \
      && (airflow pools set dbt 1 'Serialize dbt CLI across DAGs' || true)"
    restart: "no"
    depends_on:
      postgres:
        condition: service_healthy

  scheduler:
    <<: *airflow-common
    depends_on:
      postgres:
        condition: service_healthy
      airflow-init:
        condition: service_completed_successfully
    command: >
      bash -lc "airflow scheduler"

  webserver:
    <<: *airflow-common
    ports:
      - "${AIRFLOW_PORT:-8080}:8080"
    depends_on:
      postgres:
        condition: service_healthy
      scheduler:
        condition: service_healthy
      airflow-init:
        condition: service_completed_successfully
    command: >
      bash -lc "airflow webserver"

volumes:
  airflow_home:
  postgres_data: