real-estate-ai-qa/docker-compose.yaml at main · a-partha/real-estate-ai-qa · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
services:
  postgres:
    image: postgres:15-alpine
    environment:
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
      POSTGRES_DB: airflow
    volumes:
      - postgres-data:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "airflow"]
      interval: 5s
      retries: 5
    networks:
      - acris-net


  redis:
    image: redis:7-alpine
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 5s
      retries: 5
    networks:
      - acris-net

  airflow-init:
    build:
      context: ./airflow_image
    env_file:
      - .env
    container_name: airflow_init
    environment:
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/airflow
      AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/airflow
      AIRFLOW__CELERY__BROKER_URL: redis://redis:6379/0
    volumes:
      - ./dags:/opt/airflow/dags
      - ./data:/opt/airflow/data
      - ./artifacts:/opt/airflow/artifacts
      - ./dbt:/opt/airflow/dbt
    networks:
      - acris-net
    entrypoint: /bin/bash
    command:
      - -c
      - |
        while ! nc -z postgres 5432; do
          echo "Waiting for postgres..."
          sleep 1
        done
        echo "Postgres is up and running!"
        airflow db migrate
    depends_on:
      postgres:
        condition: service_healthy
      redis:
        condition: service_healthy

  airflow-webserver:
    build:
      context: ./airflow_image
    env_file:
      - .env
    environment:
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/airflow
      AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/airflow
      AIRFLOW__CELERY__BROKER_URL: redis://redis:6379/0

    volumes:
      - ./dags:/opt/airflow/dags
      - ./data:/opt/airflow/data
      - ./artifacts:/opt/airflow/artifacts
      - ./dbt:/opt/airflow/dbt
    command: webserver
    ports:
      - "8080:8080"
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
      interval: 30s
      start_period: 30s
      retries: 5
    networks:
      acris-net:
        aliases:
          - airflow-webserver
          - webserver
    depends_on:
      airflow-init:
        condition: service_completed_successfully

  airflow-scheduler:
    build:
      context: ./airflow_image
    env_file:
      - .env
    environment:
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/airflow
      AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/airflow
      AIRFLOW__CELERY__BROKER_URL: redis://redis:6379/0

    volumes:
      - ./dags:/opt/airflow/dags
      - ./data:/opt/airflow/data
      - ./artifacts:/opt/airflow/artifacts
      - ./dbt:/opt/airflow/dbt
    command: scheduler
    networks:
      - acris-net
    depends_on:
      airflow-init:
        condition: service_completed_successfully

  airflow-worker:
    build:
      context: ./airflow_image
    env_file:
      - .env
    environment:
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/airflow
      AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/airflow
      AIRFLOW__CELERY__BROKER_URL: redis://redis:6379/0

    volumes:
      - ./dags:/opt/airflow/dags
      - ./data:/opt/airflow/data
      - ./artifacts:/opt/airflow/artifacts
      - ./dbt:/opt/airflow/dbt
    command: celery worker
    networks:
      - acris-net
    depends_on:
      airflow-init:
        condition: service_completed_successfully

  api:
    build:
      context: ./api
      dockerfile: Dockerfile_Pinecone
    env_file:
      - .env
    environment:
      GOOGLE_APPLICATION_CREDENTIALS: /app/bigquery-service-key.json
    depends_on:
      airflow-webserver:
        condition: service_healthy
    volumes:
      - ./api:/app
      - ./dags:/opt/airflow/dags
      - ./artifacts:/app/artifacts:ro
      - ./web/dist:/app/web/dist
      - ./bigquery-service-key.json:/app/bigquery-service-key.json:ro
    ports:
      - "8000:8000"
    networks:
      acris-net:
        aliases:
          - api-service
    # extra_hosts:
    #   - "airflow-webserver:172.18.0.5"

volumes:
  postgres-data:

networks:
  acris-net:
    driver: bridge