docker-compose.mapped-packages.yaml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

version: '3.9'

x-aliases:
  - &common_parameters
      networks:
      - airflow-cwl-docker
  - &airflow_build_params
      build:
        context: .
        dockerfile: Dockerfile
        args:
          AIRFLOW_CONDA_ENV: "${AIRFLOW_CONDA_ENV:-none}"

  - &airflow_env
      environment:
        AIRFLOW_HOME: /opt/airflow
        AIRFLOW__CORE__EXECUTOR: LocalExecutor
        AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${POSTGRE_USER:-airflow}:${POSTGRE_PASS:-airflow}@${POSTGRE_SERVER:-postgres}/${POSTGRE_DB:-airflow}
        AIRFLOW__CORE__FERNET_KEY:
        AIRFLOW_CONN_METADATA_DB: postgresql+psycopg2://${POSTGRE_USER:-airflow}:${POSTGRE_PASS:-airflow}@${POSTGRE_SERVER:-postgres}/${POSTGRE_DB:-airflow}
        AIRFLOW_VAR__METADATA_DB_SCHEMA: ${POSTGRE_DB:-airflow}
        AIRFLOW__SCHEDULER__SCHEDULER_HEARTBEAT_SEC: 10
        AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "True"
        AIRFLOW__CORE__LOAD_EXAMPLES: ${AIRFLOW__CORE__LOAD_EXAMPLES:-False}
        AIRFLOW__CORE__DAGS_FOLDER: ${DAGS_FOLDER:-/opt/airflow/dags}
        _AIRFLOW_DB_UPGRADE: "True"
        _AIRFLOW_WWW_USER_CREATE: "True"
        AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True"
        AIRFLOW__WEBSERVER__EXPOSE_CONFIG: ${AIRFLOW__WEBSERVER__EXPOSE_CONFIG:-True}
        _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
        _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}
        AIRFLOW__WEBSERVER__BASE_URL: ${BASE_URL:-http://localhost:8080}
        AIRFLOW__LOGGING__BASE_LOG_FOLDER: "/opt/airflow/logs"
        PROJECT_DIR: "/opt/airflow/project"
        AIRFLOW_CONDA_ENV: "${AIRFLOW_CONDA_ENV:-none}"

  - &airflow_volumes
      volumes:
        - ${PROJECT_DIR:-./project}:/opt/airflow/project
        - ${DAGS_DIR:-./dags}:/opt/airflow/dags
        - ${LOGS_DIR:-./airflow-logs}:/opt/airflow/logs
        - ${CWL_TMP_FOLDER:-./cwl_tmp_folder}:/opt/airflow/cwl_tmp_folder
        - ${CWL_INPUTS_FOLDER:-./cwl_inputs_folder}:/opt/airflow/cwl_inputs_folder
        - ${CWL_OUTPUTS_FOLDER:-./cwl_outputs_folder}:/opt/airflow/cwl_outputs_folder
        - ${CWL_PICKLE_FOLDER:-./cwl_pickle_folder}:/opt/airflow/cwl_pickle_folder
        - ./cwl-airflow/cwl_airflow:/root/anaconda/envs/nsaph/lib/python3.8/site-packages/cwl_airflow
        - ./project/census/src/python/census:/root/anaconda/envs/nsaph/lib/python3.8/site-packages/census
        - ./project/epa/src/python/epa:/root/anaconda/envs/nsaph/lib/python3.8/site-packages/epa
        - ./project/gridmet/src/python/gridmet:/root/anaconda/envs/nsaph/lib/python3.8/site-packages/gridmet
        - ./project/data_platform/src/python/nsaph:/root/anaconda/envs/nsaph/lib/python3.8/site-packages/nsaph/
        - ./project/gis/nsaph_gis:/root/anaconda/envs/nsaph/lib/python3.8/site-packages/nsaph_gis/
        - ./project/nsaph_utils/nsaph_utils:/root/anaconda/envs/nsaph/lib/python3.8/site-packages/nsaph_utils
        - type: bind
          source: /var/run/docker.sock
          target: /var/run/docker.sock

services:
  scheduler:
    image: "${AIRFLOW_IMAGE_NAME:-myairflow-conda}"
    <<: *airflow_build_params
    restart: always
    user: root
    container_name: scheduler
    depends_on:
      webserver:
        condition: service_healthy
    <<: *airflow_env
    <<: *airflow_volumes
    <<: *common_parameters
    command: [ "airflow", "scheduler" ]

  webserver:
    image: "${AIRFLOW_IMAGE_NAME:-myairflow-conda}"
    <<: *airflow_build_params
    container_name: webserver
    restart: always
    user: root
    <<: *airflow_env
    <<: *airflow_volumes
    <<: *common_parameters
    ports:
      - "8080:8080"
    command: [ "airflow", "webserver" ]
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
      interval: 30s
      timeout: 4s
      retries: 5

  postgres:
    image: postgres:13
    container_name: postgres
    volumes:
      - postgres-db-volume:/var/lib/postgresql/data
    environment:
      POSTGRES_USER: ${POSTGRE_USER:-airflow}
      POSTGRES_PASSWORD: ${POSTGRE_PASS:-airflow}
      POSTGRES_DB: ${POSTGRE_DB:-airflow}
      POSTGRES_HOST_AUTH_METHOD: trust
    <<: *common_parameters
    ports:
      - "${POSTGRES_PORT:-5432}:5432"
    profiles:
      - postgres
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${POSTGRE_USER} -d ${POSTGRE_DB}"]
      interval: 3s
      timeout: 5s
      retries: 50

volumes:
  postgres-db-volume:
networks:
  airflow-cwl-docker:
    driver: bridge
    name: airflow
    ipam:
      config:
        - subnet: 172.16.238.0/24