docker-compose.override.yml.example

# docker-compose.override.yml.example
# 
# This file demonstrates how to override default configurations
# Copy to docker-compose.override.yml and customize as needed
#
# Usage: docker compose up -d (will automatically use override file)

version: '3.8'

# ==============================================================================
# ENVIRONMENT VARIABLES - Copy these to your .env file and customize
# ==============================================================================
# HOST_IP=192.168.1.100                    # Set for remote server access
# LAKEHOUSE_ROOT=./lakehouse-data           # Data directory location  
# POSTGRES_MEMORY_LIMIT=4G                 # PostgreSQL memory allocation
# SPARK_WORKER_MEMORY=16g                  # Spark worker memory per instance
# SUPERSET_WORKERS=8                       # Superset worker processes

services:
  # ==============================================================================
  # REMOTE SERVER ACCESS CONFIGURATION
  # ==============================================================================
  
  # Example: Configure lakehouse-init for remote server deployment
  lakehouse-init:
    environment:
      # Override HOST_IP for specific server deployment
      - HOST_IP=your.server.ip.address
      # Use external storage for larger deployments
      - LAKEHOUSE_ROOT=/mnt/lakehouse-storage
      # Adjust timeouts for slower networks
      - MINIO_READY_TIMEOUT=30
      - MINIO_READY_INTERVAL=5

  # ==============================================================================
  # ENHANCED SERVICE CONFIGURATIONS
  # ==============================================================================

  # Example: Enhanced MinIO configuration with SSL
  minio:
    volumes:
      - ./ssl-certs:/certs:ro
    environment:
      - MINIO_OPTS=--certs-dir /certs
    # Add extra environment variables
    # - MINIO_BROWSER_REDIRECT_URL=https://your-domain.com:9001

  # Example: Custom Spark configuration and dependencies
  spark-master:
    environment:
      # Increase memory for master
      - SPARK_MASTER_MEMORY=4g
      - SPARK_MASTER_CORES=4
    volumes:
      # Add custom Spark configuration
      - ./custom-spark-conf:/opt/spark/conf/custom:ro
      # Add custom JAR files
      - ./custom-spark-jars:/opt/spark/jars/custom:ro

  spark-worker:
    environment:
      # Custom worker memory and cores
      - SPARK_WORKER_MEMORY=16g
      - SPARK_WORKER_CORES=8
      # Add custom classpath
      - SPARK_CLASSPATH=/opt/spark/jars/custom/*
    volumes:
      # Mount custom JAR files (same as master)
      - ./custom-spark-jars:/opt/spark/jars/custom:ro
      # Mount custom datasets for local processing
      - ./local-datasets:/opt/datasets:ro

  # Example: Enhanced Jupyter configuration with DuckDB 1.3.2
  jupyter:
    environment:
      # NOTE: JUPYTER_TOKEN is auto-generated - see ./scripts/show-credentials.sh
      # Custom Jupyter token (optional override)
      # - JUPYTER_TOKEN=your-custom-token
      
      # Enable Jupyter extensions
      - JUPYTER_ENABLE_LAB=yes
      
      # Enhanced Spark settings for large analytics
      - SPARK_DRIVER_MEMORY=8g
      - SPARK_EXECUTOR_MEMORY=8g
      - SPARK_EXECUTOR_CORES=4
      - SPARK_DRIVER_MAX_RESULT_SIZE=4g
      
      # DuckDB configuration for data lake analytics
      - DUCKDB_MEMORY_LIMIT=8GB
      - DUCKDB_THREADS=4
    volumes:
      # Mount additional Python packages
      - ./custom-python-packages:/home/jovyan/custom-packages:ro
      # Mount external datasets for analysis
      - ./external-data:/home/jovyan/external-data:ro
      # Mount custom DuckDB extensions
      - ./custom-duckdb-extensions:/home/jovyan/.duckdb/extensions:ro
    # Expose additional ports if needed for custom services
    # ports:
    #   - "8889:8888"  # Alternative Jupyter port
    #   - "8050:8050"  # For Dash/Plotly apps

  # Example: Enhanced Airflow configuration
  airflow-webserver:
    environment:
      # Custom webserver settings
      - AIRFLOW__WEBSERVER__WORKERS=4
      - AIRFLOW__WEBSERVER__SECRET_KEY=your-custom-secret-key
      # Enable RBAC and custom auth
      - AIRFLOW__WEBSERVER__RBAC=True
    volumes:
      # Mount custom Airflow plugins
      - ./custom-airflow-plugins:/opt/airflow/plugins:ro
      # Mount custom requirements
      - ./custom-requirements.txt:/opt/airflow/requirements.txt:ro

  airflow-scheduler:
    environment:
      # Enhanced scheduler performance
      - AIRFLOW__SCHEDULER__MAX_THREADS=8
      - AIRFLOW__CORE__PARALLELISM=64
      - AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG=32
    volumes:
      # Same custom volumes as webserver
      - ./custom-airflow-plugins:/opt/airflow/plugins:ro
      - ./custom-requirements.txt:/opt/airflow/requirements.txt:ro

  # Example: Enhanced Superset configuration with dual analytics
  superset:
    environment:
      # NOTE: SUPERSET_SECRET_KEY is auto-generated - see ./scripts/show-credentials.sh
      # Custom secret key (optional override - use strong random string)
      # - SUPERSET_SECRET_KEY=your-very-long-random-secret-key-here
      
      # Enhanced performance settings
      - SUPERSET_WORKERS=8
      - SUPERSET_TIMEOUT=120
      - SUPERSET_WEBSERVER_TIMEOUT=120
      
      # Enable additional features
      - SUPERSET_FEATURE_FLAGS='{"ENABLE_TEMPLATE_PROCESSING": true, "DASHBOARD_NATIVE_FILTERS": true}'
      
      # Database connections (both DuckDB and PostgreSQL pre-configured)
      # - SUPERSET_CONFIG_PATH=/app/superset_config.py
    volumes:
      # Mount custom Superset configuration
      - ./custom-superset-config.py:/app/superset_config.py:ro
      # Mount custom themes/assets  
      - ./custom-superset-assets:/app/superset/static/assets/custom:ro
      # Mount additional DuckDB database files
      - ./custom-duckdb-files:/app/superset_home/databases:ro
    # Enhanced health check
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8088/health"]
      interval: 30s
      timeout: 15s
      retries: 3
      start_period: 90s

  # Example: Enhanced PostgreSQL configuration for analytics
  postgres:
    environment:
      # Performance tuning for analytics workloads
      - POSTGRES_SHARED_BUFFERS=512MB           # Increase for larger datasets
      - POSTGRES_EFFECTIVE_CACHE_SIZE=4GB       # Set to ~75% of available RAM
      - POSTGRES_MAINTENANCE_WORK_MEM=256MB     # Increase for faster index builds
      - POSTGRES_WORK_MEM=32MB                  # Per-connection memory for sorts/joins
      - POSTGRES_CHECKPOINT_COMPLETION_TARGET=0.9
      - POSTGRES_WAL_BUFFERS=64MB
      - POSTGRES_DEFAULT_STATISTICS_TARGET=100
      # Enable connection pooling
      - POSTGRES_MAX_CONNECTIONS=200
    volumes:
      # Mount custom postgres configuration
      - ./custom-postgres.conf:/etc/postgresql/postgresql.conf:ro
      # Mount initialization scripts for analytics schema
      - ./postgres-init-scripts:/docker-entrypoint-initdb.d:ro
      # Use external storage for better performance
      # - /mnt/fast-storage/postgres:/var/lib/postgresql/data
    command: >
      postgres
      -c shared_buffers=512MB
      -c effective_cache_size=4GB
      -c maintenance_work_mem=256MB
      -c checkpoint_completion_target=0.9
      -c wal_buffers=64MB
      -c default_statistics_target=100
      -c random_page_cost=1.1
      -c effective_io_concurrency=200
      -c work_mem=32MB
      -c max_connections=200

  # Example: Enhanced Portainer configuration
  portainer:
    environment:
      # Hide specific labels from view
      - PORTAINER_HIDE_LABELS=dev,staging
    volumes:
      # Mount custom Portainer data location
      # - ./portainer-data:/data
      # Mount additional Docker endpoints
      # - /var/run/docker.sock:/var/run/docker.sock:ro

  # Example: Add optional Homer service with custom config
  homer:
    profiles:
      - full  # Only start with specific profile
    volumes:
      # Mount custom Homer configuration
      - ./custom-homer-config.yml:/www/assets/config.yml:ro
      # Mount custom Homer assets
      - ./custom-homer-assets:/www/assets/custom:ro

# Example: Add custom networks
networks:
  lakehouse:
    driver: bridge
    ipam:
      config:
        - subnet: 172.20.0.0/16

# Example: Add external database for Superset metadata
  external-postgres:
    image: postgres:16
    environment:
      POSTGRES_DB: superset_meta
      POSTGRES_USER: superset
      POSTGRES_PASSWORD: secure-password-here
    volumes:
      - external-superset-db:/var/lib/postgresql/data
    networks:
      - lakehouse
    # Only start if explicitly requested
    profiles:
      - external-db

# Example: Add Redis for Airflow Celery executor (if needed)
  redis:
    image: redis:7-alpine
    command: redis-server --requirepass redis-password
    volumes:
      - redis-data:/data
    networks:
      - lakehouse
    profiles:
      - celery

# Example: Add monitoring with Grafana
  grafana:
    image: grafana/grafana:latest
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin123
      - GF_USERS_ALLOW_SIGN_UP=false
    ports:
      - "3000:3000"
    volumes:
      - grafana-data:/var/lib/grafana
      - ./grafana-dashboards:/etc/grafana/provisioning/dashboards:ro
      - ./grafana-datasources:/etc/grafana/provisioning/datasources:ro
    networks:
      - lakehouse
    profiles:
      - monitoring

# Example: Add Prometheus for metrics collection
  prometheus:
    image: prom/prometheus:latest
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--web.console.libraries=/etc/prometheus/console_libraries'
      - '--web.console.templates=/etc/prometheus/consoles'
      - '--web.enable-lifecycle'
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus-config.yml:/etc/prometheus/prometheus.yml:ro
      - prometheus-data:/prometheus
    networks:
      - lakehouse
    profiles:
      - monitoring

volumes:
  # Example: External volumes for persistent data
  external-superset-db:
    driver: local
  redis-data:
    driver: local
  grafana-data:
    driver: local
  prometheus-data:
    driver: local

# Example: Custom volume configurations
  # Use external storage for MinIO
  # minio-data:
  #   driver: local
  #   driver_opts:
  #     type: none
  #     o: bind
  #     device: /mnt/external-storage/minio

# ==============================================================================
# DUCKDB & CREDENTIAL MANAGEMENT EXAMPLES
# ==============================================================================

# Example: Create custom initialization script for DuckDB persistent secrets
# Create a file ./custom-init/setup-duckdb-secrets.sql:
#
# -- Auto-configure DuckDB persistent secrets
# CREATE PERSISTENT SECRET minio_prod (
#     TYPE S3,
#     KEY_ID 'admin',
#     SECRET '${MINIO_ROOT_PASSWORD}',
#     ENDPOINT 'minio:9000',
#     USE_SSL false,
#     URL_STYLE 'path',
#     SCOPE 's3://lakehouse'
# );
#
# -- Create additional secrets for external S3 buckets
# CREATE PERSISTENT SECRET aws_s3_prod (
#     TYPE S3,
#     KEY_ID 'your-aws-access-key',
#     SECRET 'your-aws-secret-key',
#     REGION 'us-west-2',
#     SCOPE 's3://your-prod-bucket'
# );

# Example: Mount custom initialization scripts
#  lakehouse-init:
#    volumes:
#      - ./custom-init:/host/custom-init:ro
#    environment:
#      - CUSTOM_INIT_SCRIPTS=/host/custom-init

# ==============================================================================
# PRODUCTION DEPLOYMENT EXAMPLES
# ==============================================================================

# Example configurations you can uncomment and customize:

# For production SSL termination with nginx:
#  nginx:
#    image: nginx:alpine
#    ports:
#      - "80:80"
#      - "443:443"
#    volumes:
#      - ./nginx.conf:/etc/nginx/nginx.conf:ro
#      - ./ssl-certs:/etc/nginx/ssl:ro
#    depends_on:
#      - superset
#      - jupyter
#      - airflow-webserver
#    networks:
#      - lakehouse

# For backup automation:
#  backup:
#    image: alpine:latest
#    command: |
#      sh -c "
#        apk add --no-cache rsync &&
#        while true; do
#          rsync -av /data/ /backup/lakehouse-$(date +%Y%m%d)/
#          sleep 86400
#        done
#      "
#    volumes:
#      - ${LAKEHOUSE_ROOT:-./lakehouse-data}:/data:ro
#      - ./backups:/backup
#    profiles:
#      - backup