diff --git a/009_define_file_templates.tf b/009_define_file_templates.tf index 999ada3..ae286f9 100644 --- a/009_define_file_templates.tf +++ b/009_define_file_templates.tf @@ -333,6 +333,7 @@ locals { tower_db_dns = module.connection_strings.tower_db_dns, flag_enable_groundswell = var.flag_enable_groundswell, flag_enable_data_studio = var.flag_enable_data_studio, + flag_use_wave = var.flag_use_wave, flag_use_wave_lite = var.flag_use_wave_lite, wave_lite_db_dns = module.connection_strings.wave_lite_db_dns, } diff --git a/010_prepare_config_files.tf b/010_prepare_config_files.tf index da23b5b..3752d60 100644 --- a/010_prepare_config_files.tf +++ b/010_prepare_config_files.tf @@ -61,8 +61,6 @@ resource "null_resource" "generate_independent_config_files" { # Update seqerakit prerun script to pull private cert # https://help.tower.nf/23.2/enterprise/configuration/ssl_tls/ # Note: This approach works for most clients but there can be occasional problems due to chains. - - # Update assets dependent if [[ "${var.flag_use_private_cacert}" == "true" ]]; then { @@ -77,6 +75,17 @@ resource "null_resource" "generate_independent_config_files" { fi + if [[ "${var.flag_use_private_cacert}" == "true" ]]; then + + cat > ${path.module}/assets/target/docker_compose/import-cert.sh << 'EOF' +#!/bin/sh +keytool -import -trustcacerts -cacerts -storepass changeit -noprompt -alias seqera-rootca -file /tmp/rootCA.crt >/dev/null 2>&1 || true +exec /bin/sh "$@" +EOF + chmod +x ${path.module}/assets/target/docker_compose/import-cert.sh + + fi + # THIS IS A TOTAL HACK (Wave-Lite) # Using this technique so postgres can get config files with single quotes only. # Terraform templatefile (.tpl) abandoned and we use real SQL with placeholders-to-be-replaced-by-sed diff --git a/assets/src/ansible/02_update_file_configurations.yml.tpl b/assets/src/ansible/02_update_file_configurations.yml.tpl index e9a2104..81ebcc7 100644 --- a/assets/src/ansible/02_update_file_configurations.yml.tpl +++ b/assets/src/ansible/02_update_file_configurations.yml.tpl @@ -145,8 +145,9 @@ # Grab leaf cert and stash in target/ folder cd /home/ec2-user/target/customcerts - aws s3 cp ${private_cacert_bucket_prefix}/${tower_base_url}.crt ${tower_base_url}.crt + aws s3 cp ${private_cacert_bucket_prefix}/${tower_base_url}.crt ${tower_base_url}.crt aws s3 cp ${private_cacert_bucket_prefix}/${tower_base_url}.key ${tower_base_url}.key + aws s3 cp ${private_cacert_bucket_prefix}/rootCA.crt rootCA.crt %{ endif ~} %{ if flag_enable_data_studio ~} diff --git a/assets/src/docker_compose/docker-compose.yml.tpl b/assets/src/docker_compose/docker-compose.yml.tpl index 79cc11c..d7f9185 100644 --- a/assets/src/docker_compose/docker-compose.yml.tpl +++ b/assets/src/docker_compose/docker-compose.yml.tpl @@ -95,6 +95,9 @@ services: cron: image: cr.seqera.io/private/nf-tower-enterprise/backend:${docker_version} command: -c "/tower.sh" +%{ if flag_use_private_cacert == true ~} + entrypoint: ["/bin/sh", "/tmp/import-cert.sh"] +%{ endif ~} networks: - frontend - backend @@ -102,6 +105,10 @@ services: - $HOME/target/tower_config/tower.yml:/tower.yml %{ if flag_enable_data_studio == true ~} - $HOME/target/tower_config/data-studios-rsa.pem:/data-studios-rsa.pem +%{ endif ~} +%{ if flag_use_private_cacert == true ~} + - $HOME/target/customcerts/rootCA.crt:/tmp/rootCA.crt + - $HOME/target/docker_compose/import-cert.sh:/tmp/import-cert.sh %{ endif ~} env_file: # Seqera environment variables — see https://docs.seqera.io/platform/latest/enterprise/configuration/overview for details @@ -121,6 +128,9 @@ services: %{ else ~} command: -c "/migrate-db.sh; /tower.sh" %{ endif } +%{ if flag_use_private_cacert == true ~} + entrypoint: ["/bin/sh", "/tmp/import-cert.sh"] +%{ endif ~} networks: - frontend - backend @@ -128,6 +138,10 @@ services: - $HOME/target/tower_config/tower.yml:/tower.yml %{ if flag_enable_data_studio == true ~} - $HOME/target/tower_config/data-studios-rsa.pem:/data-studios-rsa.pem +%{ endif ~} +%{ if flag_use_private_cacert == true ~} + - $HOME/target/customcerts/rootCA.crt:/tmp/rootCA.crt + - $HOME/target/docker_compose/import-cert.sh:/tmp/import-cert.sh %{ endif ~} env_file: # Seqera environment variables — see https://docs.seqera.io/platform/latest/enterprise/configuration/overview for details @@ -155,6 +169,9 @@ services: %{ else ~} command: -c "/tower.sh" %{ endif } +%{ if flag_use_private_cacert == true ~} + entrypoint: ["/bin/sh", "/tmp/import-cert.sh"] +%{ endif ~} networks: - frontend - backend @@ -164,6 +181,10 @@ services: - $HOME/target/tower_config/tower.yml:/tower.yml %{ if flag_enable_data_studio == true ~} - $HOME/target/tower_config/data-studios-rsa.pem:/data-studios-rsa.pem +%{ endif ~} +%{ if flag_use_private_cacert == true ~} + - $HOME/target/customcerts/rootCA.crt:/tmp/rootCA.crt + - $HOME/target/docker_compose/import-cert.sh:/tmp/import-cert.sh %{ endif ~} env_file: - $HOME/target/tower_config/tower.env @@ -286,8 +307,16 @@ services: # - 9099:9090 expose: - 9090 +%{ if flag_use_private_cacert == true ~} + entrypoint: ["/bin/sh", "/tmp/import-cert.sh"] + command: ["/launch.sh"] +%{ endif ~} volumes: - $HOME/target/wave_lite_config/wave-lite.yml:/work/config.yml +%{ if flag_use_private_cacert == true ~} + - $HOME/target/customcerts/rootCA.crt:/tmp/rootCA.crt + - $HOME/target/docker_compose/import-cert.sh:/tmp/import-cert.sh +%{ endif ~} #env_file: # - wave-lite.env environment: diff --git a/documentation/design_decisions.md b/documentation/design_decisions.md index efe42b6..7f1934e 100644 --- a/documentation/design_decisions.md +++ b/documentation/design_decisions.md @@ -265,3 +265,17 @@ In addition to the general design decisions noted above, there are a few decisio - **An NLB is provisioned as a second load balancer when `flag_create_load_balancer = true`.** It runs continuously once provisioned and incurs additional AWS cost. There is no option to share it with the ALB. - **The NLB uses the same subnets as the ALB (`subnet_ids_alb`).** The NLB needs to be reachable from wherever users connect to Platform, so it belongs in the same network. - **Route53 DNS must be managed within the same AWS account.** If DNS is managed externally, the `connect-ssh.*` A record must be created manually before SSH connections will resolve correctly. + +18. **rootCA.crt is mounted into backend and cron containers whenever `flag_use_private_cacert = true`** + + When a private/self-signed certificate is in use, the rootCA is unconditionally volume-mounted into the `backend` and `cron` containers and imported into their JVM trust stores at startup via the `import-cert.sh` entrypoint wrapper. This applies regardless of whether Wave is enabled. + + Per [Seqera documentation](https://docs.seqera.io/platform-enterprise/enterprise/configuration/ssl_tls#configure-seqera-to-trust-your-private-certificate): + + > "If you secure related infrastructure (such as private Git repositories) with certificates issued by a private Certificate Authority (CA), you may need to configure Seqera to trust those certificates." + + The JVM inside each container maintains its own trust store, independent of the host OS. Even if the host OS trusts the private CA (via `update-ca-trust`), Java processes in the containers will still reject TLS connections to any service using that CA unless the rootCA is explicitly imported into the container JVM's cacerts. + + Mounting the rootCA into backend/cron whenever a private cert is in use is the safe, forward-compatible default — it covers the case where Wave is later enabled, or where other private-CA-secured infrastructure is being called from within the Platform JVM. The marginal cost (a volume mount + a single `keytool` call at container start) is negligible. + + When `flag_use_wave_lite = true`, the rootCA is additionally mounted into the `wave-lite` container for the same reason. diff --git a/documentation/setup/optional_private_certificates.md b/documentation/setup/optional_private_certificates.md index 213bce4..14c6a95 100644 --- a/documentation/setup/optional_private_certificates.md +++ b/documentation/setup/optional_private_certificates.md @@ -133,8 +133,24 @@ When using private certificates with Studios, you must create custom container i - **Custom Image Configuration:** After creating your custom image, you'll need to configure Studios to use it. See the [Seqera Platform Enterprise documentation on custom containers](https://docs.seqera.io/platform-enterprise/25.1/studios/custom-envs#custom-containers) for detailed instructions. - **Multiple Image Types:** You may need to create custom images for different Studios environments (e.g., Jupyter, RStudio, VSCode) if you use multiple types. -### Other Compute Assets -- TODO: Bake cert into Nextflow worker nodes using Wave-Lite. +### Wave Lite: Custom AMI for Nextflow Compute Workers + +When using Wave Lite (`flag_use_wave_lite = true`) with a private certificate, Nextflow compute workers on AWS Batch must pull Wave-augmented container images from your self-hosted Wave Lite server. Because Wave Lite is served over your private certificate, Docker on the compute worker must trust your root CA at the OS level — this cannot be handled by an entrypoint wrapper (which only affects JVM trust stores inside containers). + +The solution is to bake your `rootCA.crt` into the host OS trust store of a custom AMI and configure your Compute Environments to use that AMI. + +**Steps:** + +1. Start from the AWS-managed Amazon Linux 2 AMI (or whichever base AMI you normally use for Batch). +2. Copy `rootCA.crt` onto the instance and add it to the OS trust store: + ```bash + sudo cp rootCA.crt /etc/pki/ca-trust/source/anchors/rootCA.crt + sudo update-ca-trust + ``` +3. Create an AMI from the instance. +4. Set the AMI ID in each Compute Environment that will run Wave Lite workloads. + +**AMI creation method:** Any approach that produces an AMI with the rootCA baked in is valid. [Packer](https://developer.hashicorp.com/packer) is a commonly used option for automating this, but launching an EC2 instance manually, running the above commands, and creating an AMI via the console or CLI works equally well. ## Runtime diff --git a/scripts/installer/validation/check_configuration.py b/scripts/installer/validation/check_configuration.py index 4bb21a5..740993d 100644 --- a/scripts/installer/validation/check_configuration.py +++ b/scripts/installer/validation/check_configuration.py @@ -80,7 +80,9 @@ def verify_only_one_true_set(data: SimpleNamespace): data.flag_do_not_use_https, ] ) - only_one_true_set([data.flag_use_aws_ses_iam_integration, data.flag_use_existing_smtp]) + only_one_true_set( + [data.flag_use_aws_ses_iam_integration, data.flag_use_existing_smtp] + ) def verify_sensitive_keys(data: SimpleNamespace, data_dictionary: dict): @@ -103,7 +105,9 @@ def verify_sensitive_keys(data: SimpleNamespace, data_dictionary: dict): data_keys = data_dictionary.keys() for key in sensitive_keys: if key in data_keys: - log_error_and_exit(f" Do not specify `{key}`. This value will be sourced from SSM.") + log_error_and_exit( + f" Do not specify `{key}`. This value will be sourced from SSM." + ) def verify_tfvars_config_dependencies(data: SimpleNamespace): @@ -153,14 +157,22 @@ def verify_tower_server_url(data: SimpleNamespace): def verify_tower_root_users(data: SimpleNamespace): """Ensure at least one root user is specified.""" if data.tower_root_users in ["REPLACE_ME", ""]: - log_error_and_exit("Please populate `tower_root_user` with at least one email address.") + log_error_and_exit( + "Please populate `tower_root_user` with at least one email address." + ) def verify_tower_self_signed_certs(data: SimpleNamespace): """Check self-signed certificate settings (if necessary).""" if data.flag_use_private_cacert: if not data.private_cacert_bucket_prefix.startswith("s3://"): - log_error_and_exit(" Field `private_cacert_bucket_prefix` must start with `s3://`") + log_error_and_exit( + " Field `private_cacert_bucket_prefix` must start with `s3://`" + ) + logger.warning( + "`flag_use_private_cacert` enabled: " + "rootCA.crt will be automatically imported into backend and cron container JVMs at startup." + ) def verify_tower_groundswell(data: SimpleNamespace): @@ -186,7 +198,9 @@ def verify_email_login_disablement(data: SimpleNamespace): """Check email login disablement scenarios.""" if data.flag_disable_email_login: if data.tower_container_version < "v23.4.5": - log_error_and_exit(" You cannot disable email login in versions earlier than 23.4.5") + log_error_and_exit( + " You cannot disable email login in versions earlier than 23.4.5" + ) oidc_flags = [ data.flag_oidc_use_generic, @@ -194,10 +208,14 @@ def verify_email_login_disablement(data: SimpleNamespace): data.flag_oidc_use_github, ] if not any(oidc_flags): - log_error_and_exit(" Email login cannot be disabled if you dont have an OIDC alternative configured.") + log_error_and_exit( + " Email login cannot be disabled if you dont have an OIDC alternative configured." + ) if data.flag_run_seqerakit: - logger.warning("Seqerakit step cannot execute if email login is not active.") + logger.warning( + "Seqerakit step cannot execute if email login is not active." + ) def verify_subnet_privacy(data: SimpleNamespace): @@ -292,10 +310,14 @@ def verify_ingress_and_egress(data: SimpleNamespace, data_dictionary: dict): """Issue reminders if ingress/egress rules seem overly loose.""" if data.sg_ingress_cidrs == "0.0.0.0/0": - logger.warning("`sg_ingress_cidrs` is completely open (HTTPs) . Consider tightening.") + logger.warning( + "`sg_ingress_cidrs` is completely open (HTTPs) . Consider tightening." + ) if data.sg_ssh_cidrs == "0.0.0.0/0": - logger.warning("`sg_ssh_cidrs` ingress is completly open (SSH). Consider tightening.") + logger.warning( + "`sg_ssh_cidrs` ingress is completly open (SSH). Consider tightening." + ) # Forgoing `data...` approeach beause I'm too dumb to figure out how to get a variable name as a string. egress_sgs = [ @@ -313,7 +335,9 @@ def verify_ingress_and_egress(data: SimpleNamespace, data_dictionary: dict): def verify_flow_logs(data: SimpleNamespace): """Issue reminder about Flow logs cost.""" if (data.flag_create_new_vpc) and (data.enable_vpc_flow_logs): - logger.warning("You have VPC Flow Logs activated. This will generate extra costs.") + logger.warning( + "You have VPC Flow Logs activated. This will generate extra costs." + ) def verify_ami_update_behaviour(data: SimpleNamespace): @@ -335,14 +359,20 @@ def verify_database_configuration(data: SimpleNamespace): if (data.db_engine == "mysql") and ("8" in data.db_engine_version): logger.warning("MySQL 8 may need TOWER_DB_URL connection string modifiers.") - if (data.tower_db_url.startswith("jdbc:")) or (data.tower_db_url.startswith("mysql:")): - log_error_and_exit("Do not include protocol in `tower_db_url`. Start with hostname.") + if (data.tower_db_url.startswith("jdbc:")) or ( + data.tower_db_url.startswith("mysql:") + ): + log_error_and_exit( + "Do not include protocol in `tower_db_url`. Start with hostname." + ) if data.tower_db_driver != "org.mariadb.jdbc.Driver": log_error_and_exit("Field `tower_db_driver` must be `org.mariadb.jdbc.Driver`.") if data.tower_db_dialect != "io.seqera.util.MySQL55DialectCollateBin": - log_error_and_exit("Field `tower_db_dialect` must be `org.mariadb.jdbc.Driver`.") + log_error_and_exit( + "Field `tower_db_dialect` must be `org.mariadb.jdbc.Driver`." + ) if data.flag_use_container_db: if data.tower_db_url != "db:3306": @@ -401,7 +431,9 @@ def verify_docker_version(data: SimpleNamespace): ) if "5.6" in data.db_engine_version: - log_error_and_exit("MySQL 5.6 is obsolete. Please chooses MySQL 5.7 in `db_engine_version`.") + log_error_and_exit( + "MySQL 5.6 is obsolete. Please chooses MySQL 5.7 in `db_engine_version`." + ) def verify_data_studio(data: SimpleNamespace): @@ -413,7 +445,10 @@ def verify_data_studio(data: SimpleNamespace): "`tower_container_version` must bv24.1.0 or higher to set `flag_enable_data_studio` to true." ) - if data.tower_container_version < "v24.3.0" and data.data_studio_container_version >= "0.7.8": + if ( + data.tower_container_version < "v24.3.0" + and data.data_studio_container_version >= "0.7.8" + ): log_error_and_exit( "`data_studio_container_version` cannot be 0.7.8+ when `tower_container_version` is less than v24.3.0." ) @@ -422,10 +457,14 @@ def verify_data_studio(data: SimpleNamespace): # https://www.geeksforgeeks.org/python-check-whether-string-contains-only-numbers-or-not/ # if re.match('[0-9]*$', data.data_studio_eligible_workspaces): if not re.findall(r"[0-9]+,[0-9]+", data.data_studio_eligible_workspaces): - log_error_and_exit("`data_studio_eligible_workspaces may only be populated by digits and commas.") + log_error_and_exit( + "`data_studio_eligible_workspaces may only be populated by digits and commas." + ) if data.flag_use_private_cacert: - logger.warning("Please see documentation to understand how to make private certs work with Studios images.") + logger.warning( + "Please see documentation to understand how to make private certs work with Studios images." + ) # Deferred until better solution comes along to get TF locals # - Add check that CONNECT_PROXY_URL and TOWER_DATA_STUDIO_CONNECT_URL are the same. @@ -488,19 +527,25 @@ def verify_data_studio_ssh(data: SimpleNamespace): def verify_not_v24_1_0(data: SimpleNamespace): """Verify that user has not selected Tower v24.1.0 (due to serialization bug).""" if data.tower_container_version == "v24.1.0": - log_error_and_exit("Tower version 24.1.0 has a fatal serialization flaw. Please use v24.1.3 or higher.") + log_error_and_exit( + "Tower version 24.1.0 has a fatal serialization flaw. Please use v24.1.3 or higher." + ) def verify_not_v24_1_1(data: SimpleNamespace): """Verify that user has not selected Tower v24.1.1 (memory leak).""" if data.tower_container_version == "v24.1.1": - log_error_and_exit("Tower version 24.1.1 has Micronaut framework flaw. Please use v24.1.3 or higher.") + log_error_and_exit( + "Tower version 24.1.1 has Micronaut framework flaw. Please use v24.1.3 or higher." + ) def verify_not_v24_1_2(data: SimpleNamespace): """Verify that user has not selected Tower v24.1.2 (Redis TLS issue).""" if data.tower_container_version == "v24.1.2": - log_error_and_exit("Tower version 24.1.2 has a TLS flaw. Please use v24.1.3 or higher.") + log_error_and_exit( + "Tower version 24.1.2 has a TLS flaw. Please use v24.1.3 or higher." + ) def verify_if_v24_1_3(data: SimpleNamespace): @@ -520,9 +565,13 @@ def verify_if_v24_1_4(data: SimpleNamespace): data.flag_oidc_use_github, ] if any(oidc_flags): - logger.warning("Tower version 24.1.4 cannot send emails. You will only be able to use your OIDC option.") + logger.warning( + "Tower version 24.1.4 cannot send emails. You will only be able to use your OIDC option." + ) else: - log_error_and_exit("Tower version 24.1.4 cannot send emails. Please use v24.1.5 or higher.") + log_error_and_exit( + "Tower version 24.1.4 cannot send emails. Please use v24.1.5 or higher." + ) def verify_connect_version_tls(data: SimpleNamespace): @@ -535,7 +584,10 @@ def verify_connect_version_tls(data: SimpleNamespace): def verify_alb_settings(data: SimpleNamespace): """Verify that user does not have contradictory settings in case of ALB vs. no ALB.""" - if data.flag_use_private_cacert and data.flag_make_instance_private_behind_public_alb: + if ( + data.flag_use_private_cacert + and data.flag_make_instance_private_behind_public_alb + ): log_error_and_exit( "Use of private cert on EC2 cannot work with `flag_make_instance_private_behind_alb = true`. Please set only one of the options to true." ) @@ -546,7 +598,9 @@ def verify_redis_version(data: SimpleNamespace): if data.tower_container_version >= "v24.2.0": if data.flag_use_container_redis: - logger.warning("Seqera Platform version >= 24.2.0 uses a Redis v7.0 container (previously Redis v6.0).") + logger.warning( + "Seqera Platform version >= 24.2.0 uses a Redis v7.0 container (previously Redis v6.0)." + ) if data.flag_create_external_redis: # TO DO @@ -554,19 +608,28 @@ def verify_redis_version(data: SimpleNamespace): # This uses a redis 7.0 as baseline and thus is compliant to the needs of 24.2.2. # In future this may need to change if Tower demands require a version > Redis 7.0 OR we # choose to make the external redis values configurable. - logger.warning("The external Elasticache instance is hardcoded to use Redis 7.0.") + logger.warning( + "The external Elasticache instance is hardcoded to use Redis 7.0." + ) else: - logger.warning("When you upgrade to Seqera Platform >= v24.2, a Redis version >= 6.2 will be required.") + logger.warning( + "When you upgrade to Seqera Platform >= v24.2, a Redis version >= 6.2 will be required." + ) def verify_wave(data: SimpleNamespace): if (data.flag_use_wave == True) and (data.flag_use_wave_lite == True): - log_error_and_exit("`flag_use_wave` and `flag_use_wave_lite` cannot both be set to true.") + log_error_and_exit( + "`flag_use_wave` and `flag_use_wave_lite` cannot both be set to true." + ) if data.flag_use_wave_lite == True: if data.flag_use_private_cacert: - logger.warning("Please see documentation to understand how to make private certs work with Wave-Lite.") + logger.warning( + "`flag_use_wave_lite` with `flag_use_private_cacert`: a custom AMI is required for compute workers. " + "See documentation/setup/optional_private_certificates.md for details." + ) def verify_ssh_access(data: SimpleNamespace): @@ -586,13 +649,16 @@ def verify_ssh_access(data: SimpleNamespace): def verify_production_deployment(data: SimpleNamespace): - if (data.flag_create_external_db == False) or (data.flag_create_external_redis == False): + if (data.flag_create_external_db == False) or ( + data.flag_create_external_redis == False + ): logger.warning( "WARNING: You are running Seqera Platform without a managed DB/Redis. This does not align to Seqera-recommended Production deployment best practices and can result in system instability." ) if (data.flag_use_wave_lite == True) and ( - (data.flag_create_external_db == False) or (data.flag_create_external_redis == False) + (data.flag_create_external_db == False) + or (data.flag_create_external_redis == False) ): logger.warning( "WARNING: You are running Wave Lite without a managed DB/Redis. This does not align to Seqera-recommended Production deployment best practices and can result in system instability." @@ -630,7 +696,9 @@ def verify_pipeline_versioning(data: SimpleNamespace): """Conduct checks if pipeline versioning is active.""" if data.tower_enable_pipeline_versioning: if data.tower_container_version < "v25.3.0": - logger.warning("Your Platform version is too old to support pipeline versioning. Must be >= v25.3.0.") + logger.warning( + "Your Platform version is too old to support pipeline versioning. Must be >= v25.3.0." + ) # All workspaces eligible. Return. if data.pipeline_versioning_eligible_workspaces == "": @@ -661,8 +729,12 @@ def verify_pipeline_versioning(data: SimpleNamespace): data = SimpleNamespace(**data_dictionary) # Check minimum container version - if not ((data.tower_container_version).startswith("v")) or (data.tower_container_version < "v23.1.0"): - log_error_and_exit("Tower version minimum is 23.1.0 (for Parameter Store integration).") + if not ((data.tower_container_version).startswith("v")) or ( + data.tower_container_version < "v23.1.0" + ): + log_error_and_exit( + "Tower version minimum is 23.1.0 (for Parameter Store integration)." + ) # Check known problem Tower versions print("\n")