Skip S3 deployment on branch builds by default

Uploading ETL outputs to S3 incurs egress fees that, for a full PUDL
build, cost more than running the entire ETL. Branch builds currently
deploy to both GCS and S3 purely as a test, then delete the staged data
again. The nightly build exercises the real S3 deployment every night, so
re-testing it on every branch build has little marginal value, while the
GCS deployment is free and covers nearly all the same code paths.

Branch builds now deploy to GCS but not S3 by default. The build-pudl and
deploy-pudl workflow_dispatch forms expose deploy_to_gcs / deploy_to_s3
checkboxes to override this per run, and when neither target is enabled
build-pudl skips triggering deploy-pudl entirely (e.g. a build run only
to regenerate row counts). Nightly and stable deployments are unchanged
and still deploy to both.

- DeploymentPlan gains tri-state deploy_to_gcs / deploy_to_s3 overrides
  and symmetric upload_to_gcs / upload_to_s3 properties: GCS defaults on
  for every deploy type, S3 defaults on for nightly/stable and off for
  branch builds. A plan that would deploy nowhere is rejected.
- upload_outputs() only builds the fs clients and upload targets for the
  enabled destinations; _assert_permanent_paths_are_empty tolerates a
  missing filesystem.
- pudl_deploy grows --deploy-gcs/--no-deploy-gcs and
  --deploy-s3/--no-deploy-s3 flags, threaded through resolve_build.
- deploy-pudl.yml and build-pudl.yml expose matching workflow_dispatch
  inputs; build-pudl passes the resolved values to the batch job, and
  pudl_batch.sh forwards them to deploy-pudl and gates both
  trigger_deployment call sites.
- Add unit tests and a release notes entry.

Closes #5557

Author: zaneselvans

.github/workflows/build-pudl.yml

@@ -2,6 +2,17 @@
 name: build-pudl
 on:
   workflow_dispatch:
+    inputs:
+      deploy_to_gcs:
+        type: boolean
+        description: "Deploy build outputs to GCS? (branch builds only)"
+        required: false
+        default: true
+      deploy_to_s3:
+        type: boolean
+        description: "Deploy build outputs to S3? (branch builds only; S3 egress fees are large)"
+        required: false
+        default: false
   push:
     tags:
       # Run a build when a stable tag is pushed. If a build has already been run
@@ -19,6 +30,10 @@ env:
   SKIP_BUILD: "false"
   GIT_TAG: ""
   BUILD_ID: ""
+  # Nightly and stable builds always deploy to both targets; branch builds
+  # (workflow_dispatch) override these from the workflow inputs below.
+  DEPLOY_TO_GCS: "true"
+  DEPLOY_TO_S3: "true"
 
 jobs:
   build_and_deploy_pudl:
@@ -56,9 +71,13 @@ jobs:
               echo "DEPLOYMENT_ENVIRONMENT=production" >> "$GITHUB_ENV"
               BUILD_ID_RAW="nightly-$BUILD_ID_SUFFIX"
           elif [[ ${{ github.event_name }} == "workflow_dispatch" ]]; then
-              echo "DEPLOYMENT_ENVIRONMENT=staging" >> "$GITHUB_ENV"
               BUILD_ID_RAW="branch-$BUILD_ID_SUFFIX"
-              echo "GIT_TAG=$BUILD_ID_RAW" >> "$GITHUB_ENV"
+              {
+                echo "DEPLOYMENT_ENVIRONMENT=staging"
+                echo "DEPLOY_TO_GCS=${{ inputs.deploy_to_gcs }}"
+                echo "DEPLOY_TO_S3=${{ inputs.deploy_to_s3 }}"
+                echo "GIT_TAG=$BUILD_ID_RAW"
+              } >> "$GITHUB_ENV"
           elif [[ ${{ github.event_name }} == "push" ]]; then
               echo "GIT_TAG=${{ github.ref_name }}" >>  "$GITHUB_ENV"
               echo "DEPLOYMENT_ENVIRONMENT=production" >> "$GITHUB_ENV"
@@ -73,6 +92,8 @@ jobs:
           echo "GIT_TAG: $GIT_TAG"
           echo "BUILD_ID: $BUILD_ID"
           echo "DEPLOYMENT_ENVIRONMENT: $DEPLOYMENT_ENVIRONMENT"
+          echo "DEPLOY_TO_GCS: $DEPLOY_TO_GCS"
+          echo "DEPLOY_TO_S3: $DEPLOY_TO_S3"
 
       - name: Tag build
         if: ${{ (env.SKIP_BUILD != 'true') && (github.event_name != 'push') }}
@@ -172,6 +193,8 @@ jobs:
             --container-env BUILD_ID=${{ env.BUILD_ID }} \
             --container-env BUILD_REF=${{ github.ref_name }} \
             --container-env DEPLOYMENT_ENVIRONMENT=${{ env.DEPLOYMENT_ENVIRONMENT }} \
+            --container-env DEPLOY_TO_GCS=${{ env.DEPLOY_TO_GCS }} \
+            --container-env DEPLOY_TO_S3=${{ env.DEPLOY_TO_S3 }} \
             --container-env GCP_BILLING_PROJECT=${{ secrets.GCP_BILLING_PROJECT }} \
             --container-env GITHUB_ACTION_TRIGGER=${{ github.event_name }} \
             --container-env GIT_TAG=${{ env.GIT_TAG }} \

.github/workflows/deploy-pudl.yml

@@ -15,6 +15,16 @@ on:
         options:
           - staging
           - production
+      deploy_to_gcs:
+        type: boolean
+        description: "Upload outputs to GCS?"
+        required: false
+        default: true
+      deploy_to_s3:
+        type: boolean
+        description: "Upload outputs to S3? (S3 egress fees are large)"
+        required: false
+        default: true
 
 jobs:
   deploy_pudl:
@@ -170,6 +180,8 @@ jobs:
             --container-arg="${{ env.GIT_TAG }}" \
             --container-arg="--environment" \
             --container-arg="${{ env.DEPLOYMENT_ENVIRONMENT }}" \
+            --container-arg="${{ inputs.deploy_to_gcs && '--deploy-gcs' || '--no-deploy-gcs' }}" \
+            --container-arg="${{ inputs.deploy_to_s3 && '--deploy-s3' || '--no-deploy-s3' }}" \
             --container-env GITHUB_TOKEN=${{ secrets.PUDL_BOT_PAT }} \
             --container-env AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
             --container-env AWS_DEFAULT_REGION=${{ secrets.AWS_DEFAULT_REGION }} \

builds/pudl_batch.sh

@@ -62,10 +62,19 @@ function trigger_deployment() {
             --repo catalyst-cooperative/pudl \
             --ref "${BUILD_REF}" \
             -f "git_tag=${GIT_TAG}" \
-            -f "deployment_environment=${DEPLOYMENT_ENVIRONMENT}" &&
+            -f "deployment_environment=${DEPLOYMENT_ENVIRONMENT}" \
+            -f "deploy_to_gcs=${DEPLOY_TO_GCS}" \
+            -f "deploy_to_s3=${DEPLOY_TO_S3}" &&
         set -x
 }
 
+function any_deployment_target_enabled() {
+    # If neither cloud storage target is enabled there's nothing for deploy-pudl
+    # to do (branch builds don't update git branches, redeploy the viewer, or
+    # trigger Zenodo), so we skip triggering it entirely.
+    [[ "${DEPLOY_TO_GCS}" == "true" || "${DEPLOY_TO_S3}" == "true" ]]
+}
+
 function stage_emoji() {
     local stage_status=$1
     if [[ $stage_status == "$STAGE_SKIPPED" ]]; then
@@ -259,6 +268,10 @@ TRIGGER_DEPLOYMENT_DURATION=""
 
 # Set these variables *only* if they are not already set by the container or workflow:
 : "${PUDL_GCS_OUTPUT:=gs://builds.catalyst.coop/$BUILD_ID}"
+# Nightly/stable builds deploy to both cloud storage targets; branch builds set
+# these explicitly via the build-pudl workflow inputs.
+: "${DEPLOY_TO_GCS:=true}"
+: "${DEPLOY_TO_S3:=true}"
 # Keep the nightly Dagster config path repo-relative so the same pixi task commands
 # work both locally and inside the nightly build container.
 : "${DG_NIGHTLY_CONFIG:=src/pudl/package_data/settings/dg_nightly.yml}"
@@ -270,12 +283,16 @@ trap cleanup_on_exit EXIT
 
 # Check if there are any existing builds associated with the current commit
 if pixi run pudl_check_for_build "$GIT_TAG"; then
-    run_stage TRIGGER_DEPLOYMENT_STATUS TRIGGER_DEPLOYMENT_DURATION trigger_deployment
-    if any_stage_failed "$TRIGGER_DEPLOYMENT_STATUS"; then
-        echo "Found successful build, but failed to trigger deployment"
-        exit 1
+    if any_deployment_target_enabled; then
+        run_stage TRIGGER_DEPLOYMENT_STATUS TRIGGER_DEPLOYMENT_DURATION trigger_deployment
+        if any_stage_failed "$TRIGGER_DEPLOYMENT_STATUS"; then
+            echo "Found successful build, but failed to trigger deployment"
+            exit 1
+        fi
+        echo "Found a successful build and triggered a deployment"
+    else
+        echo "Found a successful build; skipping deployment (no GCS or S3 target enabled)"
     fi
-    echo "Found a successful build and triggered a deployment"
     exit 0
 fi
 
@@ -316,7 +333,11 @@ require_stage_success "$DATA_VALIDATION_STATUS"
 require_stage_success "$ROW_COUNT_VALIDATION_STATUS"
 require_stage_success "$SAVE_OUTPUTS_STATUS"
 
-run_stage TRIGGER_DEPLOYMENT_STATUS TRIGGER_DEPLOYMENT_DURATION trigger_deployment
+if any_deployment_target_enabled; then
+    run_stage TRIGGER_DEPLOYMENT_STATUS TRIGGER_DEPLOYMENT_DURATION trigger_deployment
+else
+    echo "Skipping deployment trigger: neither GCS nor S3 deployment is enabled."
+fi
 
 # Notify Zulip about entire pipeline's success or failure;
 if any_stage_failed \

docs/release_notes.rst

@@ -156,6 +156,15 @@ Developer Experience
   so a shared Cloud Monitoring dashboard can filter resource-usage metrics by
   pipeline. VM sizes and the ETL's process and thread parallelism were tuned to
   match measured resource usage and stop oversubscribing the CPUs. See :pr:`5545`.
+* Branch builds (``build-pudl`` runs triggered via ``workflow_dispatch``) now skip
+  the S3 deployment by default and only deploy to GCS. S3 egress fees cost more than
+  a full ETL run, and the nightly build already exercises the real S3 deployment
+  every night. The ``build-pudl`` and ``deploy-pudl`` workflow-dispatch forms expose
+  ``deploy_to_gcs`` / ``deploy_to_s3`` checkboxes to override this per run, and when
+  neither target is enabled ``build-pudl`` skips triggering ``deploy-pudl``
+  altogether (e.g. a build run only to regenerate row counts). Nightly and stable
+  deployments are unchanged and still deploy to both. See issue:`5557` and PR
+  :pr:`5558`.
 * Fixed several issues with how ``dbt_helper update-tables`` renders ``schema.yml``
   (:mod:`pudl.dbt_schema`): long ``description:`` fields are now wrapped into readable
   paragraph blocks and strings that need quoting prefer double quotes. This now matches

src/pudl/deploy/pudl.py

@@ -64,6 +64,11 @@ class DeploymentPlan(BaseModel):
 
     git_tag: str
     environment: Literal["staging", "production"]
+    # Tri-state overrides for the cloud storage upload targets. ``None`` means "use
+    # the default for this deploy type" (see ``upload_to_gcs``/``upload_to_s3``); an
+    # explicit bool forces the target on or off regardless of deploy type.
+    deploy_to_gcs: bool | None = None
+    deploy_to_s3: bool | None = None
 
     @property
     def deploy_type(self) -> DeploymentType:
@@ -79,6 +84,39 @@ def _validate_branch_only_targets_staging(self) -> "DeploymentPlan":
             )
         return self
 
+    @model_validator(mode="after")
+    def _validate_has_an_upload_target(self) -> "DeploymentPlan":
+        if not self.upload_to_gcs and not self.upload_to_s3:
+            raise ValueError(
+                f"Deployment for git_tag={self.git_tag!r} has neither GCS nor S3 "
+                "uploads enabled -- there would be nothing to deploy. A build that "
+                "shouldn't deploy anywhere simply shouldn't trigger deploy-pudl."
+            )
+        return self
+
+    @property
+    def upload_to_gcs(self) -> bool:
+        """Whether this deployment uploads outputs to GCS.
+
+        Defaults to ``True`` for every deploy type -- GCS has no egress fees and is
+        the primary distribution target -- but a build can still explicitly opt out
+        (e.g. an S3-only test).
+        """
+        return self.deploy_to_gcs if self.deploy_to_gcs is not None else True
+
+    @property
+    def upload_to_s3(self) -> bool:
+        """Whether this deployment uploads outputs to S3.
+
+        Nightly and stable deploys default to ``True``. Branch builds default to
+        ``False``: S3 egress costs more than a full ETL run, and the nightly build
+        exercises the real S3 deployment every night anyway. A branch build can
+        still opt in explicitly when that's the thing being tested.
+        """
+        if self.deploy_to_s3 is not None:
+            return self.deploy_to_s3
+        return self.deploy_type != DeploymentType.BRANCH
+
     @property
     def path_suffixes(self) -> list[str]:
         """Cloud storage path suffixes this deployment uploads to.
@@ -317,8 +355,8 @@ def _upload_to_path(fs, path: str, source_dir: Path, clear_first: bool) -> None:
 
 
 def _assert_permanent_paths_are_empty(
-    gcs_fs: gcsfs.GCSFileSystem,
-    s3_fs: s3fs.S3FileSystem,
+    gcs_fs: gcsfs.GCSFileSystem | None,
+    s3_fs: s3fs.S3FileSystem | None,
     path_suffixes: list[str],
     immutable_suffixes: frozenset[str],
 ) -> None:
@@ -339,6 +377,8 @@ def _assert_permanent_paths_are_empty(
             (gcs_fs, f"gs://pudl.catalyst.coop/{suffix}/"),
             (s3_fs, f"s3://pudl.catalyst.coop/{suffix}/"),
         ):
+            if fs is None:
+                continue
             if fs.exists(path):
                 raise RuntimeError(
                     f"Refusing to deploy to {path}: it's a permanent, "
@@ -352,15 +392,18 @@ def upload_outputs(
     source_dir: Path,
     path_suffixes: list[str],
     immutable_suffixes: frozenset[str] = frozenset(),
+    upload_to_gcs: bool = True,
+    upload_to_s3: bool = True,
 ) -> None:
     """Upload outputs to cloud storage paths.
 
-    Uploads all files from source directory to GCS and S3 using the provided path
-    suffixes. Each suffix is uploaded to both gs://pudl.catalyst.coop/{suffix}/ and
-    s3://pudl.catalyst.coop/{suffix}/. Any existing objects at a suffix are removed
-    first, unless that suffix is listed in ``immutable_suffixes`` -- a permanent,
-    hold-protected versioned release path is never cleared, and instead must not
-    exist at all yet (see ``_assert_permanent_paths_are_empty``).
+    Uploads all files from source directory to GCS and/or S3 using the provided path
+    suffixes. Each enabled destination gets every suffix uploaded to
+    gs://pudl.catalyst.coop/{suffix}/ and/or s3://pudl.catalyst.coop/{suffix}/. Any
+    existing objects at a suffix are removed first, unless that suffix is listed in
+    ``immutable_suffixes`` -- a permanent, hold-protected versioned release path is
+    never cleared, and instead must not exist at all yet (see
+    ``_assert_permanent_paths_are_empty``).
 
     Each (suffix, destination) pair is uploaded concurrently: GCS and S3 are separate
     network destinations, and this is I/O-bound work that releases the GIL.
@@ -371,32 +414,41 @@ def upload_outputs(
         immutable_suffixes: Path suffixes that should never be cleared before upload
             (e.g. a permanent stable-version path like "v2026.7.0"). It's an error
             for one of these paths to already exist.
+        upload_to_gcs: Whether to upload to GCS.
+        upload_to_s3: Whether to upload to S3. Branch builds skip S3 by default
+            because its egress fees are large and the nightly build tests it anyway.
 
     Raises:
+        ValueError: If neither ``upload_to_gcs`` nor ``upload_to_s3`` is enabled.
         RuntimeError: If a permanent, immutable path already has content.
     """
     logger.info("Uploading outputs to cloud storage")
 
+    if not upload_to_gcs and not upload_to_s3:
+        raise ValueError(
+            "upload_outputs called with neither GCS nor S3 uploads enabled."
+        )
     if not source_dir.exists():
         raise ValueError(f"Source directory does not exist: {source_dir}")
     if not any(source_dir.iterdir()):
         raise ValueError(f"Source directory is empty: {source_dir}")
 
     # NOTE (2026-02-11): our GCS distribution bucket is requester pays.
-    gcs_fs = gcsfs.GCSFileSystem(requester_pays=True)
-    s3_fs = s3fs.S3FileSystem()
+    gcs_fs = gcsfs.GCSFileSystem(requester_pays=True) if upload_to_gcs else None
+    s3_fs = s3fs.S3FileSystem() if upload_to_s3 else None
 
     _assert_permanent_paths_are_empty(gcs_fs, s3_fs, path_suffixes, immutable_suffixes)
 
+    destinations = [
+        (fs, scheme) for fs, scheme in ((gcs_fs, "gs"), (s3_fs, "s3")) if fs is not None
+    ]
     upload_targets = []
     for suffix in path_suffixes:
         clear_first = suffix not in immutable_suffixes
-        upload_targets.append(
-            (gcs_fs, f"gs://pudl.catalyst.coop/{suffix}/", clear_first)
-        )
-        upload_targets.append(
-            (s3_fs, f"s3://pudl.catalyst.coop/{suffix}/", clear_first)
-        )
+        for fs, scheme in destinations:
+            upload_targets.append(
+                (fs, f"{scheme}://pudl.catalyst.coop/{suffix}/", clear_first)
+            )
 
     with ThreadPoolExecutor(max_workers=len(upload_targets)) as executor:
         futures = [
@@ -704,15 +756,23 @@ class ResolvedBuild:
 
 
 def resolve_build(
-    git_tag: str, environment: Literal["staging", "production"]
+    git_tag: str,
+    environment: Literal["staging", "production"],
+    deploy_to_gcs: bool | None = None,
+    deploy_to_s3: bool | None = None,
 ) -> ResolvedBuild:
     """Resolve the deployment plan, locate the build, and set up local logging.
 
     Raises if ``git_tag`` doesn't look like a nightly/stable/branch tag, if a
-    branch tag is being deployed to production, or if no successful build exists
-    for the tag yet.
+    branch tag is being deployed to production, if no successful build exists
+    for the tag yet, or if both cloud storage upload targets are disabled.
     """
-    plan = DeploymentPlan(git_tag=git_tag, environment=environment)
+    plan = DeploymentPlan(
+        git_tag=git_tag,
+        environment=environment,
+        deploy_to_gcs=deploy_to_gcs,
+        deploy_to_s3=deploy_to_s3,
+    )
 
     build_path = get_build_from_tag(git_tag)
     build_id = build_path.name

src/pudl/scripts/pudl_deploy.py

@@ -84,6 +84,8 @@ def _deploy_outputs(
         source_dir=source_dir,
         path_suffixes=plan.path_suffixes,
         immutable_suffixes=plan.immutable_suffixes,
+        upload_to_gcs=plan.upload_to_gcs,
+        upload_to_s3=plan.upload_to_s3,
     )
 
     if plan.redeploy_eel_hole:
@@ -150,11 +152,30 @@ def _deploy_outputs(
     ),
     show_default=True,
 )
+@click.option(
+    "--deploy-gcs/--no-deploy-gcs",
+    "deploy_to_gcs",
+    default=None,
+    help=(
+        "Force GCS upload on or off. If unset, defaults to on for every deploy type."
+    ),
+)
+@click.option(
+    "--deploy-s3/--no-deploy-s3",
+    "deploy_to_s3",
+    default=None,
+    help=(
+        "Force S3 upload on or off. If unset, defaults to on for nightly/stable "
+        "deploys and off for branch builds (S3 egress fees are large)."
+    ),
+)
 @click.pass_context
 def main(
     ctx: click.Context,
     git_tag: str,
     environment: Literal["staging", "production"],
+    deploy_to_gcs: bool | None,
+    deploy_to_s3: bool | None,
 ) -> None:
     """Deploy PUDL ETL outputs to cloud storage and external services.
 
@@ -197,6 +218,8 @@ def main(
             stage_results=stage_results,
             git_tag=git_tag,
             environment=environment,
+            deploy_to_gcs=deploy_to_gcs,
+            deploy_to_s3=deploy_to_s3,
         )
         # run_stage's default fail_hard=True re-raises on failure instead of
         # returning, so reaching this line means resolve_build succeeded.