MRG: Merge pull request #598 from octue/improve-monitor-message-tests

Remove dataflow deployment CLI command

Author: cortadocodes

.devcontainer/devcontainer.json

@@ -59,7 +59,7 @@
 
   // Use 'postCreateCommand' to run commands after the container is created
   // TODO update this to install all extras once https://github.com/python-poetry/poetry/issues/3413 is solved
-  "postCreateCommand": "poetry install --extras hdf5 --extras dataflow",
+  "postCreateCommand": "poetry install --extras hdf5",
 
   // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
   "remoteUser": "vscode",

.github/workflows/release.yml

@@ -71,8 +71,6 @@ jobs:
     runs-on: ubuntu-latest
     needs: run-tests
     steps:
-      - uses: actions/checkout@v3
-
       - name: Create Release
         uses: actions/create-release@v1
         env:

.github/workflows/update-pull-request.yml

@@ -1,30 +1,26 @@
 # This workflow updates the pull request description with an auto-generated section containing the categorised commit
-# message headers of the commits since the last pull request merged into main. The auto generated section is enveloped
-# between two comments: "<!--- START AUTOGENERATED NOTES --->" and "<!--- END AUTOGENERATED NOTES --->". Anything
-# outside these in the description is left untouched. Auto-generated updates can be skipped for a commit if
+# message headers of the pull request's commits. The auto generated section is enveloped between two comments:
+# "<!--- START AUTOGENERATED NOTES --->" and "<!--- END AUTOGENERATED NOTES --->". Anything outside these in the
+# description is left untouched. Auto-generated updates can be skipped for a commit if
 # "<!--- SKIP AUTOGENERATED NOTES --->" is added to the pull request description.
 
 name: update-pull-request
 
-# Only trigger for pull requests into main branch.
-on:
-  pull_request:
-    branches:
-      - main
+on: pull_request
 
 jobs:
   description:
     if: "!contains(github.event.pull_request.body, '<!--- SKIP AUTOGENERATED NOTES --->')"
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - uses: octue/generate-pull-request-description@1.0.0.beta-2
-      id: pr-description
-      with:
-        pull_request_url: ${{ github.event.pull_request.url }}
-        api_token: ${{ secrets.GITHUB_TOKEN }}
-    - name: Update pull request body
-      uses: riskledger/update-pr-description@v2
-      with:
-        body: ${{ steps.pr-description.outputs.pull_request_description }}
-        token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: octue/[email protected]
+        id: pr-description
+        with:
+          pull_request_url: ${{ github.event.pull_request.url }}
+          api_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Update pull request body
+        uses: riskledger/update-pr-description@v2
+        with:
+          body: ${{ steps.pr-description.outputs.pull_request_description }}
+          token: ${{ secrets.GITHUB_TOKEN }}

Dockerfile

@@ -10,9 +10,8 @@ RUN poetry config virtualenvs.create false
 # Install python dependencies. Note that poetry installs any root packages by default, but this is not available at this
 # stage of caching dependencies. So we do a dependency-only install here to cache the dependencies, then a full poetry
 # install post-create to install the root package, which will change more rapidly than dependencies.
-COPY octue/cloud/deployment/google/dataflow/setup.py setup.py
 COPY pyproject.toml poetry.lock ./
-RUN poetry install --no-ansi --no-interaction -E dataflow --only=main --no-root -v
+RUN poetry install --no-ansi --no-interaction --only=main --no-root -v
 
 COPY . .
-RUN poetry install --no-ansi --no-interaction -E dataflow --only=main -v
+RUN poetry install --no-ansi --no-interaction --only=main -v

docs/source/creating_services.rst

@@ -5,7 +5,7 @@ One of the main features of the Octue SDK is to allow you to easily create servi
 return answers. They can run locally on any machine or be deployed to the cloud. Currently:
 
 - The backend communication between twins uses Google Pub/Sub whether they're local or deployed
-- The deployment options are Google Cloud Run or Google Dataflow
+- Services are deployed to Google Cloud Run
 - The language of the entrypoint must by ``python3`` (you can call processes using other languages within this though)
 
 
@@ -96,7 +96,6 @@ Dockerfile (optional)
         provides a default ``Dockerfile`` for these purposes that will work for most cases:
 
         - For deploying to `Google Cloud Run <https://github.com/octue/octue-sdk-python/blob/main/octue/cloud/deployment/google/cloud_run/Dockerfile>`_
-        - For deploying to `Google Dataflow <https://github.com/octue/octue-sdk-python/blob/main/octue/cloud/deployment/google/dataflow/Dockerfile>`_
 
         However, you may need to write and provide your own ``Dockerfile`` if your app requires:
 

docs/source/index.rst

@@ -40,7 +40,7 @@ Key features
 **Create, run, and deploy your apps as services**
 
 - No need to change your app - just wrap it
-- Use the ``octue`` CLI to run your service locally or deploy it to Google Cloud Run or Google Dataflow
+- Use the ``octue`` CLI to run your service locally or deploy it to Google Cloud Run
 - Create JSON-schema interfaces to explicitly define the form of configuration, input, and output data
 - Ask other services questions as part of your app (i.e. build trees of services)
 - Automatically display readable, colourised logs, or use your own log handler

docs/source/inter_service_compatibility.rst

@@ -349,53 +349,7 @@ def get_crash_diagnostics(cloud_path, local_path, download_datasets):
 
 @octue_cli.group()
 def deploy():
-    """Deploy a python app to the cloud as an Octue service or digital twin."""
-
-
-@deploy.command()
-@click.option(
-    "-c",
-    "--service-config",
-    type=click.Path(exists=True, dir_okay=False),
-    default="octue.yaml",
-    show_default=True,
-    help="The path to an `octue.yaml` file defining the service to deploy.",
-)
-@click.option("--no-cache", is_flag=True, help="If provided, don't use the Docker cache when building the image.")
-@click.option("--update", is_flag=True, help="If provided, allow updates to an existing service.")
-@click.option(
-    "--dataflow-job-only",
-    is_flag=True,
-    help="If provided, skip creating and running the build trigger and just deploy a pre-built image to Dataflow",
-)
-@click.option("--image-uri", type=str, default=None, help="The actual image URI to use when creating the Dataflow job.")
-@click.option(
-    "--revision-tag",
-    type=str,
-    default=None,
-    help="A tag to use for this revision of the service (e.g. 1.3.7). This overrides the `OCTUE_SERVICE_REVISION_TAG` "
-    "environment variable if it's present. If this option isn't given and the environment variable isn't present, a "
-    "random 'cool name' tag is generated e.g 'curious-capybara'.",
-)
-def dataflow(service_config, no_cache, update, dataflow_job_only, image_uri, revision_tag):
-    """Deploy a python app to Google Dataflow as an Octue service or digital twin."""
-    if bool(importlib.util.find_spec("apache_beam")):
-        # Import the Dataflow deployer only if the `apache-beam` package is available (due to installing `octue` with
-        # the `dataflow` extras option).
-        from octue.cloud.deployment.google.dataflow.deployer import DataflowDeployer
-    else:
-        raise ImportWarning(
-            "To use this CLI command, you must install `octue` with the `dataflow` option e.g. "
-            "`pip install octue[dataflow]`"
-        )
-
-    deployer = DataflowDeployer(service_config, revision_tag=revision_tag)
-
-    if dataflow_job_only:
-        deployer.create_streaming_dataflow_job(image_uri=image_uri, update=update)
-        return
-
-    deployer.deploy(no_cache=no_cache, update=update)
+    """A collection of commands to aid deploying a python app to the cloud as an Octue service or digital twin."""
 
 
 @deploy.command()