MRG: Merge pull request #14 from octue/test/test-new-log-handling

Deploy using GitHub actions

Author: cortadocodes

.github/workflows/cd.yaml

@@ -0,0 +1,136 @@
+name: cd
+
+on:
+  push:
+    branches: main
+
+  workflow_dispatch:
+    inputs:
+      debug_enabled:
+        description: "Enable tmate debug"
+        type: boolean
+        default: "false"
+
+jobs:
+  info:
+    runs-on: ubuntu-latest
+
+    timeout-minutes: 5
+
+    permissions:
+      contents: read
+
+    outputs:
+      branch_tag_kebab: ${{ steps.get-deployment-info.outputs.branch_tag_kebab }}
+      branch_tag_screaming: ${{ steps.get-deployment-info.outputs.branch_tag_screaming}}
+      image_latest_artefact: ${{ steps.get-deployment-info.outputs.image_latest_artefact}}
+      image_latest_tag: ${{ steps.get-deployment-info.outputs.image_latest_tag }}
+      image_version_artefact: ${{ steps.get-deployment-info.outputs.image_version_artefact}}
+      image_version_tag: ${{ steps.get-deployment-info.outputs.image_version_tag }}
+      short_sha: ${{ steps.get-deployment-info.outputs.short_sha }}
+      gcp_project_name: ${{ steps.get-deployment-info.outputs.gcp_project_name}}
+      gcp_project_number: ${{ steps.get-deployment-info.outputs.gcp_project_number}}
+      gcp_region: ${{ steps.get-deployment-info.outputs.gcp_region}}
+      gcp_resource_affix: ${{ steps.get-deployment-info.outputs.gcp_resource_affix}}
+      gcp_service_name: ${{ steps.get-deployment-info.outputs.gcp_service_name}}
+      version: ${{ steps.get-deployment-info.outputs.version }}
+      version_slug: ${{ steps.get-deployment-info.outputs.version_slug }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install poetry
+        uses: snok/install-poetry@v1
+
+      - name: Get deployment info
+        id: get-deployment-info
+        uses: octue/[email protected]
+        with:
+          gcp_project_name: octue-sdk-python
+          gcp_project_number: 437801218871
+          gcp_region: europe-west1
+          gcp_resource_affix: octue
+          gcp_service_name: example-service-cloud-run
+          gcp_environment: main
+
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    needs: info
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Authenticate with GCP Workload Identity
+        id: auth
+        uses: google-github-actions/auth@v0
+        with:
+          # NOTE: If setting create_credentials_file=true, .dockerignore file must include `gha-creds-*.json` to avoid baking these credentials into build
+          create_credentials_file: true
+          workload_identity_provider: projects/${{ needs.info.outputs.gcp_project_number }}/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider
+          service_account: github-actions@${{ needs.info.outputs.gcp_project_name }}.iam.gserviceaccount.com
+
+      - name: Setup gcloud
+        uses: "google-github-actions/setup-gcloud@v0"
+
+      - name: Configure Docker for GCP
+        run: gcloud auth configure-docker ${{ needs.info.outputs.gcp_region }}-docker.pkg.dev
+
+      - name: Setup tmate session [DEBUG]
+        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true'}}
+        uses: mxschmitt/action-tmate@v3
+
+      - name: Get default Octue Cloud Run Dockerfile
+        run: wget https://raw.githubusercontent.com/octue/octue-sdk-python/main/octue/cloud/deployment/google/cloud_run/Dockerfile
+
+      - name: Build and push container with latest and version tags
+        # Note: We don't push containers with shas because we'd end up with terabytes in storage (an image for every commit)
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          platforms: linux/amd64
+          file: ./Dockerfile
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          tags: |
+            ${{ needs.info.outputs.image_version_artefact}}
+            ${{ needs.info.outputs.image_latest_artefact}}
+
+      - name: Deploy to Cloud Run service
+        id: deploy-service
+        uses: google-github-actions/deploy-cloudrun@v0
+        with:
+          env_vars: |
+            OCTUE_SERVICE_NAMESPACE=${{ needs.info.outputs.gcp_resource_affix }}
+            OCTUE_SERVICE_NAME=${{ needs.info.outputs.gcp_service_name }}
+            OCTUE_SERVICE_REVISION_TAG=${{ needs.info.outputs.version_slug }}
+            COMPUTE_PROVIDER=GOOGLE_CLOUD_RUN
+          service: ${{ needs.info.outputs.gcp_resource_affix }}-${{ needs.info.outputs.gcp_service_name }}
+          image: ${{ needs.info.outputs.image_version_artefact }}
+          region: ${{ needs.info.outputs.gcp_region }}
+          tag: v${{ needs.info.outputs.version_slug }}
+          flags: "--allow-unauthenticated"
+
+      - name: Show deployed service URL
+        run: echo "${{ steps.deploy-service.outputs.url }}"
+
+      - name: Create topic and subscription
+        uses: octue/[email protected]
+        with:
+          project_name: ${{ needs.info.outputs.gcp_project_name }}
+          service_namespace: ${{ needs.info.outputs.gcp_resource_affix }}
+          service_name: ${{ needs.info.outputs.gcp_service_name }}
+          service_revision_tag: ${{ needs.info.outputs.version_slug }}
+          push_endpoint: ${{ steps.deploy-service.outputs.url }}

octue.yaml

@@ -3,7 +3,7 @@ services:
     name: example-service-cloud-run
     app_source_path: example_service_cloud_run
     project_name: octue-sdk-python
-    region: europe-west2
+    region: europe-west1
     repository_owner: octue
     repository_name: example-service-cloud-run
     minimum_instances: 0

poetry.lock

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "example-service-cloud-run"
-version = "0.2.0"
+version = "0.2.1"
 description = "An example Octue service for deploying to Google Cloud Run."
 authors = ["Marcus Lugg <[email protected]>"]
 license = "MIT"
@@ -9,7 +9,7 @@ packages = [{include = "example_service_cloud_run"}]
 
 [tool.poetry.dependencies]
 python = "^3.9"
-octue = "0.43.2"
+octue = "0.45.0"
 
 [tool.poetry.group.dev.dependencies]
 gcp-storage-emulator = "2022.4.13"

pyproject.toml

@@ -9,7 +9,7 @@
 
 REPOSITORY_ROOT = os.path.dirname(os.path.dirname(__file__))
 TWINE_PATH = os.path.join(REPOSITORY_ROOT, "twine.json")
-TEST_BUCKET_NAME = "octue-test-bucket"
+TEST_BUCKET_NAME = "octue-sdk-python-test-bucket"
 
 
 class TestApp(unittest.TestCase):
@@ -24,10 +24,18 @@ def test_app(self):
         runner = Runner(
             app_src=os.path.join(REPOSITORY_ROOT, "example_service_cloud_run"),
             twine=TWINE_PATH,
+            # Output location is taken from the app configuration automatically in a service but not if `Runner` is used
+            # by itself.
+            output_location="gs://octue-sdk-python-test-bucket/example_output_datasets",
         )
 
         with patch("google.cloud.storage.blob.Blob.generate_signed_url", mock_generate_signed_url):
-            analysis = runner.run(input_values={"n_iterations": 3})
+            with self.assertLogs() as logging_context:
+                analysis = runner.run(input_values={"n_iterations": 3})
+
+        # Check log messages from app and submodules are emitted.
+        self.assertEqual(logging_context.records[0].name, "app")
+        self.assertEqual(logging_context.records[1].name, "example_service_cloud_run.submodule")
 
         # Check the output values.
         self.assertEqual(analysis.output_values, [1, 2, 3, 4, 5])