Merge pull request #90 from octue/task-emulation

Allow Task Queue Emulation

Author: thclark

.devcontainer/devcontainer.json

@@ -51,7 +51,7 @@
   },
 
   // Use 'forwardPorts' to make a list of ports inside the container available locally.
-  "forwardPorts": [80, 443, 5000, 7045, 7046, 7047, 7048, 7049, 8000, 8080],
+  "forwardPorts": [5432, 6379, 8000, 8132],
 
   // Use 'postAttachCommand' to run commands after the container is created and attached (ie git is available).
   // This is preferential to postCreateCommand as invoking git (eg for private poetry installs or addition of

.devcontainer/docker-compose.yml

@@ -1,10 +1,16 @@
-version: '3.8'
-
 # Developers:
 #
 #  This file is under version control; changes made here will affect everyone.
 #  If you need to customise just for yourself, add your personal overrides to docker-compose.developer.yml
+
+# Networking:
 #
+#  This setup runs containers on the same network as the database container, with the
+#     network_mode: service:db
+#  definition. This allows "forwardPorts" in devcontainer.json to include 5432 (postgres),
+#  6437 (redis), 8123 (tasks emulator). Use "forwardPorts instead of "ports" properties here,
+#  to enable forwarding from a Codespace (which doesn't work otherwise).
+
 services:
   web:
     build:
@@ -27,9 +33,7 @@ services:
     # Overrides default command so things don't shut down after the process ends
     command: sleep infinity
 
-    # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function to include 5432
-    # Use "forwardPorts" in **devcontainer.json** to forward an app port locally, because adding the "ports" property to this
-    # file will not forward from a Codespace.
+    # Run on the same network as the database container
     network_mode: service:db
 
     # Use the non-root user for all processes
@@ -45,6 +49,13 @@ services:
     depends_on:
       - db
 
+  tasks:
+    image: ghcr.io/aertje/cloud-tasks-emulator:latest
+    command: -host 0.0.0.0 -port 8123 -queue "projects/octue-django-gcp/locations/europe-west1/queues/example-primary"
+    network_mode: service:db
+    depends_on:
+      - db
+
   db:
     image: postgres:13.0
     environment:

django_gcp/exceptions.py

@@ -10,11 +10,11 @@ class IncorrectTaskUsageError(Exception):
     """Raised when one of the Task classes is directly instantiated, instead of subclassed"""
 
 
-class InvalidEndpointError(Exception):
-    """Raised when a specified endpoint isn't valid"""
+class InvalidTaskDomainError(Exception):
+    """Raised when the specified tasks domain isn't valid"""
 
     def __init__(self, endpoint):
-        message = f"The endpoint is invalid for use with a GCP pub/sub push subscription. Endpoints need to be valid and secure. ({endpoint}) "
+        message = f"The GCP_TASKS_DOMAIN setting is invalid for use with GCP Tasks or PubSub push subscriptions. Domain must use https:// protocol in production. Given value: {endpoint}"
         super().__init__(message)
 
 

django_gcp/tasks/_patch_cloud_scheduler.py

@@ -2,11 +2,10 @@
 
 This module is extracted from gcp_pilot, a third party library.
 
-We were relying on gcp_pilot, which is mostly well engineered, to provide a better abstraction than the google cloud apis themselves. However,
-the authors of gcp_pilot keep shifting their python dependency range - it's super tight for no clear reason and that prevents downstream users
-from being abole to maintain systems. It's a headache.
+We were relying on gcp_pilot, which is mostly well engineered, to provide a better abstraction than the google cloud apis themselves. However, the authors of gcp_pilot keep shifting their python dependency range - it's super tight for no clear reason and that prevents downstream users from being able to maintain systems. It's a headache.
 
-We also have the probelm that they don't bother producing any release notes, so updating is a bear.
+We also have the problem that they don't bother producing any release notes, so updating is a bear.
 
-But, we've got very little time right now, so are simply porting some of their code here for the time being to overcome the dependency problems.
-We will come back and refacotr the required aspects to remove this private module.
+But, we've got very little time right now, so are simply porting some of their code here for the time being to overcome the dependency problems. We will come back and refactor the required aspects to remove this private module.
+
+YOU SHOULD NOT IMPORT FROM OR PATCH THIS MODULE, IT IS PRIVATE AND WILL BE REMOVED WITHOUT NOTIFYING BREAKING CHANGES.

django_gcp/tasks/_patch_cloud_tasks.py

@@ -287,25 +287,6 @@ def _unbind_email_from_policy(self, email: str, role: str, policy: Dict):
         return policy
 
 
-class AppEngineBasedService:
-    # This mixin must be used for all clients that uses GCP's resources that are based on App Engine
-    # such as Cloud Scheduler.
-    # They have a peculiar behaviour because the App Engine, when enabled the first time,
-    # is created in a region and this region cannot be changed ever.
-    # So these clients cannot just choose a desired region to work on, they must use the App Engine's
-    # previously chosen region.
-    def _set_location(self, location: str = None):
-        project_location = self._get_project_default_location()
-
-        explicit_location = location
-        if explicit_location and explicit_location != project_location:
-            logger.warning(
-                f"Location {explicit_location} cannot be set in {self.__class__.__name__}. "
-                f"It uses App Engine's default location for your project: {project_location}"
-            )
-        return project_location
-
-
 def friendly_http_error(func):
     _reasons = {
         "notFound": exceptions.NotFound,

django_gcp/tasks/_pilot/README.md

@@ -21,6 +21,12 @@ class FailedPrecondition(Exception):
     pass
 
 
+class DoesNotExist(Exception):
+    def __init__(self, resource):
+        message = f"{resource} does not exist."
+        super().__init__(message)
+
+
 class DeletedRecently(FailedPrecondition):
     def __init__(self, resource, blocked_period="1 week"):
         message = f"{resource} was probably deleted recently. Cannot reuse name for {blocked_period}."

django_gcp/tasks/_pilot/app_engine.py

@@ -18,7 +18,6 @@ def __init__(self, project_id: str = "potato-dev", location: str = "moon-dark1",
         managers = [
             patch("google.auth.default", return_value=(credentials, project_id)),
             patch("django_gcp.tasks._pilot.base.GoogleCloudPilotAPI._set_location", return_value=location),
-            patch("django_gcp.tasks._pilot.base.AppEngineBasedService._set_location", return_value=location),
         ]
         self.stack = ExitStack()
         for mgr in managers: