Merge pull request #15 from alinabuzachis/async_create_pattern

AAP-47753: Implement run_pattern_task() for PatternViewSet

Author: alinabuzachis

.pre-commit-config.yaml

@@ -52,3 +52,5 @@ repos:
       - id: mypy
         pass_filenames: false
         args: [--config-file=pyproject.toml, --ignore-missing-imports, "."]
+        additional_dependencies:
+          - types-requests

CONTRIBUTING.md

@@ -13,6 +13,7 @@ Hi there! We're excited to have you as a contributor.
     - [Set env variables for development](#set-env-variables-for-development)
     - [Configure postgres and run the dispatcher service](#configure-postgres-and-run-the-dispatcher-service)
     - [Configure and run the application](#configure-and-run-the-application)
+    - [Development configuration](#development-configuration)
   - [Updating dependencies](#updating-dependencies)
   - [Running linters and code checks](#running-linters-and-code-checks)
   - [Running tests](#running-tests)
@@ -85,6 +86,31 @@ In a separate terminal window, run:
 
 The application can be reached in your browser at `https://localhost:8000/`. The Django admin UI is accessible at `https://localhost:8000/admin` and the available API endpoints will be listed in the 404 information at `http://localhost:8000/api/pattern-service/v1/`.
 
+### Development Configuration
+
+Default configuration values for connecting to the Ansible Automation Platform (AAP) service are defined in `development_defaults.py`:
+
+```bash
+AAP_URL = "http://localhost:44926"        # Base URL of your AAP instance
+AAP_VALIDATE_CERTS = False                # Whether to verify SSL certificates
+AAP_USERNAME = "admin"                    # Username for AAP authentication
+AAP_PASSWORD = "password"                 # Password for AAP authentication
+```
+
+*Note*: These defaults are placeholders for local development only. You must provide proper values for your environment by setting environment variables prefixed with `PATTERN_SERVICE_` or via a `.env` file.
+For example:
+
+```bash
+export PATTERN_SERVICE_AAP_URL="http://your-ip-address:44926"
+export PATTERN_SERVICE_AAP_VALIDATE_CERTS="False"
+export PATTERN_SERVICE_AAP_USERNAME="admin"
+export PATTERN_SERVICE_AAP_PASSWORD="your-password"
+```
+
+This ensures secure and correct operation in your deployment or testing environment.
+
+Dynaconf will prioritize environment variables and values in `.env` over defaults in `development_defaults.py`.
+
 ## Updating dependencies
 
 Project dependencies for all environments are specified in the [pyproject.toml file](./pyproject.toml). A requirements.txt file is generated for each environment using pip-compile, to simplify dependency installation with pip.

core/apps.py

@@ -4,6 +4,8 @@
 from django.apps import AppConfig
 from django.conf import settings
 
+from core.utils import validate_url
+
 logger = logging.getLogger(__name__)
 
 
@@ -12,5 +14,29 @@ class CoreConfig(AppConfig):
     name = "core"
 
     def ready(self) -> None:
+        # List of required AAP variables
+        required_vars = [
+            "AAP_URL",
+            "AAP_USERNAME",
+            "AAP_PASSWORD",
+        ]
+
+        # Check that each required setting is defined
+        missing = [var for var in required_vars if not getattr(settings, var, None)]
+        if missing:
+            logger.error(
+                f"Missing required configuration variables: {', '.join(missing)}"
+            )
+            raise RuntimeError(
+                f"Required AAP variable not defined: {', '.join(missing)}. "
+            )
+
+        # Validate AAP_URL
+        try:
+            settings.AAP_URL = validate_url(settings.AAP_URL)
+        except ValueError as e:
+            logger.error(f"AAP_URL validation failed: {e}")
+            raise
+
         # Configure dispatcher
         dispatcher_setup(config=settings.DISPATCHER_CONFIG)

core/models.py

@@ -1,5 +1,9 @@
 from __future__ import annotations
 
+from typing import Any
+from typing import Dict
+from typing import Optional
+
 from ansible_base.lib.abstract_models import CommonModel
 from django.db import models
 
@@ -85,12 +89,54 @@ class Meta:
         app_label = "core"
         ordering = ["id"]
 
-    status_choices = (
-        ("Initiated", "Initiated"),
-        ("Running", "Running"),
-        ("Completed", "Completed"),
-        ("Failed", "Failed"),
-    )
+    class Status(models.TextChoices):
+        INITIATED = "Initiated"
+        RUNNING = "Running"
+        COMPLETED = "Completed"
+        FAILED = "Failed"
 
-    status: models.CharField = models.CharField(max_length=20, choices=status_choices)
+    status: models.CharField = models.CharField(max_length=20, choices=Status.choices)
     details: models.JSONField = models.JSONField(null=True, blank=True)
+
+    def set_status(
+        self,
+        new_status: str,
+        details: Optional[Dict[str, Any]] = None,
+        save_immediately: bool = True,
+    ) -> None:
+        """
+        Safely update the task's status and optional details.
+
+        Args:
+            new_status (str): The new status (must be one of Status.choices).
+            details (dict, optional): Additional info about this status update.
+            save_immediately (bool): If True, saves the instance to the database
+                immediately.
+
+        Raises:
+            ValueError: If the provided status is invalid.
+        """
+        if new_status not in self.Status.values:
+            raise ValueError(
+                f"Invalid status '{new_status}'. Allowed values: {self.Status.values}"
+            )
+
+        self.status = new_status
+        self.details = details or {}
+        if save_immediately:
+            self.save(update_fields=["status", "details"])
+
+    def mark_initiated(self, details: Optional[Dict[str, Any]] = None) -> None:
+        self.set_status(self.Status.INITIATED, details)
+
+    def mark_running(self, details: Optional[Dict[str, Any]] = None) -> None:
+        self.set_status(self.Status.RUNNING, details)
+
+    def mark_completed(self, details: Optional[Dict[str, Any]] = None) -> None:
+        self.set_status(self.Status.COMPLETED, details)
+
+    def mark_failed(self, details: Optional[Dict[str, Any]] = None) -> None:
+        self.set_status(self.Status.FAILED, details)
+
+    def __str__(self) -> str:
+        return f"Task #{self.pk} - {self.status}"

core/task_runner.py

@@ -0,0 +1,57 @@
+import json
+import logging
+import os
+
+from core.utils.controller import build_collection_uri
+from core.utils.controller import download_collection
+
+from .models import Pattern
+from .models import Task
+
+logger = logging.getLogger(__name__)
+
+
+def run_pattern_task(pattern_id: int, task_id: int) -> None:
+    """
+    Orchestrates downloading a collection and saving a pattern definition.
+
+    Args:
+        pattern_id (int): The ID of the pattern to process.
+        task_id (int): The ID of the task.
+
+    Raises:
+        FileNotFoundError: If the pattern definition is not found.
+        Exception: If any other error occurs.
+    """
+    task = Task.objects.get(id=task_id)
+    task.mark_initiated({"info": "Processing started"})
+    try:
+        pattern = Pattern.objects.get(id=pattern_id)
+        task.mark_running({"info": "Processing pattern"})
+        with download_collection(
+            pattern.collection_name, pattern.collection_version
+        ) as collection_path:
+            path_to_definition = os.path.join(
+                collection_path,
+                "extensions",
+                "patterns",
+                pattern.pattern_name,
+                "meta",
+                "pattern.json",
+            )
+            with open(path_to_definition, "r") as file:
+                definition = json.load(file)
+
+            pattern.pattern_definition = definition
+            pattern.collection_version_uri = build_collection_uri(
+                pattern.collection_name, pattern.collection_version
+            )
+            pattern.save(update_fields=["pattern_definition", "collection_version_uri"])
+        task.mark_completed({"info": "Pattern processed successfully"})
+    except FileNotFoundError:
+        logger.error(f"Could not find pattern definition for task {task_id}")
+        task.mark_failed({"error": "Pattern definition not found."})
+    except Exception as e:
+        error_message = f"An unexpected error occurred {str(e)}."
+        logger.exception(f"Task {task_id} failed unexpectedly.")
+        task.mark_failed({"error": error_message})

core/tests/test_controller_client.py

@@ -0,0 +1,8 @@
+import core.utils.controller.client as cc
+
+
+def test_get_http_session():
+    """Subsequent calls without force_refresh must not return the *same* object."""
+    s1 = cc.get_http_session()
+    s2 = cc.get_http_session()
+    assert s1 is not s2

core/tests/test_controller_helpers.py

@@ -0,0 +1,130 @@
+import os
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+import pytest
+
+from core.utils.controller import build_collection_uri
+from core.utils.controller import download_collection
+
+
+@pytest.fixture
+def mock_tar_data():
+    """A fixture to provide mock tarball data."""
+    return b"This is some mock tarball content"
+
+
+@pytest.fixture
+def mock_download_success(mock_tar_data):
+    """
+    Corrected fixture to mock a successful download scenario,
+    including mocking os.makedirs and ensuring cleanup.
+    """
+    with (
+        patch("core.utils.controller.helpers.get") as mock_get,
+        patch("core.utils.controller.helpers.tarfile.open") as mock_tar_open,
+        patch("core.utils.controller.helpers.os.makedirs") as mock_makedirs,
+        patch("core.utils.controller.helpers.shutil.rmtree") as mock_rmtree,
+        patch(
+            "core.utils.controller.helpers.tempfile.mkdtemp",
+            return_value="/mock/temp/dir",
+        ) as mock_mkdtemp,
+    ):
+
+        mock_response = MagicMock()
+        mock_response.raw = mock_tar_data
+        mock_get.return_value = mock_response
+
+        mock_tar_open.return_value.__enter__.return_value = MagicMock()
+
+        yield mock_get, mock_tar_open, mock_makedirs, mock_mkdtemp, mock_rmtree
+
+
+@pytest.fixture
+def mock_download_failure():
+    """
+    Fixture to mock a download failure scenario by raising an exception
+    and mocking the cleanup functions.
+    """
+    with (
+        patch(
+            "core.utils.controller.helpers.get", side_effect=Exception("Network error")
+        ) as mock_get,
+        patch("core.utils.controller.helpers.shutil.rmtree") as mock_rmtree,
+        patch("core.utils.controller.helpers.os.makedirs"),
+        patch(
+            "core.utils.controller.helpers.tempfile.mkdtemp",
+            return_value="/mock/temp/dir",
+        ) as mock_mkdtemp,
+    ):
+
+        yield mock_get, mock_rmtree, mock_mkdtemp
+
+
+@pytest.mark.parametrize(
+    "collection_name, version, expected_uri",
+    [
+        (
+            "another.collection",
+            "2.0.0",
+            (
+                "http://localhost:44926/api/galaxy/v3/plugin/ansible/content/published/"
+                "collections/artifacts/another-collection-2.0.0.tar.gz"
+            ),
+        ),
+        (
+            "edge.case",
+            "0.1.0-beta",
+            (
+                "http://localhost:44926/api/galaxy/v3/plugin/ansible/content/published/"
+                "collections/artifacts/edge-case-0.1.0-beta.tar.gz"
+            ),
+        ),
+    ],
+)
+def test_build_collection_uri(collection_name, version, expected_uri):
+    """
+    Tests that various collection names and versions build the correct URI.
+    """
+    assert build_collection_uri(collection_name, version) == expected_uri
+
+
+def test_download_collection_success(mock_download_success):
+    """
+    Tests the successful download and extraction of a collection.
+    """
+    mock_get, mock_tar_open, mock_makedirs, mock_mkdtemp, mock_rmtree = (
+        mock_download_success
+    )
+
+    collection_name = "my_namespace.my_collection"
+    version = "1.0.0"
+    expected_path = os.path.join("/mock/temp/dir", "my_namespace.my_collection-1.0.0")
+
+    with download_collection(collection_name, version) as path:
+        # Assert that the correct path was yielded
+        assert path == expected_path
+
+        mock_mkdtemp.assert_called_once()
+        mock_makedirs.assert_called_once_with(expected_path, exist_ok=True)
+        mock_get.assert_called_once()
+        mock_tar_open.assert_called_once()
+
+    mock_rmtree.assert_called_once_with("/mock/temp/dir")
+
+
+def test_download_collection_failure(mock_download_failure):
+    """
+    Tests that an exception during download.
+    """
+    mock_get, mock_rmtree, mock_mkdtemp = mock_download_failure
+
+    collection_name = "my_namespace.my_collection"
+    version = "1.0.0"
+
+    with pytest.raises(Exception, match="Network error"):
+        with download_collection(collection_name, version):
+            pass
+
+    mock_mkdtemp.assert_called_once()
+    mock_rmtree.assert_called_once_with("/mock/temp/dir")