[2.22] Migrate to Django 5 STORAGES and get_domain()

.github/workflows/scripts/install.sh

@@ -95,7 +95,7 @@ if [ "$TEST" = "s3" ]; then
   sed -i -e '$a s3_test: true\
 minio_access_key: "'$MINIO_ACCESS_KEY'"\
 minio_secret_key: "'$MINIO_SECRET_KEY'"\
-pulp_scenario_settings: {"flatpak_index": false, "token_auth_disabled": true}\
+pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "flatpak_index": false, "token_auth_disabled": true}\
 pulp_scenario_env: {}\
 ' vars/main.yaml
   export PULP_API_ROOT="/rerouted/djnd/"
@@ -109,7 +109,7 @@ if [ "$TEST" = "azure" ]; then
       - ./azurite:/etc/pulp\
     command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0"' vars/main.yaml
   sed -i -e '$a azure_test: true\
-pulp_scenario_settings: {"flatpak_index": true}\
+pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "flatpak_index": true}\
 pulp_scenario_env: {}\
 ' vars/main.yaml
 fi

pulp_container/app/registry.py

@@ -12,12 +12,12 @@
 from aiohttp.web_exceptions import HTTPTooManyRequests
 from django_guid import set_guid
 from django_guid.utils import generate_guid
-from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import IntegrityError
 from multidict import MultiDict
 
 from pulpcore.plugin.content import Handler, PathNotResolved
+from pulpcore.plugin.util import get_domain
 from pulpcore.plugin.models import RemoteArtifact, Content, ContentArtifact
 from pulpcore.plugin.content import ArtifactResponse
 from pulpcore.plugin.tasking import dispatch
@@ -83,9 +83,10 @@ async def _dispatch(artifact, headers):
         full_headers["Docker-Content-Digest"] = headers["Docker-Content-Digest"]
         full_headers["Docker-Distribution-API-Version"] = "registry/2.0"
 
-        if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem":
+        domain = get_domain()
+        if domain.storage_class == "pulpcore.app.models.storage.FileSystem":
             file = artifact.file
-            path = os.path.join(settings.MEDIA_ROOT, file.name)
+            path = os.path.join(domain.get_storage().location, file.name)
             if not os.path.exists(path):
                 raise Exception("Expected path '{}' is not found".format(path))
             return web.FileResponse(path, headers=full_headers)

pulp_container/app/registry_api.py

@@ -18,7 +18,6 @@
 from tempfile import NamedTemporaryFile
 
 from django.core.exceptions import ObjectDoesNotExist
-from django.core.files.storage import default_storage as storage
 from django.core.files.base import ContentFile, File
 from django.db import IntegrityError, transaction
 from django.db.models import F, Value
@@ -30,7 +29,7 @@
 from pulpcore.plugin.models import Artifact, ContentArtifact, UploadChunk
 from pulpcore.plugin.files import PulpTemporaryUploadedFile
 from pulpcore.plugin.tasking import add_and_remove, dispatch
-from pulpcore.plugin.util import get_objects_for_user, get_url
+from pulpcore.plugin.util import get_objects_for_user, get_url, get_domain
 from pulpcore.plugin.exceptions import TimeoutException
 
 from rest_framework.exceptions import (
@@ -596,7 +595,7 @@ def get_manifest_config(self, manifest):
                 except Artifact.DoesNotExist:
                     log.warning(f"Manifest {manifest.pk}'s config blob was not found.")
                 else:
-                    with storage.open(config_artifact.file.name) as file:
+                    with get_domain().get_storage().open(config_artifact.file.name) as file:
                         raw_data = file.read()
                     config_data = json.loads(raw_data)
                     config["labels"] = config_data.get("config", {}).get("Labels")
@@ -989,14 +988,15 @@ def __init__(self, *args, **kwargs):
         """
         super().__init__(*args, **kwargs)
 
+        domain = get_domain()
         if (
-            settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem"
-            or not settings.REDIRECT_TO_OBJECT_STORAGE
+            domain.storage_class == "pulpcore.app.models.storage.FileSystem"
+            or not domain.redirect_to_object_storage
         ):
             self.redirects_class = FileStorageRedirects
-        elif settings.DEFAULT_FILE_STORAGE == "storages.backends.s3boto3.S3Boto3Storage":
+        elif domain.storage_class == "storages.backends.s3boto3.S3Boto3Storage":
             self.redirects_class = S3StorageRedirects
-        elif settings.DEFAULT_FILE_STORAGE == "storages.backends.azure_storage.AzureStorage":
+        elif domain.storage_class == "storages.backends.azure_storage.AzureStorage":
             self.redirects_class = AzureStorageRedirects
         else:
             raise NotImplementedError()
@@ -1195,6 +1195,8 @@ def put(self, request, path, pk=None):
         """
         Responds with the actual manifest
         """
+        domain = get_domain()
+        storage = domain.get_storage()
         # iterate over all the layers and create
         chunk = request.META["wsgi.input"]
         artifact = self.receive_artifact(chunk)

pulp_container/app/tasks/sign.py

@@ -8,6 +8,7 @@
 from django.db.models import Q
 
 from pulpcore.plugin.models import Repository
+from pulpcore.plugin.util import get_domain
 
 from pulp_container.app.models import (
     ManifestSignature,
@@ -106,7 +107,7 @@ async def create_signature(manifest, reference, signing_service):
         if not manifest.data:
             # TODO: BACKWARD COMPATIBILITY - remove after fully migrating to artifactless manifest
             artifact = await manifest._artifacts.aget()
-            if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem":
+            if get_domain().storage_class != "pulpcore.app.models.storage.FileSystem":
                 async with tempfile.NamedTemporaryFile(dir=".", mode="wb", delete=False) as tf:
                     await tf.write(await sync_to_async(artifact.file.read)())
                     await tf.flush()

pulp_container/app/utils.py

@@ -10,7 +10,7 @@
 
 from asgiref.sync import sync_to_async
 from jsonschema import Draft7Validator, validate, ValidationError
-from django.core.files.storage import default_storage as storage
+from pulpcore.plugin.util import get_domain
 from django.db import IntegrityError
 from functools import partial
 from rest_framework.exceptions import Throttled
@@ -310,6 +310,7 @@ async def save_artifact(artifact_attributes):
 
 
 def get_content_data(saved_artifact):
+    storage = get_domain().get_storage()
     with storage.open(saved_artifact.file.name, mode="rb") as file:
         raw_data = file.read()
     content_data = json.loads(raw_data)

pulp_container/tests/functional/api/test_content_cache.py

@@ -30,13 +30,7 @@
 
 from pulp_container.constants import MEDIA_TYPE
 
-STANDARD_FILE_STORAGE_FRAMEWORKS = [
-    "django.core.files.storage.FileSystemStorage",
-    "pulpcore.app.models.storage.FileSystem",
-]
-
 cli_client = cli.Client(config.get_config())
-DEFAULT_FILE_STORAGE = utils.get_pulp_setting(cli_client, "DEFAULT_FILE_STORAGE")
 CACHE_ENABLED = utils.get_pulp_setting(cli_client, "CACHE_ENABLED")
 
 PULP_CONTENT_HOST_BASE_URL = config.get_config().get_base_url()
@@ -190,13 +184,10 @@ def check_cache(self, url, expected_metadata, headers):
 
     def fetch_response_metadata(self, response):
         """Retrieve metadata from the passed response and normalize status code for redirects."""
-        if DEFAULT_FILE_STORAGE in STANDARD_FILE_STORAGE_FRAMEWORKS:
-            return response.status_code, response.headers.get("X-PULP-CACHE")
-        else:
-            if response.history:
-                response = response.history[0]
-                response.status_code = 200
-            return response.status_code, response.headers.get("X-PULP-CACHE")
+        if response.history:
+            response = response.history[0]
+            response.status_code = 200
+        return response.status_code, response.headers.get("X-PULP-CACHE")
 
 
 def cache_status_first_func(i):

template_config.yml

@@ -55,10 +55,39 @@ pulp_settings:
   flatpak_index: true
 pulp_settings_azure:
   flatpak_index: true
+  MEDIA_ROOT: ''
+  STORAGES:
+    default:
+      BACKEND: storages.backends.azure_storage.AzureStorage
+      OPTIONS:
+        account_key: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+        account_name: devstoreaccount1
+        azure_container: pulp-test
+        connection_string: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;
+        expiration_secs: 120
+        location: pulp3
+        overwrite_files: true
+    staticfiles:
+      BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage
 pulp_settings_gcp: null
 pulp_settings_s3:
   flatpak_index: false
   token_auth_disabled: true
+  MEDIA_ROOT: ''
+  STORAGES:
+    default:
+      BACKEND: storages.backends.s3boto3.S3Boto3Storage
+      OPTIONS:
+        access_key: AKIAIT2Z5TDYPX3ARJBA
+        addressing_style: path
+        bucket_name: pulp3
+        default_acl: '@none'
+        endpoint_url: http://minio:9000
+        region_name: eu-central-1
+        secret_key: fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS
+        signature_version: s3v4
+    staticfiles:
+      BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage
 pydocstyle: true
 release_email: pulp-infra@redhat.com
 release_user: pulpbot