Fix running e2e backend

.github/workflows/run-ci-cd.yaml

@@ -251,7 +251,7 @@ jobs:
 
       - name: Run frontend end-to-end tests
         run: |
-          docker run --env-file frontend/.env.example owasp/nest:test-frontend-e2e-latest pnpm run test:e2e
+          docker run --env-file frontend/.env.e2e.example owasp/nest:test-frontend-e2e-latest pnpm run test:e2e
 
   set-release-version:
     name: Set release version

.github/workflows/setup-e2e-environment/action.yaml

@@ -7,23 +7,18 @@ runs:
     steps:
       - name: Wait for database to be ready
         run: |
-          until docker exec ${{ job.services.db.id }} pg_isready -U nest_user_e2e -d nest_db_e2e; do
-            echo "Waiting for database..."
-            sleep 5
-          done
+          timeout 5m bash -c '
+            until docker exec ${{ job.services.db.id }} pg_isready -U nest_user_e2e -d nest_db_e2e; do
+              echo "Waiting for database..."
+              sleep 5
+            done
+          '
         shell: bash
 
       - name: Install PostgreSQL client
         run: sudo apt-get install -y postgresql-client
         shell: bash
 
-      - name: Load Postgres data
-        env:
-          PGPASSWORD: nest_user_e2e_password
-        run: |
-          gunzip -c backend/data/nest-e2e.sql.gz | psql -h localhost -U nest_user_e2e -d nest_db_e2e
-        shell: bash
-
       - name: Build backend e2e image
         uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83
         with:
@@ -42,18 +37,29 @@ runs:
           docker run -d --rm --name e2e-nest-backend \
             --env-file backend/.env.e2e.example \
             --network host \
+            -e DJANGO_DB_HOST=localhost \
             -p 9000:9000 \
             owasp/nest:test-backend-e2e-latest \
             sh -c '
+              python manage.py migrate &&
               gunicorn wsgi:application --bind 0.0.0.0:9000
           '
         shell: bash
 
       - name: Waiting for the backend to be ready
         run: |
-          until wget --spider http://localhost:9000/a; do
-            echo "Waiting for backend..."
-            sleep 5
-          done
+          timeout 5m bash -c '
+            until wget --spider http://localhost:9000/a; do
+              echo "Waiting for backend..."
+              sleep 5
+            done
+          '
           echo "Backend is up!"
         shell: bash
+
+      - name: Load Postgres data
+        env:
+          PGPASSWORD: nest_user_e2e_password
+        run: |
+          gunzip -c backend/data/nest.sql.gz | psql -h localhost -U nest_user_e2e -d nest_db_e2e
+        shell: bash

CONTRIBUTING.md

@@ -286,6 +286,8 @@ Ensure that all `.env` files are saved in **UTF-8 format without BOM (Byte Order
 
 1. **Load Initial Data**:
 
+   - Make sure you have `gzip` installed on your machine.
+
    - Open a new terminal session and run the following command to populate the database with initial data from fixtures:
 
    ```bash

backend/.env.e2e.example

@@ -6,7 +6,7 @@ DJANGO_AWS_ACCESS_KEY_ID=None
 DJANGO_AWS_SECRET_ACCESS_KEY=None
 DJANGO_SETTINGS_MODULE=settings.e2e
 DJANGO_CONFIGURATION=E2E
-DJANGO_DB_HOST=None
+DJANGO_DB_HOST=db
 DJANGO_DB_NAME=nest_db_e2e
 DJANGO_DB_USER=nest_user_e2e
 DJANGO_DB_PASSWORD=nest_user_e2e_password

backend/Makefile

@@ -49,22 +49,7 @@ django-shell:
 
 dump-data:
 	@echo "Dumping Nest data"
-	@CMD="python manage.py dumpdata \
-		github \
-		owasp \
-		slack.Conversation \
-		slack.Member \
-		slack.Message \
-		slack.Workspace \
-		--indent=4 \
-		--natural-foreign \
-		--natural-primary -o data/nest.json" $(MAKE) exec-backend-command
-	@CMD="sed -E -i 's/(\"[^\"]*email\"): *\"([^\"]|\\\")*\"/\1: \"\"/g' data/nest.json" $(MAKE) exec-backend-command
-	@CMD="gzip -f data/nest.json" $(MAKE) exec-backend-command
-
-dump-data-e2e:
-	@echo "Dumping Nest e2e data"
-	@CMD="pg_dumpall -U nest_user_e2e --clean | gzip -9 > backend/data/nest-e2e.sql.gz" $(MAKE) exec-db-command-e2e
+	@CMD="python manage.py dump_data" $(MAKE) exec-backend-command-it
 
 enrich-data: \
 	github-enrich-issues \
@@ -84,11 +69,11 @@ index-data:
 
 load-data:
 	@echo "Loading Nest data"
-	@CMD="python manage.py load_data" $(MAKE) exec-backend-command
+	@gunzip -c backend/data/nest.sql.gz | docker exec -i nest-db psql -U nest_user_dev -d nest_db_dev
 
 load-data-e2e:
 	@echo "Loading Nest e2e data"
-	@gunzip -c backend/data/nest-e2e.sql.gz | docker exec -i e2e-nest-db psql -U nest_user_e2e -d nest_db_e2e
+	@gunzip -c backend/data/nest.sql.gz | docker exec -i e2e-nest-db psql -U nest_user_e2e -d nest_db_e2e
 
 merge-migrations:
 	@CMD="python manage.py makemigrations --merge" $(MAKE) exec-backend-command

backend/apps/api/rest/v0/__init__.py

@@ -57,6 +57,17 @@
         ],
         "throttle": [],
     }
+elif settings.IS_E2E_ENVIRONMENT:
+    api_settings_customization = {
+        "auth": None,
+        "servers": [
+            {
+                "description": "E2E",
+                "url": settings.SITE_URL,
+            }
+        ],
+        "throttle": [],
+    }
 elif settings.IS_STAGING_ENVIRONMENT:
     api_settings_customization = {
         "servers": [

backend/apps/common/management/commands/dump_data.py

@@ -0,0 +1,127 @@
+"""Dump masked data from the database into a compressed file."""
+
+import contextlib
+import os
+from pathlib import Path
+from subprocess import CalledProcessError, run
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+from psycopg2 import ProgrammingError, connect, sql
+
+DEFAULT_DATABASE = settings.DATABASES["default"]
+DB_HOST = DEFAULT_DATABASE.get("HOST", "localhost")
+DB_PORT = str(DEFAULT_DATABASE.get("PORT", "5432"))
+DB_USER = DEFAULT_DATABASE.get("USER", "")
+DB_PASSWORD = DEFAULT_DATABASE.get("PASSWORD", "")
+DB_NAME = DEFAULT_DATABASE.get("NAME", "")
+
+
+class Command(BaseCommand):
+    help = "Create a dump of selected db tables."
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--output",
+            default=str(Path(settings.BASE_DIR) / "data" / "nest.sql.gz"),
+            help="Output dump path (default: data/nest.sql.gz)",
+        )
+        parser.add_argument(
+            "-t",
+            "--table",
+            action="append",
+            dest="tables",
+            default=[
+                "public.owasp_*",
+                "public.github_*",
+                "public.slack_members",
+                "public.slack_workspaces",
+                "public.slack_conversations",
+                "public.slack_messages",
+            ],
+            help="Table pattern to include",
+        )
+
+    def handle(self, *args, **options):
+        output_path = Path(options["output"]).resolve()
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        tables = options["tables"] or []
+
+        env = os.environ.copy()
+        env["PGPASSWORD"] = DB_PASSWORD
+
+        temp_db = f"temp_{DB_NAME}"
+        try:
+            self._execute_sql("postgres", [f"CREATE DATABASE {temp_db} TEMPLATE {DB_NAME};"])
+
+            self.stdout.write(self.style.SUCCESS(f"Created temporary DB: {temp_db}"))
+
+            table_list = self._execute_sql(temp_db, [self._table_list_query()])
+            self._execute_sql(temp_db, self._remove_emails([row[0] for row in table_list]))
+            self.stdout.write(self.style.SUCCESS("Removed emails from temporary DB"))
+
+            dump_cmd = [
+                "pg_dump",
+                "-h",
+                DB_HOST,
+                "-p",
+                DB_PORT,
+                "-U",
+                DB_USER,
+                "-d",
+                temp_db,
+                "--compress=9",
+                "--clean",
+            ]
+            dump_cmd += [f"--table={table}" for table in tables]
+            dump_cmd += ["-f", str(output_path)]
+
+            run(dump_cmd, check=True, env=env)
+            self.stdout.write(self.style.SUCCESS(f"Created dump: {output_path}"))
+        except CalledProcessError as e:
+            message = f"Command failed: {e.cmd}"
+            raise CommandError(message) from e
+        finally:
+            try:
+                self._execute_sql("postgres", [f"DROP DATABASE IF EXISTS {temp_db};"])
+            except CalledProcessError:
+                self.stderr.write(
+                    self.style.WARNING(f"Failed to drop temp DB {temp_db} (ignored).")
+                )
+
+    def _table_list_query(self) -> str:
+        return """
+        SELECT table_name
+        FROM information_schema.columns
+        WHERE table_schema = 'public' AND column_name = 'email';
+        """
+
+    def _remove_emails(self, tables: list[str]) -> list[str]:
+        return [
+            sql.SQL("UPDATE {table} SET email = '';").format(table=sql.Identifier(table))
+            for table in tables
+        ]
+
+    def _execute_sql(
+        self,
+        dbname: str,
+        sql_queries: list[str],
+    ):
+        connection = connect(
+            dbname=dbname,
+            user=DB_USER,
+            password=DB_PASSWORD,
+            host=DB_HOST,
+            port=DB_PORT,
+        )
+        connection.autocommit = True
+
+        rows = []
+        with connection.cursor() as cursor:
+            for sql_query in sql_queries:
+                cursor.execute(sql_query)
+                with contextlib.suppress(ProgrammingError):
+                    rows.extend(cursor.fetchall())
+        connection.close()
+
+        return rows

backend/pyproject.toml

@@ -94,6 +94,10 @@ lint.per-file-ignores."**/management/commands/*.py" = [
   "D102", # https://docs.astral.sh/ruff/rules/undocumented-public-method/
   "T201", # https://docs.astral.sh/ruff/rules/print/
 ]
+lint.per-file-ignores."**/management/commands/dump_data.py" = [
+  "S603", # https://docs.astral.sh/ruff/rules/subprocess-without-shell-equals-true/,
+  "S607", # https://docs.astral.sh/ruff/rules/start-process-with-partial-path/,
+]
 lint.per-file-ignores."**/migrations/*.py" = [
   "D100",   # https://docs.astral.sh/ruff/rules/undocumented-public-module/
   "D101",   # https://docs.astral.sh/ruff/rules/undocumented-public-class/

backend/settings/e2e.py

@@ -9,15 +9,22 @@ class E2E(Base):
     """End-to-end testing configuration."""
 
     APP_NAME = "OWASP Nest E2E Testing"
+    SITE_URL = "http://localhost:9000"
 
     ALLOWED_ORIGINS = (
         "http://frontend:3000",  # NOSONAR
         "http://localhost:3000",
     )
+
+    CACHES = {
+        "default": {
+            "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
+        }
+    }
+
     CORS_ALLOWED_ORIGINS = ALLOWED_ORIGINS
     CSRF_TRUSTED_ORIGINS = ALLOWED_ORIGINS
 
-    DEBUG = False
     IS_E2E_ENVIRONMENT = True
     LOGGING = {}
     PUBLIC_IP_ADDRESS = values.Value()