Add logging to v2 APIs

We have added middleware logging for request/response to API calls via v2 APIs. The logging
information includes the IP address, user (if authenticated), request details, and response details.
Enabling or disabling this logging can be configured through setting.ENABLE_API_LOGGING.

Author: hoangpn

promgen/middleware.py

@@ -17,7 +17,9 @@
 caching system to set a key and then triggering the actual event from middleware
 """
 
+import json
 import logging
+import uuid
 from http import HTTPStatus
 from threading import local
 
@@ -36,6 +38,12 @@
 
 
 class PromgenMiddleware:
+    ACCESS_LOG_FIELD_LIMITS = {
+        "body": 8192,
+        "description": 512,
+        "clause": 8192,
+    }
+
     def __init__(self, get_response):
         self.get_response = get_response
 
@@ -53,8 +61,46 @@ def __call__(self, request):
         if request.user.is_authenticated:
             _user.value = request.user
 
+        # Log all requests to our v2 API endpoints
+        if settings.ENABLE_API_LOGGING and request.path.startswith("/rest/v2/"):
+            try:
+                # Generate a trace ID for each request
+                trace_id = str(uuid.uuid4())
+                request.trace_id = trace_id
+                # Log the IP address of the request
+                ip_address = request.META.get("REMOTE_ADDR")
+                logger.info(f"[Trace ID: {trace_id}] IP Address: {ip_address}")
+                # Log the user if authenticated
+                if request.user.is_authenticated:
+                    logger.info(f"[Trace ID: {trace_id}] User: {request.user.username}")
+                # Log the request details
+                logger.info(
+                    f"[Trace ID: {request.trace_id}] Request: {request.method} {request.get_full_path()} - body size: {len(request.body) if request.body else 0} bytes"
+                )
+                if request.body and request.headers["Content-Type"] == "application/json":
+                    # Only log first 512 characters of the request body to avoid flooding the logs
+                    logger.info(
+                        f"[Trace ID: {request.trace_id}] Request body: {self.truncate_json_fields(json.loads(request.body), self.ACCESS_LOG_FIELD_LIMITS)}"
+                    )
+            except Exception as e:
+                logger.exception(
+                    f"[Trace ID: {request.trace_id}] An error occurred when parsing request: {str(e)}"
+                )
+
         response = self.get_response(request)
 
+        # Log all responses to our v2 API endpoints
+        if settings.ENABLE_API_LOGGING and request.path.startswith("/rest/v2/"):
+            try:
+                # Log the response details
+                logger.info(
+                    f"[Trace ID: {request.trace_id}] Response status: {response.status_code} - content size: {len(response.content)} bytes"
+                )
+            except Exception as e:
+                logger.exception(
+                    f"[Trace ID: {request.trace_id}] An error occurred when logging response: {str(e)}"
+                )
+
         triggers = {
             "Config": trigger_write_config.send,
             "Rules": trigger_write_rules.send,
@@ -67,6 +113,36 @@ def __call__(self, request):
                     messages.warning(request, "Error queueing %s " % msg)
         return response
 
+    @staticmethod
+    def truncate_json_fields(data, field_limits, default_limits=128):
+        """
+        Recursively truncate fields in a JSON object based on a given field-length mapping.
+
+        Args:
+            data (dict): The JSON object to process.
+            field_limits (dict): A mapping of field names to their maximum lengths.
+            default_limits (int): Default maximum length for fields not specified in field_limits.
+
+        Returns:
+            dict: A new JSON object with truncated fields.
+        """
+        truncated_data = {}
+        for field, value in data.items():
+            if isinstance(value, dict):
+                # Recursively process nested dictionaries
+                truncated_data[field] = PromgenMiddleware.truncate_json_fields(value, field_limits)
+            elif field in field_limits and isinstance(value, str):
+                truncated_data[field] = value[: field_limits[field]] + (
+                    "..." if len(value) > field_limits[field] else ""
+                )
+            elif isinstance(value, str):
+                truncated_data[field] = value[:default_limits] + (
+                    "..." if len(value) > default_limits else ""
+                )
+            else:
+                truncated_data[field] = value
+        return truncated_data
+
 
 def get_current_user():
     return getattr(_user, "value", None)

promgen/settings.py

@@ -237,3 +237,5 @@
     "DESCRIPTION": "Non-GET APIs require authentication. "
     "Please login Promgen and access the Profile page to get a Token.",
 }
+
+ENABLE_API_LOGGING = env.bool("ENABLE_API_LOGGING", default=False)