feat(ai): modified blur detector by adding edge density and flicker rate

backend/src/index.ts

@@ -19,6 +19,9 @@ import { authorizationChecker } from './shared/functions/authorizationChecker.js
 import { currentUserChecker } from './shared/functions/currentUserChecker.js';
 import { startCron } from './utils/startCron.js';
 import { GLOBAL_TYPES } from './types.js';
+import dns from 'dns';
+
+dns.setServers(["1.1.1.1", "8.8.8.8"]);
 
 const app = express();
 const globalRateLimiter = createRateLimiter();

backend/src/modules/notifications/services/InviteService.ts

@@ -85,7 +85,7 @@ export class InviteService extends BaseService {
           `Before you begin, please carefully read and follow the instructions below to ensure a smooth and compliant experience:\n` +
           `- Speaking is strictly prohibited. If the system detects speaking, there is zero tolerance, and the video will immediately roll back to the start, pausing with an alert dialog.\n` +
           `- Ensure your camera remains uninterrupted. Any camera interruptions will be detected and may result in penalty score increases and video rollback.\n` +
-          `- Do not use a blurred background. The AI proctoring system tracks background clarity. A blurred background may trigger penalties and video rollback.\n` +
+          `- Do not use virtual backgrounds, background blur, or any software-based background effects. The AI proctoring system detects artificial background manipulation and may trigger penalties.\n` +
           `- No other person should appear near you during the session. The system monitors for additional individuals in the camera’s view. Detection of more than one person leads to immediate video rollback and a pause until the area is clear.\n` +
           `- Allow microphone access. The system needs mic access to detect speaking, which is strictly prohibited and may result in penalties and video rollback.\n\n` +
           `By following these rules, you help maintain the integrity and fairness of the course environment.\n\n` +
@@ -268,7 +268,7 @@ export class InviteService extends BaseService {
   //             courseVersionId,
   //           ))
   //           : false;
-
+//--
   //         const invite = new Invite(
   //           email,
   //           new ObjectId(courseId),

backend/src/shared/database/interfaces/ISettingRepository.ts

@@ -22,7 +22,7 @@ import {
 // Enum representing the different components of proctoring that can be enabled or disabled.
 export enum ProctoringComponent {
   CAMERAMICRO = 'cameraMic',
-  BLURDETECTION = 'blurDetection', // bulrDetection
+  BLURDETECTION = 'blurDetection', // blurDetection
   FACECOUNTDETECTION = 'faceCountDetection', // faceCountDetection
   HANDGESTUREDETECTION = 'handGestureDetection', // handGestureDetection
   VOICEDETECTION = 'voiceDetection', // voiceDetection

backend/tsconfig.json

@@ -30,7 +30,7 @@
       "#courses/*": ["./modules/courses/*"],
       "#users/*": ["./modules/users/*"],
       "#quizzes/*": ["./modules/quizzes/*"],
-      "#settings/*": ["./modules/settings/*"],
+      "#setting/*": ["./modules/setting/*"],
       "#ejectionPolicy/*": ["./modules/ejectionPolicy/*"]
     },
 

frontend/package.json

@@ -24,6 +24,7 @@
     "@emotion/styled": "^11.14.0",
     "@mediapipe/face_detection": "^0.4.1646425229",
     "@mediapipe/face_mesh": "^0.4.1633559619",
+    "@mediapipe/selfie_segmentation": "^0.1.1675465747",
     "@mediapipe/tasks-audio": "0.10.22-rc.20250304",
     "@mediapipe/tasks-vision": "0.10.22-rc.20250304",
     "@mui/icons-material": "^7.1.0",

frontend/src/components/EditProctoringModal.tsx

@@ -73,7 +73,8 @@ export function ProctoringModal({
       try {
         const result = await getSettings(courseId, courseVersionId);
         if (result) {
-          setDetectors(result.settings?.proctors?.detectors?.map((d: any) => ({ name: d.detectorName, enabled: d.settings.enabled })))
+          //setDetectors(result.settings?.proctors?.detectors?.map((d: any) => ({ name: d.detectorName, enabled: d.settings.enabled })))
+          setDetectors(result.settings?.proctors?.detectors?.map((d: any) => ({name: d.detectorName,enabled:d.detectorName === ProctoringComponent.BLURDETECTION? true: d.settings.enabled})))//Backend independent
           setLinearProgressionEnabled(result.settings?.linearProgressionEnabled)
           setSeekForwardEnabled(result.settings?.seekForwardEnabled ?? false)
           setIsPublic(result.settings?.isPublic ?? false)

frontend/src/components/ai/BlurDetector.tsx

@@ -2,16 +2,30 @@ import React, { useEffect, useRef } from "react";
 
 import type { BlurDetectionProps } from "@/types/ai.types";
 
+import { SelfieSegmentation } from "@mediapipe/selfie_segmentation";
+
+/*const frameDiff = (prev: ImageData, curr: ImageData): number => {
+  let diff = 0;
+  for (let i = 0; i < prev.data.length; i += 4) {
+    diff += Math.abs(prev.data[i] - curr.data[i]);
+  }
+  return diff / (prev.data.length / 4);
+};*/
+
+
 const BlurDetection: React.FC<BlurDetectionProps> = ({ videoRef, setIsBlur }) => {
   const workerRef = useRef<Worker | null>(null);
   const blurStartTimeRef = useRef<number | null>(null);
+  const segmentationRef = useRef<SelfieSegmentation | null>(null);
+  const prevFrameRef = useRef<ImageData | null>(null);
+
 
   useEffect(() => {
     workerRef.current = new Worker(new URL("./BlurDetectorWorker.ts", import.meta.url), {
       type: "module",
     });
 
-    workerRef.current.onmessage = (event) => {
+    /*workerRef.current.onmessage = (event) => {
       const { isBlurry } = event.data;
 
       if (isBlurry) {
@@ -33,22 +47,126 @@ const BlurDetection: React.FC<BlurDetectionProps> = ({ videoRef, setIsBlur }) =>
       }
     };
 
+    return () => {
+      workerRef.current?.terminate();
+    };
+  }, [setIsBlur]);*/
+  //Updated response handler
+    workerRef.current.onmessage = (event) => {
+      const { score } = event.data;
+
+      const isSuspicious = score > 0.6;
+
+      if (isSuspicious) {
+        if (!blurStartTimeRef.current) {
+          blurStartTimeRef.current = Date.now();
+        } else {
+          const elapsedTime = Date.now() - blurStartTimeRef.current;
+
+          if (elapsedTime >= 2000 && elapsedTime < 2200) {
+            // warning
+          } else if (elapsedTime >= 5000 && elapsedTime < 5200) {
+            // flag
+            blurStartTimeRef.current = null;
+          }
+        }
+        setIsBlur("Yes");
+      } else {
+        blurStartTimeRef.current = null;
+        setIsBlur("No");
+      }
+    };
+
     return () => {
       workerRef.current?.terminate();
     };
   }, [setIsBlur]);
 
+  //Init Mediapipe
+  useEffect(() => {
+    const segmentation = new SelfieSegmentation({
+      locateFile: (file) =>
+        `https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation/${file}`,
+    });
+
+    segmentation.setOptions({
+      modelSelection: 1,
+    });
+
+    segmentation.onResults((results) => {
+      processSegmentation(results);
+    });
+
+    segmentationRef.current = segmentation;
+  }, []);
+
+  //Process segmentation results
+  const processSegmentation = (results: any) => {
+    const video = videoRef.current;
+    if (!video || !workerRef.current) return;
+
+    const width = video.videoWidth;
+    const height = video.videoHeight;
+
+    const canvas = document.createElement("canvas");
+    const ctx = canvas.getContext("2d");
+
+    canvas.width = width;
+    canvas.height = height;
+
+    if (!ctx) return;
+
+    try {
+      // Draw frame
+      ctx.drawImage(video, 0, 0, width, height);
+      const frame = ctx.getImageData(0, 0, width, height);
+
+      // Flicker detection
+      let flick = 0;
+      if (prevFrameRef.current) {
+        for (let i = 0; i < frame.data.length; i += 4) {
+          flick += Math.abs(frame.data[i] - prevFrameRef.current.data[i]);
+        }
+        flick /= frame.data.length / 4;
+      }
+
+      prevFrameRef.current = frame;
+
+      // Get segmentation mask
+      const maskCanvas = document.createElement("canvas");
+      const maskCtx = maskCanvas.getContext("2d");
+
+      maskCanvas.width = width;
+      maskCanvas.height = height;
+
+      maskCtx?.drawImage(results.segmentationMask, 0, 0, width, height);
+      const mask = maskCtx?.getImageData(0, 0, width, height);
+
+      // Send to worker (UPDATED FORMAT)
+      workerRef.current.postMessage({
+        frame,
+        mask,
+        flick,
+      });
+
+    } catch (error) {
+      console.warn("Segmentation failed:", error);
+    }
+  };
+
+  //Frame loop (updated)
+
   useEffect(() => {
     if (!videoRef.current) return;
 
-    const captureFrame = () => {
+    const captureFrame = async() => {
       const video = videoRef.current;
 
       if (!video || video.readyState !== 4 || video.videoWidth === 0 || video.videoHeight === 0) {
         return; // Skip processing if video isn't ready
       }
 
-      const canvas = document.createElement("canvas");
+      /*const canvas = document.createElement("canvas");
       const ctx = canvas.getContext("2d");
 
       // Set canvas dimensions to match video
@@ -64,7 +182,21 @@ const BlurDetection: React.FC<BlurDetectionProps> = ({ videoRef, setIsBlur }) =>
           console.warn("Failed to capture frame:", error);
         }
       }
-    };
+    };*/
+    if (!segmentationRef.current) return;
+    try {
+      // Use MediaPipe instead of manual canvas extraction
+      await segmentationRef.current?.send({ image: video });
+
+    } catch (error) {
+      console.warn("Segmentation send failed:", error);
+    }
+  };
+
+  const interval = setInterval(captureFrame, 500); // ~2 FPS (optimized)
+
+  return () => clearInterval(interval);
+}, [videoRef]);
 
     //   if (video) {
     //     const canvas = document.createElement("canvas");
@@ -79,9 +211,9 @@ const BlurDetection: React.FC<BlurDetectionProps> = ({ videoRef, setIsBlur }) =>
     //   }
     // };
 
-    const interval = setInterval(captureFrame, 500);
-    return () => clearInterval(interval);
-  }, [videoRef]);
+    //const interval = setInterval(captureFrame, 500);
+    //return () => clearInterval(interval);
+  //}, [videoRef]);
 
   return null; // No need to return a video element
 };