feat: Add performance metrics for virtual background TFLite

react/features/stream-effects/virtual-background/JitsiStreamBackgroundEffect.ts

@@ -1,5 +1,7 @@
 import { VIRTUAL_BACKGROUND_TYPE } from '../../virtual-background/constants';
 
+import logger from '../../virtual-background/logger';
+
 import {
     CLEAR_TIMEOUT,
     SET_TIMEOUT,
@@ -37,6 +39,16 @@ export default class JitsiStreamBackgroundEffect {
     _virtualImage: HTMLImageElement;
     _virtualVideo: HTMLVideoElement;
 
+    // Performance metrics used for evaluating TFLite model performance against newer models
+    // Tracks the total time taken for inferences across a batch of frames
+    _inferenceTimeSum: number;
+    // Tracks the total time taken for post processing across a batch of frames
+    _postProcessingTimeSum: number;
+    // Tracks the number of frames processed in the current interval
+    _frameCount: number;
+    // Timestamp (in ms) of the last reported logging event
+    _lastReportTime: number;
+
     /**
      * Represents a modified video MediaStream track.
      *
@@ -62,6 +74,12 @@ export default class JitsiStreamBackgroundEffect {
         this._outputCanvasElement = document.createElement('canvas');
         this._outputCanvasElement.getContext('2d');
         this._inputVideoElement = document.createElement('video');
+
+        // Initialize performance metric tracking variables for the effect
+        this._inferenceTimeSum = 0;
+        this._postProcessingTimeSum = 0;
+        this._frameCount = 0;
+        this._lastReportTime = performance.now();
     }
 
     /**
@@ -164,8 +182,40 @@ export default class JitsiStreamBackgroundEffect {
      */
     _renderMask() {
         this.resizeSource();
+
+        // Track the execution time specifically for background inference (TFLite processing)
+        const inferenceStart = performance.now();
         this.runInference();
+        const inferenceEnd = performance.now();
+        this._inferenceTimeSum += (inferenceEnd - inferenceStart);
+
+        // Track the execution time specifically for post-processing operations (rendering, applying blur, etc.)
+        const postProcessingStart = performance.now();
         this.runPostProcessing();
+        const postProcessingEnd = performance.now();
+        this._postProcessingTimeSum += (postProcessingEnd - postProcessingStart);
+
+        this._frameCount++;
+
+        // Calculate and log the average performance metrics over an interval of 30 frames
+        // This gives PR reviewers and developers concrete benchmarking data when migrating to MediaPipe
+        if (this._frameCount >= 30) {
+            const now = performance.now();
+            const elapsedMs = now - this._lastReportTime;
+
+            // Calculate achieved frames per second explicitly based on the elapsed worker time
+            const fps = (this._frameCount / elapsedMs) * 1000;
+            const avgInference = this._inferenceTimeSum / this._frameCount;
+            const avgPostProcessing = this._postProcessingTimeSum / this._frameCount;
+
+            logger.info(`[Virtual Background] Avg Inference: ${avgInference.toFixed(2)}ms | Avg Post-Processing: ${avgPostProcessing.toFixed(2)}ms | FPS: ${fps.toFixed(1)}`);
+
+            // Reset performance accumulation metrics for the next frame batch
+            this._inferenceTimeSum = 0;
+            this._postProcessingTimeSum = 0;
+            this._frameCount = 0;
+            this._lastReportTime = now;
+        }
 
         this._maskFrameTimerWorker.postMessage({
             id: SET_TIMEOUT,

react/features/virtual-background/components/UploadImageButton.tsx

@@ -98,6 +98,14 @@ function UploadImageButton({
 
         reader.readAsDataURL(imageFile[0]);
         reader.onload = async () => {
+            // resizeImage expects a base64 string. 
+            // readAsDataURL always populates reader.result with a string, 
+            // but the TS FileReader type includes ArrayBuffer | null.
+            // This type guard explicitly satisfies the TypeScript compiler.
+            if (typeof reader.result !== 'string') {
+                return;
+            }
+
             const url = await resizeImage(reader.result);
             const uuId = uuidv4();