Implement Subject Detection for Intelligent Reframing

[sdntsng]

Problem Statement

Our current reframing system in src/routes/reframe.js uses simple center-crop logic that doesn't account for video content. This results in:

• Important subjects (faces, people, objects) being cut off or poorly framed
• Suboptimal social media content that lacks visual impact
• Static reframing that doesn't adapt to content changes throughout the video
• Poor user experience compared to professional tools like OpusClip or AutoFlip

Current Implementation

File: backend/src/routes/reframe.js:45-120

Current reframing logic:

// Simple center crop without content awareness
const cropFilter = `crop=${targetWidth}:${targetHeight}:(iw-ow)/2:(ih-oh)/2`;

Issues:

• No object detection or subject tracking
• Fixed center positioning regardless of content
• Cannot identify important visual elements
• No intelligence about what should remain in frame

Research Insights

Google AutoFlip Approach

• Uses MediaPipe for object detection
• Prioritizes faces, people, text overlays, and logos
• Implements weighted importance scoring
• Three reframing strategies: stationary, panning, tracking

Industry Best Practices

• Face detection as highest priority
• Person detection for full-body shots
• Text overlay preservation
• Logo and branding element retention
• Motion tracking for moving subjects

Proposed Solution

Phase 1: Add MediaPipe Integration

Implement object detection pipeline using MediaPipe or TensorFlow.js:

// New file: src/utils/objectDetection.js
const tf = require('@tensorflow/tfjs-node');
const faceapi = require('face-api.js');

class SubjectDetector {
  constructor() {
    this.faceDetectionModel = null;
    this.objectDetectionModel = null;
  }
  
  async initialize() {
    // Load pre-trained models
    await faceapi.nets.ssdMobilenetv1.loadFromDisk('./models');
    this.faceDetectionModel = faceapi.nets.ssdMobilenetv1;
    
    // Load COCO object detection model
    this.objectDetectionModel = await tf.loadGraphModel('./models/coco-ssd');
  }
  
  async detectSubjects(videoPath, timePoints = []) {
    const detections = [];
    
    for (const timePoint of timePoints) {
      const frame = await this.extractFrame(videoPath, timePoint);
      
      // Detect faces
      const faces = await this.detectFaces(frame);
      
      // Detect objects (person, car, etc.)
      const objects = await this.detectObjects(frame);
      
      detections.push({
        time: timePoint,
        faces: faces,
        objects: objects,
        importance: this.calculateImportance(faces, objects)
      });
    }
    
    return detections;
  }
  
  calculateImportance(faces, objects) {
    const weights = {
      face: 1.0,      // Highest priority
      person: 0.8,    // High priority
      text: 0.6,      // Medium priority
      object: 0.3     // Lower priority
    };
    
    let totalImportance = 0;
    
    // Weight faces heavily
    faces.forEach(face => {
      totalImportance += weights.face * face.confidence;
    });
    
    // Weight detected persons
    objects.filter(obj => obj.class === 'person').forEach(person => {
      totalImportance += weights.person * person.confidence;
    });
    
    return totalImportance;
  }
}

Phase 2: Intelligent Crop Calculation

Update reframe logic to use subject detection:

// Enhanced reframe.js
const SubjectDetector = require('../utils/objectDetection');

router.post('/generate', async (req, res) => {
  try {
    const { transcriptId, aspectRatio, startTime, endTime } = req.body;
    
    // Initialize subject detector
    const detector = new SubjectDetector();
    await detector.initialize();
    
    // Sample video at multiple points for subject analysis
    const samplePoints = generateSamplePoints(startTime, endTime, 5);
    const subjectData = await detector.detectSubjects(inputPath, samplePoints);
    
    // Calculate optimal crop region based on subjects
    const cropRegion = calculateOptimalCrop(subjectData, aspectRatio);
    
    // Generate smart crop filter
    const smartCropFilter = buildSmartCropFilter(cropRegion, aspectRatio);
    
    // Apply FFmpeg with intelligent cropping
    ffmpeg(inputPath)
      .videoFilters([smartCropFilter])
      .output(outputPath)
      .run();
      
  } catch (error) {
    console.error('Smart reframing failed:', error);
    // Fallback to center crop
    const fallbackFilter = `crop=${targetWidth}:${targetHeight}:(iw-ow)/2:(ih-oh)/2`;
  }
});

function calculateOptimalCrop(subjectData, aspectRatio) {
  const subjects = [];
  
  // Collect all important subjects across timeframes
  subjectData.forEach(frame => {
    frame.faces.forEach(face => {
      subjects.push({
        x: face.box.x,
        y: face.box.y,
        width: face.box.width,
        height: face.box.height,
        importance: 1.0,
        type: 'face'
      });
    });
    
    frame.objects.filter(obj => obj.class === 'person').forEach(person => {
      subjects.push({
        x: person.bbox[0],
        y: person.bbox[1],
        width: person.bbox[2],
        height: person.bbox[3],
        importance: 0.8,
        type: 'person'
      });
    });
  });
  
  if (subjects.length === 0) {
    // No subjects detected, use center crop
    return { strategy: 'center', x: 0, y: 0 };
  }
  
  // Calculate weighted center of important subjects
  const weightedCenter = calculateWeightedCenter(subjects);
  
  // Determine crop strategy based on subject distribution
  const strategy = determineCropStrategy(subjects, aspectRatio);
  
  return {
    strategy: strategy,
    centerX: weightedCenter.x,
    centerY: weightedCenter.y,
    subjects: subjects
  };
}

function buildSmartCropFilter(cropRegion, aspectRatio) {
  const { width: targetWidth, height: targetHeight } = getAspectRatioDimensions(aspectRatio);
  
  switch (cropRegion.strategy) {
    case 'face-focused':
      // Center on detected faces
      return `crop=${targetWidth}:${targetHeight}:${cropRegion.centerX - targetWidth/2}:${cropRegion.centerY - targetHeight/2}`;
      
    case 'person-tracking':
      // Follow person movement (requires temporal analysis)
      return buildTrackingFilter(cropRegion, targetWidth, targetHeight);
      
    case 'center':
    default:
      // Fallback to center crop
      return `crop=${targetWidth}:${targetHeight}:(iw-ow)/2:(ih-oh)/2`;
  }
}

Technical Implementation

Dependencies to Add

{
  "@tensorflow/tfjs-node": "^4.0.0",
  "face-api.js": "^0.22.2",
  "opencv4nodejs": "^5.6.0"
}

Model Integration

1. Face Detection - Use face-api.js with SSD MobileNet
2. Object Detection - TensorFlow.js COCO-SSD model
3. Frame Extraction - FFmpeg frame extraction utilities

Files to Modify

1. backend/src/routes/reframe.js

   • Add subject detection integration
   • Implement smart crop calculation
   • Add fallback mechanisms

2. backend/src/utils/objectDetection.js (new file)

   • Subject detection class
   • Model loading and inference
   • Importance scoring algorithms

3. backend/package.json

   • Add AI/ML dependencies
   • Update build scripts if needed

Performance Considerations

Optimization Strategies

• Frame Sampling - Analyze only key frames, not every frame
• Model Caching - Load models once, reuse across requests
• Async Processing - Run detection in parallel with other operations
• Graceful Degradation - Always fallback to center crop if detection fails

Resource Management

// Efficient frame sampling
function generateSamplePoints(startTime, endTime, sampleCount = 5) {
  const duration = endTime - startTime;
  const interval = duration / (sampleCount - 1);
  
  return Array.from({ length: sampleCount }, (_, i) => 
    startTime + (i * interval)
  );
}

// Memory cleanup
process.on('exit', () => {
  if (detector) {
    detector.cleanup();
  }
});

Testing Strategy

Accuracy Testing

• Face detection accuracy on various video types
• Person detection in different scenarios
• Crop quality compared to manual selection
• Performance with different video resolutions

Edge Cases

• Videos with no detectable subjects
• Multiple people in frame
• Fast-moving subjects
• Poor lighting conditions
• Animated/cartoon content

Success Metrics

• Subject detection accuracy >85%
• Improved crop quality vs. center crop (user testing)
• Processing time increase <30%
• Fallback mechanism reliability 100%
• Memory usage within acceptable limits
• Integration with existing reframe workflow seamless

Implementation Priority

Effort: High (7-10 days)
Priority: Medium
Risk: Medium (new AI/ML dependencies)
Dependencies: None (builds on existing reframe system)

This enhancement transforms our basic reframing into intelligent, content-aware video adaptation while maintaining compatibility with our existing FFmpeg-based pipeline.

[samyaksgupta]

imporved implementation able to detect faces much clearly in 1:1 ratio video
https://storage.googleapis.com/vinci-dev/clips/reframed/podcast_instagram_square_reframed.mp4

next task to improve 9:16 ratio videos
current improved output after doing some changes

need to make changes so that video covers the entire face inside the 9:16 ratio