Skip to content

Fixup format selection fallbacks #444

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Fixup format selection fallbacks #444

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7b3d629
Fixup format selection fallbacks
RichLogan Jun 4, 2024
0a6d81b
Merge remote-tracking branch 'origin/main' into camera-to-encoder-config
RichLogan Jun 5, 2024
55acb5e
Further
RichLogan Jun 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 91 additions & 24 deletions Decimus/CaptureManager.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ class CaptureManager: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
private let metricsSubmitter: MetricsSubmitter?
private let granularMetrics: Bool
private let warmupTime: TimeInterval = 0.75
private var selectedFormat: [AVCaptureDevice: AVCaptureDevice.Format] = [:]

init(metricsSubmitter: MetricsSubmitter?, granularMetrics: Bool) throws {
guard AVCaptureMultiCamSession.isMultiCamSupported else {
Expand Down Expand Up @@ -147,10 +148,15 @@ class CaptureManager: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
}
}

// Rules for setting a format.
// - The highest resolution requested will be preferred.
// - The frame rate chosen will be the lower of the highest selected resolution's highest frame rate,
// and the requested frame rate.
private func setBestDeviceFormat(device: AVCaptureDevice, config: VideoCodecConfig) throws {
try device.lockForConfiguration()
defer { device.unlockForConfiguration() }

// Get available formats for the given resolution requirements.
let allowableFormats = device.formats.reversed().filter { format in
var supported = format.isMultiCamSupported &&
format.formatDescription.dimensions.width == config.width &&
Expand All @@ -162,55 +168,108 @@ class CaptureManager: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
return supported
}

guard let bestFormat = allowableFormats.first(where: { format in
return format.videoSupportedFrameRateRanges.contains { $0.maxFrameRate == Float64(config.fps) }
}) else {
// Did we get a match? If not, best effort.
guard allowableFormats.count > 0 else {
Self.logger.debug("""
[\(device.localizedName)] Couldn't find an exact format match for: \(config.getResolutionString()).
Staying with \(device.activeFormat)
""")
return
}

// TODO: What we want to do here is pick the best matching format with a >= FPS.
// TODO: Once we've picked that, we can set the durations to match our actual FPS need.
// TODO: For example, if we want 30fps but we can get a 60fps format, pick the higher, and downscale the duration.

// Is there a format that matches the required frame rate?
let bestFormat: AVCaptureDevice.Format
if let frameRateMatched = allowableFormats.first(where: {
$0.videoSupportedFrameRateRanges.contains { $0.maxFrameRate >= Float64(config.fps) }
}) {
// This matches our frame rate target.
bestFormat = frameRateMatched
} else {
// Otherwise, pick the best <= fame rate.
Self.logger.warning("[\(device.localizedName)] Camera does not support requested frame rate: \(config.fps)")
guard let backupFormat = allowableFormats.first(where: {
$0.videoSupportedFrameRateRanges.contains { $0.maxFrameRate <= Float64(config.fps) }
}) else {
Self.logger.error(
"[\(device.localizedName)] Couldn't select a backup format. Staying with \(device.activeFormat)")
return
}
bestFormat = backupFormat
}

// Now that we have a format, should we switch to it?
let shouldSwitch: Bool
if let existingFormat = self.selectedFormat[device] {
// Only switch to upgrades.
let resolutionUpgrade = {
bestFormat.formatDescription.dimensions.width >= existingFormat.formatDescription.dimensions.width &&
bestFormat.formatDescription.dimensions.height >= existingFormat.formatDescription.dimensions.height
}()
let frameRateUpgrade = {
let newFrameRate = bestFormat.videoSupportedFrameRateRanges.reduce(into: 0) {
$0 = $1.maxFrameRate > $0 ? $1.maxFrameRate : $0
}
let existingFrameRate = existingFormat.videoSupportedFrameRateRanges.reduce(into: 0) {
$0 = $1.maxFrameRate > $0 ? $1.maxFrameRate : $0
}
return resolutionUpgrade && newFrameRate > existingFrameRate
}()
shouldSwitch = resolutionUpgrade || frameRateUpgrade
} else {
// Should always switch if we haven't before.
shouldSwitch = true
}

guard shouldSwitch else {
Self.logger.debug(
"[\(device.localizedName)] Not switching format from \(device.activeFormat) to \(bestFormat)")
return
}

// Actually switch the camera format.
self.session.beginConfiguration()
device.activeFormat = bestFormat
self.selectedFormat[device] = bestFormat
Self.logger.info(
"[\(device.localizedName)] Setting format: \(device.activeFormat) from \(config.getResolutionString())")
if device.activeFormat.supportedColorSpaces.contains(.sRGB) {
device.activeColorSpace = .sRGB
}
device.activeVideoMinFrameDuration = .init(value: 1, timescale: CMTimeScale(config.fps))
device.activeVideoMaxFrameDuration = .init(value: 1, timescale: CMTimeScale(config.fps))
self.session.commitConfiguration()
}

private func addCamera(listener: FrameListener) throws {
// Device is already setup, add this delegate.
let device = listener.device

if var cameraFrameListeners = self.multiVideoDelegate[device] {
let ranges = device.activeFormat.videoSupportedFrameRateRanges
guard let maxFramerateRange = ranges.max(by: { $0.maxFrameRate > $1.maxFrameRate }) else {
throw "No framerate set"
}

if let config = listener.codec {
if maxFramerateRange.maxFrameRate < Float64(config.fps) {
try setBestDeviceFormat(device: device, config: config)
}
}
// Always probe to see if there's a better format.
if let config = listener.codec {
try setBestDeviceFormat(device: device, config: config)
}

// If device is already setup, just add this delegate.
if var cameraFrameListeners = self.multiVideoDelegate[device] {
cameraFrameListeners.append(listener)
self.multiVideoDelegate[device] = cameraFrameListeners
return
}

// Setup device.
if let config = listener.codec {
try setBestDeviceFormat(device: device, config: config)
}

// Prepare IO.
let input: AVCaptureDeviceInput = try .init(device: device)
let output: AVCaptureVideoDataOutput = .init()
let lossless420 = kCVPixelFormatType_Lossless_420YpCbCr8BiPlanarVideoRange
let lossy420 = kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarVideoRange
output.videoSettings = [:]
let pre = output.availableVideoPixelFormatTypes
if output.availableVideoPixelFormatTypes.contains(where: {
$0 == lossless420
$0 == lossy420
}) {
output.videoSettings[kCVPixelBufferPixelFormatTypeKey as String] = lossless420
output.videoSettings[kCVPixelBufferPixelFormatTypeKey as String] = lossy420
Self.logger.debug("[\(device.localizedName)] Using compressed pixel format")
}
output.videoSettings[AVVideoColorPropertiesKey] = [
AVVideoColorPrimariesKey: AVVideoColorPrimaries_ITU_R_709_2,
Expand Down Expand Up @@ -244,7 +303,7 @@ class CaptureManager: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
Self.logger.info("Adding capture device: \(listener.device.localizedName)")

#if !os(tvOS)
if listener.device.deviceType == .builtInMicrophone {
if listener.device.deviceType == .microphone {
throw CaptureManagerError.noAudio
}
#endif
Expand All @@ -267,6 +326,7 @@ class CaptureManager: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {

guard deviceListeners.count == 0 else {
// There are other listeners still, so update and stop.
// TODO: Theoretically we could reevaluate the format here.
self.multiVideoDelegate[device] = deviceListeners
return
}
Expand All @@ -285,6 +345,7 @@ class CaptureManager: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
outputs.removeValue(forKey: output.key)
}
session.commitConfiguration()
self.selectedFormat.removeValue(forKey: device)
Self.logger.info("Removing input for \(device.localizedName)")
}

Expand Down Expand Up @@ -378,3 +439,9 @@ extension UIDeviceOrientation {
}
}
#endif

extension VideoCodecConfig {
func getResolutionString() -> String {
"\(self.width)x\(self.height)@\(self.fps)"
}
}