Skip to content

Commit 3be4c56

Browse files
committed
feat: add multi audio track detection for imported recordings #88
1 parent be018e9 commit 3be4c56

3 files changed

Lines changed: 344 additions & 1 deletion

File tree

Backend/Media/FFmpegService.cs

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -292,6 +292,95 @@ public static TimeSpan ExtractDuration(string ffmpegOutput)
292292
return TimeSpan.Zero;
293293
}
294294

295+
private static readonly Regex _streamHeaderRegex = new(
296+
@"^\s*Stream #\d+:\d+[^\n]*?:\s*(\w+):",
297+
RegexOptions.Compiled);
298+
299+
private static readonly Regex _metadataTagRegex = new(
300+
@"^\s+(\w+)\s*:\s*(.+?)\s*$",
301+
RegexOptions.Compiled);
302+
303+
private static readonly HashSet<string> _genericHandlerNames = new(StringComparer.OrdinalIgnoreCase)
304+
{
305+
"SoundHandler",
306+
"OBS Audio Handler"
307+
};
308+
309+
/// <summary>
310+
/// Extracts per-track audio names from the stderr output of `ffmpeg -i`.
311+
/// Used as a fallback for non-OBS MP4s that carry standard `title` or
312+
/// non-generic `handler_name` tags. Entries are null for tracks with
313+
/// no usable tag, so callers can decide how to label them. Returns
314+
/// null if fewer than two audio streams are present.
315+
/// </summary>
316+
public static List<string?>? ExtractAudioTrackNames(string ffmpegOutput)
317+
{
318+
if (string.IsNullOrEmpty(ffmpegOutput))
319+
{
320+
return null;
321+
}
322+
323+
var tracks = new List<string?>();
324+
bool inAudioStream = false;
325+
string? currentTitle = null;
326+
string? currentHandler = null;
327+
328+
void Flush()
329+
{
330+
if (!inAudioStream) return;
331+
string? resolved = null;
332+
if (!string.IsNullOrWhiteSpace(currentTitle))
333+
{
334+
resolved = currentTitle;
335+
}
336+
else if (!string.IsNullOrWhiteSpace(currentHandler) && !_genericHandlerNames.Contains(currentHandler!))
337+
{
338+
resolved = currentHandler;
339+
}
340+
tracks.Add(resolved);
341+
inAudioStream = false;
342+
currentTitle = null;
343+
currentHandler = null;
344+
}
345+
346+
foreach (var rawLine in ffmpegOutput.Split('\n'))
347+
{
348+
var line = rawLine.TrimEnd('\r');
349+
350+
var streamMatch = _streamHeaderRegex.Match(line);
351+
if (streamMatch.Success)
352+
{
353+
Flush();
354+
string streamType = streamMatch.Groups[1].Value;
355+
if (streamType.Equals("Audio", StringComparison.OrdinalIgnoreCase))
356+
{
357+
inAudioStream = true;
358+
}
359+
continue;
360+
}
361+
362+
if (!inAudioStream) continue;
363+
364+
var tagMatch = _metadataTagRegex.Match(line);
365+
if (!tagMatch.Success) continue;
366+
367+
string key = tagMatch.Groups[1].Value;
368+
string value = tagMatch.Groups[2].Value;
369+
if (key.Equals("title", StringComparison.OrdinalIgnoreCase))
370+
{
371+
currentTitle = value;
372+
}
373+
else if (key.Equals("handler_name", StringComparison.OrdinalIgnoreCase))
374+
{
375+
currentHandler = value;
376+
}
377+
}
378+
379+
Flush();
380+
381+
return tracks.Count >= 2 ? tracks : null;
382+
}
383+
295384
/// <summary>
296385
/// Gets video duration from a file
297386
/// </summary>

Backend/Media/ImportService.cs

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,52 @@ public static async Task ExecuteImport(string[] selectedFiles, Content.ContentTy
201201
// Extract recording date from file
202202
DateTime recordingDate = File.GetCreationTime(sourceFile);
203203

204+
// Probe audio track layout so the multi-track player UI
205+
// activates for imported recordings that carry more than
206+
// one audio stream. Primary probe: walk the MP4 box tree
207+
// directly (catches OBS's custom `trak/udta/name` atoms,
208+
// which ffmpeg's mov demuxer doesn't surface). Fallback
209+
// probe: parse `ffmpeg -i` stderr for standard `title` /
210+
// non-generic `handler_name` tags on tracks the MP4 box
211+
// walker didn't find a name for.
212+
List<string>? audioTrackNames = null;
213+
var rawNames = await Mp4BoxReader.ReadAudioTrackNamesAsync(sourceFile);
214+
if (rawNames != null)
215+
{
216+
bool anyUnnamed = rawNames.Exists(n => string.IsNullOrWhiteSpace(n));
217+
if (anyUnnamed)
218+
{
219+
try
220+
{
221+
string probeOutput = await FFmpegService.GetMetadata(sourceFile);
222+
var ffmpegNames = FFmpegService.ExtractAudioTrackNames(probeOutput);
223+
if (ffmpegNames != null)
224+
{
225+
int common = Math.Min(ffmpegNames.Count, rawNames.Count);
226+
for (int t = 0; t < common; t++)
227+
{
228+
if (string.IsNullOrWhiteSpace(rawNames[t]) && !string.IsNullOrWhiteSpace(ffmpegNames[t]))
229+
{
230+
rawNames[t] = ffmpegNames[t];
231+
}
232+
}
233+
}
234+
}
235+
catch (Exception probeEx)
236+
{
237+
Log.Warning($"FFmpeg track-name fallback failed for {originalFileName}: {probeEx.Message}");
238+
}
239+
}
240+
241+
audioTrackNames = new List<string>(rawNames.Count);
242+
for (int t = 0; t < rawNames.Count; t++)
243+
{
244+
string? name = rawNames[t];
245+
audioTrackNames.Add(string.IsNullOrWhiteSpace(name) ? $"Track {t + 1}" : name!);
246+
}
247+
Log.Information($"Detected {audioTrackNames.Count} audio tracks in {originalFileName}: {string.Join(", ", audioTrackNames)}");
248+
}
249+
204250
// Send progress after file copy
205251
try
206252
{
@@ -220,7 +266,7 @@ public static async Task ExecuteImport(string[] selectedFiles, Content.ContentTy
220266
}
221267

222268
// Create metadata file with detected game name and date
223-
await ContentService.CreateMetadataFile(targetFilePath, contentType, "Unknown", null, originalFileName.Replace("_", " "), recordingDate != DateTime.MinValue ? recordingDate : null, isImported: true);
269+
await ContentService.CreateMetadataFile(targetFilePath, contentType, "Unknown", null, originalFileName.Replace("_", " "), recordingDate != DateTime.MinValue ? recordingDate : null, isImported: true, audioTrackNames: audioTrackNames);
224270

225271
// Ensure file is fully written to disk/network before thumbnail generation
226272
await GeneralUtils.EnsureFileReady(targetFilePath);

Backend/Media/Mp4BoxReader.cs

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
using System.Text;
2+
using Serilog;
3+
4+
namespace Segra.Backend.Media
5+
{
6+
/// <summary>
7+
/// Minimal ISO-BMFF (MP4) box reader used to extract per-track audio names
8+
/// from Segra/OBS recordings. OBS writes each audio encoder's name into a
9+
/// custom `trak/udta/name` atom that FFmpeg's mov demuxer does not surface,
10+
/// so we walk the box tree ourselves.
11+
/// </summary>
12+
internal static class Mp4BoxReader
13+
{
14+
/// <summary>
15+
/// Returns the raw per-track names of all audio tracks in the file, in
16+
/// container order. An entry is null when the track has no `udta/name`
17+
/// atom, which lets callers fall back to other probes (e.g. ffmpeg's
18+
/// standard title/handler_name tags) before labeling. Returns null
19+
/// when the file isn't parsable as ISO-BMFF, or when it contains fewer
20+
/// than two audio tracks.
21+
/// </summary>
22+
public static async Task<List<string?>?> ReadAudioTrackNamesAsync(string filePath)
23+
{
24+
try
25+
{
26+
await using var fs = new FileStream(
27+
filePath,
28+
FileMode.Open,
29+
FileAccess.Read,
30+
FileShare.Read,
31+
bufferSize: 65536,
32+
useAsync: true);
33+
34+
byte[]? moov = await ReadTopLevelBoxPayloadAsync(fs, "moov");
35+
if (moov == null) return null;
36+
37+
var rawNames = new List<string?>();
38+
ParseMoov(moov, rawNames);
39+
40+
return rawNames.Count >= 2 ? rawNames : null;
41+
}
42+
catch (Exception ex)
43+
{
44+
Log.Warning($"Mp4BoxReader failed for {filePath}: {ex.Message}");
45+
return null;
46+
}
47+
}
48+
49+
// Scans top-level boxes and returns the payload bytes of the first box
50+
// matching `fourCc`, or null if not found. Handles both 32-bit and
51+
// 64-bit box sizes and the "extends to end-of-file" sentinel.
52+
private static async Task<byte[]?> ReadTopLevelBoxPayloadAsync(FileStream fs, string fourCc)
53+
{
54+
long fileSize = fs.Length;
55+
long pos = 0;
56+
byte[] header = new byte[16];
57+
58+
while (pos + 8 <= fileSize)
59+
{
60+
fs.Position = pos;
61+
await fs.ReadExactlyAsync(header.AsMemory(0, 8));
62+
63+
uint boxSize32 = ReadUInt32BE(header, 0);
64+
string type = Encoding.ASCII.GetString(header, 4, 4);
65+
66+
long payloadStart;
67+
long totalBoxSize;
68+
if (boxSize32 == 1)
69+
{
70+
await fs.ReadExactlyAsync(header.AsMemory(8, 8));
71+
totalBoxSize = (long)ReadUInt64BE(header, 8);
72+
payloadStart = pos + 16;
73+
}
74+
else if (boxSize32 == 0)
75+
{
76+
totalBoxSize = fileSize - pos;
77+
payloadStart = pos + 8;
78+
}
79+
else
80+
{
81+
totalBoxSize = boxSize32;
82+
payloadStart = pos + 8;
83+
}
84+
85+
if (totalBoxSize < 8 || pos + totalBoxSize > fileSize) return null;
86+
87+
if (type == fourCc)
88+
{
89+
long payloadSize = (pos + totalBoxSize) - payloadStart;
90+
if (payloadSize < 0 || payloadSize > int.MaxValue) return null;
91+
byte[] payload = new byte[payloadSize];
92+
fs.Position = payloadStart;
93+
await fs.ReadExactlyAsync(payload);
94+
return payload;
95+
}
96+
97+
pos += totalBoxSize;
98+
}
99+
100+
return null;
101+
}
102+
103+
// Walks `trak` boxes inside the moov payload and, for each audio trak,
104+
// appends its `udta/name` content (or null if absent) in container order.
105+
private static void ParseMoov(byte[] moov, List<string?> audioTrackNames)
106+
{
107+
foreach (var trak in EnumerateBoxes(moov, 0, moov.Length))
108+
{
109+
if (trak.Type != "trak") continue;
110+
111+
bool isAudio = false;
112+
string? trackName = null;
113+
114+
foreach (var child in EnumerateBoxes(moov, trak.PayloadStart, trak.PayloadLength))
115+
{
116+
if (child.Type == "mdia")
117+
{
118+
foreach (var mdiaChild in EnumerateBoxes(moov, child.PayloadStart, child.PayloadLength))
119+
{
120+
if (mdiaChild.Type != "hdlr") continue;
121+
// hdlr: fullbox(4) + pre_defined(4) + handler_type(4) + ...
122+
if (mdiaChild.PayloadLength >= 12)
123+
{
124+
string handlerType = Encoding.ASCII.GetString(moov, mdiaChild.PayloadStart + 8, 4);
125+
isAudio = handlerType == "soun";
126+
}
127+
break;
128+
}
129+
}
130+
else if (child.Type == "udta")
131+
{
132+
foreach (var udtaChild in EnumerateBoxes(moov, child.PayloadStart, child.PayloadLength))
133+
{
134+
if (udtaChild.Type != "name") continue;
135+
int len = udtaChild.PayloadLength;
136+
while (len > 0 && moov[udtaChild.PayloadStart + len - 1] == 0) len--;
137+
if (len > 0)
138+
{
139+
trackName = Encoding.UTF8.GetString(moov, udtaChild.PayloadStart, len);
140+
}
141+
break;
142+
}
143+
}
144+
}
145+
146+
if (isAudio)
147+
{
148+
audioTrackNames.Add(trackName);
149+
}
150+
}
151+
}
152+
153+
// Iterates direct children of a box region, yielding each child's
154+
// four-cc type and payload span.
155+
private static IEnumerable<BoxSpan> EnumerateBoxes(byte[] data, int start, int length)
156+
{
157+
int pos = start;
158+
int end = start + length;
159+
while (pos + 8 <= end)
160+
{
161+
uint boxSize32 = ReadUInt32BE(data, pos);
162+
string type = Encoding.ASCII.GetString(data, pos + 4, 4);
163+
164+
int payloadStart;
165+
int totalSize;
166+
if (boxSize32 == 1)
167+
{
168+
if (pos + 16 > end) yield break;
169+
ulong large = ReadUInt64BE(data, pos + 8);
170+
if (large > int.MaxValue) yield break;
171+
totalSize = (int)large;
172+
payloadStart = pos + 16;
173+
}
174+
else if (boxSize32 == 0)
175+
{
176+
totalSize = end - pos;
177+
payloadStart = pos + 8;
178+
}
179+
else
180+
{
181+
totalSize = (int)boxSize32;
182+
payloadStart = pos + 8;
183+
}
184+
185+
if (totalSize < 8 || pos + totalSize > end) yield break;
186+
187+
int payloadLength = (pos + totalSize) - payloadStart;
188+
yield return new BoxSpan(type, payloadStart, payloadLength);
189+
pos += totalSize;
190+
}
191+
}
192+
193+
private readonly record struct BoxSpan(string Type, int PayloadStart, int PayloadLength);
194+
195+
private static uint ReadUInt32BE(byte[] buf, int offset)
196+
{
197+
return ((uint)buf[offset] << 24)
198+
| ((uint)buf[offset + 1] << 16)
199+
| ((uint)buf[offset + 2] << 8)
200+
| buf[offset + 3];
201+
}
202+
203+
private static ulong ReadUInt64BE(byte[] buf, int offset)
204+
{
205+
return ((ulong)ReadUInt32BE(buf, offset) << 32) | ReadUInt32BE(buf, offset + 4);
206+
}
207+
}
208+
}

0 commit comments

Comments
 (0)