Skip to content

Commit ccc17af

Browse files
anonfadedanonfaded
committed
feat: enhance timestamp synchronization and handling for camera switch scenarios
1 parent 5ff29ea commit ccc17af

3 files changed

Lines changed: 205 additions & 62 deletions

File tree

‎app/src/main/java/com/fadcam/opengl/GLRecordingPipeline.java‎

Lines changed: 172 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ public class GLRecordingPipeline {
5959
private Surface pendingPreviewToApply = null;
6060
private Runnable pendingPreviewApplyRunnable = null;
6161
private final Object previewApplyLock = new Object();
62+
private final Object timestampLock = new Object(); // Synchronization lock for timestamp fields
6263
private final String orientation;
6364
private final int sensorOrientation;
6465

@@ -83,50 +84,51 @@ public interface SegmentCallback {
8384
/**
8485
* Gets a synchronized timestamp for audio frames based on the video timeline.
8586
* This ensures audio and video timestamps are properly aligned.
87+
* Accounts for pause durations to maintain sync during camera switch.
8688
*/
8789
private long getSynchronizedAudioTimestamp() {
88-
synchronized (timestampLock) {
89-
if (recordingStartTimeNanos == -1) {
90-
// First call - initialize the recording start time
91-
recordingStartTimeNanos = System.nanoTime();
92-
return 0; // First audio frame starts at 0
93-
}
94-
95-
// Calculate elapsed time since recording started
96-
long elapsedNanos = System.nanoTime() - recordingStartTimeNanos;
97-
return elapsedNanos / 1000L; // Convert to microseconds
98-
}
90+
// Audio timestamp is calculated based on sample count, which naturally
91+
// excludes pause periods (no samples recorded during pause)
92+
// So we don't need to do anything special here
93+
return -1; // Indicates: use the PTS calculated in audio thread
9994
}
10095

10196
private void initializeVideoTimestamp(long cameraTimestampNanos) {
102-
synchronized (timestampLock) {
103-
if (firstVideoTimestampNanos == -1) {
104-
firstVideoTimestampNanos = cameraTimestampNanos;
105-
if (recordingStartTimeNanos == -1) {
106-
recordingStartTimeNanos = System.nanoTime();
107-
}
108-
// Log.d(TAG, "Video timestamp initialized: camera=" + cameraTimestampNanos +
109-
// ", recording_start=" + recordingStartTimeNanos);
110-
}
111-
}
97+
// Not needed anymore - PTS is calculated from frame counter
98+
// Keeping this method for compatibility with existing code
11299
}
113100

114101
/**
115102
* Gets a synchronized timestamp for video frames that aligns with audio.
116-
* This converts camera timestamps to recording timeline.
103+
* Returns -1 during pause to skip frames, preserving the existing timestamp logic.
104+
*
105+
* CRITICAL FIX: Use SYSTEM CLOCK elapsed time (same as audio thread),
106+
* not camera timestamps. This ensures audio and video use the SAME timing base.
117107
*/
118108
public long getSynchronizedVideoTimestamp(long cameraTimestampNanos) {
119-
synchronized (timestampLock) {
120-
if (firstVideoTimestampNanos == -1 || recordingStartTimeNanos == -1) {
121-
// Initialize if not done yet
122-
initializeVideoTimestamp(cameraTimestampNanos);
123-
return 0; // First video frame starts at 0
109+
// If paused, return -1 to signal frame skip in renderToEncoder
110+
if (isPaused) {
111+
return -1; // Skip this frame
112+
}
113+
114+
// Use System.nanoTime() reference (same as audio thread for sync)
115+
// Initialize recording start time on first frame
116+
if (recordingStartSystemTimeNanos == -1) {
117+
synchronized (timestampLock) {
118+
if (recordingStartSystemTimeNanos == -1) {
119+
recordingStartSystemTimeNanos = System.nanoTime();
120+
Log.d(TAG, "[VIDEO_TIMESTAMP] Recording system time reference initialized: " +
121+
recordingStartSystemTimeNanos + " nanos");
122+
}
124123
}
125-
126-
// Calculate offset from first video frame
127-
long videoOffsetNanos = cameraTimestampNanos - firstVideoTimestampNanos;
128-
return videoOffsetNanos / 1000L; // Convert to microseconds
129124
}
125+
126+
// Calculate relative timestamp from system time reference (same as audio!)
127+
// This ensures both audio and video use the exact same timing base
128+
long elapsedNanos = System.nanoTime() - recordingStartSystemTimeNanos - totalPauseDurationNanos;
129+
long ptsUs = Math.max(0, elapsedNanos / 1000L); // Convert to microseconds, ensure non-negative
130+
131+
return ptsUs;
130132
}
131133

132134
private long maxFileSizeBytes = Long.MAX_VALUE;
@@ -188,9 +190,16 @@ public long getSynchronizedVideoTimestamp(long cameraTimestampNanos) {
188190
private Float locationLongitude = null;
189191

190192
// Timestamp synchronization fields
191-
private long recordingStartTimeNanos = -1;
192-
private long firstVideoTimestampNanos = -1;
193-
private final Object timestampLock = new Object();
193+
private long recordingStartTimeNanos = -1; // System.nanoTime() when recording starts (VIDEO reference)
194+
private long recordingStartSystemTimeNanos = -1; // System.nanoTime() for audio thread reference (same as video!)
195+
private volatile boolean isPaused = false; // Track pause state for audio thread
196+
197+
// Pause/resume tracking fields
198+
private long pauseStartTimeNanos = -1; // System.nanoTime() when pause starts
199+
private long totalPauseDurationNanos = 0; // Accumulated pause duration
200+
private long lastVideoPtsBeforePauseUs = 0; // Last video PTS before pause
201+
private long lastAudioPtsBeforePauseUs = 0; // Last audio PTS before pause
202+
private boolean isCameraSwitchPause = false; // Flag to indicate if pause is due to camera switch
194203

195204
// scheduler-----------
196205
// Update watermark on a low-frequency handler to avoid per-frame overhead and
@@ -454,6 +463,11 @@ public void startRecording() {
454463
com.fadcam.Log.d(TAG, "Starting recording pipeline");
455464
} catch (Throwable ignore) {
456465
}
466+
467+
// Reset timestamp tracking at recording start
468+
recordingStartTimeNanos = -1; // Will be initialized on first VIDEO frame
469+
recordingStartSystemTimeNanos = System.nanoTime(); // Initialize for AUDIO thread reference NOW
470+
Log.i(TAG, "[RECORDING_START] Audio/Video timing reference initialized: " + recordingStartSystemTimeNanos);
457471

458472
// Make sure we have a valid renderer and surfaces
459473
if (glRenderer == null || encoderInputSurface == null) {
@@ -1107,7 +1121,7 @@ private void setupMuxer() throws IOException {
11071121
if (segmentNumber == 1) {
11081122
synchronized (timestampLock) {
11091123
recordingStartTimeNanos = -1;
1110-
firstVideoTimestampNanos = -1;
1124+
recordingStartSystemTimeNanos = -1;
11111125
}
11121126
}
11131127

@@ -1331,9 +1345,11 @@ private void drainEncoder() {
13311345

13321346
videoSamplesWritten++;
13331347
lastVideoPts = bufferInfo.presentationTimeUs;
1334-
if (videoSamplesWritten % 60 == 0) {
1335-
Log.d(TAG, String.format("VIDEO: #%d, %.1fs, %db",
1336-
videoSamplesWritten, bufferInfo.presentationTimeUs / 1000000.0, bufferInfo.size));
1348+
// Log every frame for debugging (can reduce frequency later)
1349+
if (videoSamplesWritten % 30 == 0) {
1350+
Log.i(TAG, String.format("[VIDEO_WRITE] Sample #%d, PTS=%.3fs (%dus), size=%db, keyframe=%s",
1351+
videoSamplesWritten, bufferInfo.presentationTimeUs / 1000000.0,
1352+
bufferInfo.presentationTimeUs, bufferInfo.size, isKeyframe));
13371353
}
13381354

13391355
segmentBytesWritten += bufferInfo.size;
@@ -1740,7 +1756,7 @@ public void stopRecording() {
17401756
isRecording = false;
17411757
// Clear retry/time bases
17421758
recordingStartTimeNanos = -1;
1743-
firstVideoTimestampNanos = -1;
1759+
recordingStartSystemTimeNanos = -1;
17441760
}
17451761

17461762
/**
@@ -1771,17 +1787,36 @@ public Surface getCameraInputSurface() {
17711787
}
17721788

17731789
/**
1774-
* Pauses the recording pipeline (no-op, for API compatibility).
1790+
* Pauses the recording pipeline with proper timestamp tracking.
1791+
* During pause, we record the last known PTS values to maintain
1792+
* timeline continuity when resuming (especially important for camera switch).
17751793
*/
17761794
public void pauseRecording() {
17771795
if (!isRecording || isStopped) {
17781796
Log.w(TAG, "Cannot pause recording - recording is not active");
17791797
return;
17801798
}
17811799

1782-
Log.d(TAG, "Pausing recording");
1800+
Log.i(TAG, "========== PAUSE RECORDING ===========");
1801+
1802+
synchronized (timestampLock) {
1803+
// Record pause start time for duration tracking
1804+
pauseStartTimeNanos = System.nanoTime();
1805+
isPaused = true;
1806+
1807+
// Save last known PTS values before pause
1808+
lastVideoPtsBeforePauseUs = lastVideoPts;
1809+
lastAudioPtsBeforePauseUs = lastAudioPts;
1810+
1811+
Log.i(TAG, "[PAUSE] Pause started at " + (pauseStartTimeNanos / 1_000_000L) + "ms");
1812+
Log.i(TAG, "[PAUSE] Last video PTS: " + lastVideoPtsBeforePauseUs + "us (" + (lastVideoPtsBeforePauseUs / 1000.0) + "ms)");
1813+
Log.i(TAG, "[PAUSE] Last audio PTS: " + lastAudioPtsBeforePauseUs + "us (" + (lastAudioPtsBeforePauseUs / 1000.0) + "ms)");
1814+
Log.i(TAG, "[PAUSE] Total pause duration so far: " + (totalPauseDurationNanos / 1_000_000L) + "ms");
1815+
Log.i(TAG, "[PAUSE] Video samples written: " + videoSamplesWritten);
1816+
Log.i(TAG, "[PAUSE] Audio samples written: " + audioSamplesWritten);
1817+
}
17831818

1784-
// Simply set the recording flag to false to stop encoding new frames
1819+
// Set recording flag to false to stop encoding new frames
17851820
isRecording = false;
17861821

17871822
// Pause audio recording if enabled
@@ -1798,6 +1833,20 @@ public void pauseRecording() {
17981833

17991834
Log.d(TAG, "Recording paused successfully");
18001835
}
1836+
1837+
/**
1838+
* Prepares for a camera switch by setting the appropriate flags.
1839+
* Call this before pauseRecording() when switching cameras.
1840+
*/
1841+
public void prepareCameraSwitch() {
1842+
synchronized (timestampLock) {
1843+
isCameraSwitchPause = true;
1844+
Log.i(TAG, "========== PREPARE CAMERA SWITCH ===========");
1845+
Log.i(TAG, "[CAMERA_SWITCH] Prepared - timestamps will be adjusted on resume");
1846+
Log.i(TAG, "[CAMERA_SWITCH] Current video PTS: " + lastVideoPts + "us");
1847+
Log.i(TAG, "[CAMERA_SWITCH] Current audio PTS: " + lastAudioPts + "us");
1848+
}
1849+
}
18011850

18021851
/**
18031852
* Resumes the recording pipeline (no-op, for API compatibility).
@@ -1810,6 +1859,21 @@ public void resumeRecording() {
18101859

18111860
Log.d(TAG, "Resuming recording");
18121861

1862+
Log.i(TAG, "========== RESUME RECORDING ===========");
1863+
1864+
synchronized (timestampLock) {
1865+
// Calculate pause duration for logging
1866+
if (pauseStartTimeNanos > 0) {
1867+
long pauseDuration = System.nanoTime() - pauseStartTimeNanos;
1868+
totalPauseDurationNanos += pauseDuration;
1869+
Log.i(TAG, "[RESUME] This pause duration: " + (pauseDuration / 1_000_000L) + "ms");
1870+
Log.i(TAG, "[RESUME] Total pause duration: " + (totalPauseDurationNanos / 1_000_000L) + "ms");
1871+
}
1872+
1873+
isPaused = false;
1874+
pauseStartTimeNanos = -1;
1875+
}
1876+
18131877
// Resume audio recording if enabled
18141878
if (audioRecordingEnabled && audioRecord != null) {
18151879
try {
@@ -2183,6 +2247,7 @@ private void setupAudio() {
21832247

21842248
/**
21852249
* Starts the audio thread to read PCM and feed the encoder.
2250+
* Handles pause/resume for camera switch scenarios.
21862251
*/
21872252
private void startAudioThread() {
21882253
if (!audioRecordingEnabled || audioThreadRunning)
@@ -2196,13 +2261,35 @@ private void startAudioThread() {
21962261
final int aacFrameSize = 1024 * bytesPerFrame; // 1024 PCM frames per AAC frame
21972262
final int readBufferSize = Math.max(aacFrameSize * 4, 131072); // >= 128 KiB
21982263
byte[] readBuffer = new byte[readBufferSize];
2199-
long audioFramesWritten = 0L; // PCM frames (not bytes)
2200-
long lastPtsUs = 0L;
2264+
long lastPtsUs = 0L;
2265+
22012266
while (audioThreadRunning) {
2267+
// Check if we're paused - wait for resume
2268+
if (isPaused) {
2269+
// During pause, don't read audio - just wait
2270+
try {
2271+
Thread.sleep(10);
2272+
} catch (InterruptedException e) {
2273+
Thread.currentThread().interrupt();
2274+
}
2275+
continue;
2276+
}
2277+
2278+
// Check if audioRecord is actually recording
2279+
if (audioRecord.getRecordingState() != android.media.AudioRecord.RECORDSTATE_RECORDING) {
2280+
// AudioRecord is stopped (during pause), wait a bit
2281+
try {
2282+
Thread.sleep(10);
2283+
} catch (InterruptedException e) {
2284+
Thread.currentThread().interrupt();
2285+
}
2286+
continue;
2287+
}
2288+
22022289
int read = audioRecord.read(readBuffer, 0, readBuffer.length);
22032290
if (read > 0) {
22042291
int offset = 0;
2205-
while (offset < read && audioThreadRunning) {
2292+
while (offset < read && audioThreadRunning && !isPaused) {
22062293
int inputBufferIndex = audioEncoder.dequeueInputBuffer(10000);
22072294
if (inputBufferIndex < 0) {
22082295
// Encoder busy; break and try next loop iteration
@@ -2215,13 +2302,33 @@ private void startAudioThread() {
22152302
codecInput.clear();
22162303
int toCopy = Math.min(codecInput.remaining(), read - offset);
22172304
codecInput.put(readBuffer, offset, toCopy);
2218-
// PTS based on frames written so far (stable, drift-free)
2219-
long ptsUs = (audioFramesWritten * 1_000_000L) / audioSampleRate;
2220-
audioEncoder.queueInputBuffer(inputBufferIndex, 0, toCopy, ptsUs, 0);
2221-
lastPtsUs = ptsUs;
2305+
2306+
// CRITICAL FIX: Audio PTS must sync with video PTS!
2307+
// Both use elapsed time from recordingStartSystemTimeNanos reference point
2308+
// This ensures they use the SAME timing base (system clock, not frame count)
2309+
synchronized (timestampLock) {
2310+
long elapsedNanos = System.nanoTime() - recordingStartSystemTimeNanos - totalPauseDurationNanos;
2311+
long ptsUs = Math.max(0, elapsedNanos / 1000L); // Convert to microseconds
2312+
2313+
// Ensure monotonically increasing PTS
2314+
if (ptsUs <= lastPtsUs && lastPtsUs > 0) {
2315+
ptsUs = lastPtsUs + 1; // Ensure at least 1us increment
2316+
}
2317+
2318+
audioEncoder.queueInputBuffer(inputBufferIndex, 0, toCopy, ptsUs, 0);
2319+
lastPtsUs = ptsUs;
2320+
}
2321+
22222322
// Advance counters
22232323
offset += toCopy;
2224-
audioFramesWritten += (toCopy / bytesPerFrame);
2324+
}
2325+
} else if (read < 0) {
2326+
// Error or AudioRecord stopped
2327+
Log.w(TAG, "AudioRecord.read returned " + read + " - likely paused or error");
2328+
try {
2329+
Thread.sleep(10);
2330+
} catch (InterruptedException e) {
2331+
Thread.currentThread().interrupt();
22252332
}
22262333
}
22272334
}
@@ -2233,14 +2340,16 @@ private void startAudioThread() {
22332340
while (eosRetries > 0 && !eosQueued) {
22342341
int inputBufferIndex = audioEncoder.dequeueInputBuffer(50000); // 50ms timeout per retry
22352342
if (inputBufferIndex >= 0) {
2236-
// Use lastPtsUs from our audio timeline to avoid duration jumps
2237-
long eosPtsUs = (audioFramesWritten * 1_000_000L) / audioSampleRate;
2238-
if (eosPtsUs < lastPtsUs)
2239-
eosPtsUs = lastPtsUs; // monotonic safeguard
2240-
audioEncoder.queueInputBuffer(inputBufferIndex, 0, 0, eosPtsUs,
2241-
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
2242-
Log.d(TAG, "Audio EOS queued at PTS=" + eosPtsUs + "us (" + (eosPtsUs / 1000000.0) + "s)");
2243-
eosQueued = true;
2343+
// Use system time for EOS PTS too, to match monotonic timeline
2344+
synchronized (timestampLock) {
2345+
long eosPtsUs = Math.max(0, (System.nanoTime() - recordingStartSystemTimeNanos - totalPauseDurationNanos) / 1000L);
2346+
if (eosPtsUs < lastPtsUs)
2347+
eosPtsUs = lastPtsUs; // monotonic safeguard
2348+
audioEncoder.queueInputBuffer(inputBufferIndex, 0, 0, eosPtsUs,
2349+
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
2350+
Log.d(TAG, "Audio EOS queued at PTS=" + eosPtsUs + "us (" + (eosPtsUs / 1000000.0) + "s)");
2351+
eosQueued = true;
2352+
}
22442353
} else {
22452354
eosRetries--;
22462355
Log.w(TAG, "Failed to get input buffer for audio EOS, retries left: " + eosRetries);
@@ -2360,9 +2469,11 @@ private void drainAudioEncoder() {
23602469

23612470
audioSamplesWritten++;
23622471
lastAudioPts = bufferInfo.presentationTimeUs;
2363-
if (audioSamplesWritten % 100 == 0) {
2364-
Log.d(TAG, String.format("AUDIO: #%d, %.1fs, %db",
2365-
audioSamplesWritten, bufferInfo.presentationTimeUs / 1000000.0, bufferInfo.size));
2472+
// Log every 50 samples for debugging
2473+
if (audioSamplesWritten % 50 == 0) {
2474+
Log.i(TAG, String.format("[AUDIO_WRITE] Sample #%d, PTS=%.3fs (%dus), size=%db",
2475+
audioSamplesWritten, bufferInfo.presentationTimeUs / 1000000.0,
2476+
bufferInfo.presentationTimeUs, bufferInfo.size));
23662477
}
23672478
// Detect silence pattern (constant 512-byte AAC frames)
23682479
if (bufferInfo.size == 512) {

0 commit comments

Comments
 (0)