fix(nests): stamp Opus frames at frame-index × frame-duration, not wall clock
The legacy-container timestamp on each audio frame was previously
captured at send-time via `frameStartMark.elapsedNow()`. That mark
advances on wall clock, but `current.send(...)` is a suspending call
that backs off on transport backpressure — so a frame captured at T
gets stamped at T+δ once the previous send drains, and the watcher's
WebCodecs AudioDecoder schedules playback at the wrong instant. Audible
as a slowly-growing offset between speaker and listener over the life
of a session that's seen any meaningful send-side queueing.
Replace with a frame-index counter pinned to the codec grid:
`timestampUs = nextFrameIndex * AudioFormat.FRAME_DURATION_US`. The
counter advances once per captured PCM frame BEFORE the encode/mute/
send path, so:
- Encoder failures, empty-encoded frames, and muted frames each
consume one slot — preserving the wall-clock gap on the wire so
a 5 s mute shows up as a 5 s timestamp jump (matches the prior
wall-clock behaviour for mute and silence).
- Send-time stalls don't drift the timestamp: it's computed at the
top of the iteration, not after the suspending send.
- Survives publisher hot-swap (single-broadcast, single-encoder;
the new publisher inherits the running counter) — same guarantee
the prior `TimeMark` had.
Also adds [AudioFormat.FRAME_DURATION_US] = 20_000 (960 / 48 kHz × 1e6)
so the constant lives next to FRAME_SIZE_SAMPLES rather than being
implicit in every call site.
https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47
This commit is contained in:
@@ -36,6 +36,15 @@ object AudioFormat {
|
||||
/** 20 ms at 48 kHz. */
|
||||
const val FRAME_SIZE_SAMPLES: Int = 960
|
||||
|
||||
/**
|
||||
* Duration of one Opus frame in microseconds — derived from
|
||||
* [FRAME_SIZE_SAMPLES] / [SAMPLE_RATE_HZ]. Used by the publisher
|
||||
* to stamp each frame at frame-index × this value, giving hang.js's
|
||||
* watcher a perfect 20 ms cadence even when the broadcaster's send
|
||||
* loop suspends on transport backpressure.
|
||||
*/
|
||||
const val FRAME_DURATION_US: Long = (FRAME_SIZE_SAMPLES.toLong() * 1_000_000L) / SAMPLE_RATE_HZ
|
||||
|
||||
/** Bytes per PCM 16-bit sample. */
|
||||
const val BYTES_PER_SAMPLE: Int = 2
|
||||
}
|
||||
|
||||
+36
-9
@@ -27,8 +27,6 @@ import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.cancelAndJoin
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlin.time.TimeMark
|
||||
import kotlin.time.TimeSource
|
||||
|
||||
/**
|
||||
* Mirror of [NestBroadcaster] but driving a moq-lite
|
||||
@@ -207,15 +205,44 @@ class NestMoqLiteBroadcaster(
|
||||
// will skip the leading varint as a microsecond
|
||||
// timestamp; they MUST receive a real timestamp or
|
||||
// their decoder picks up garbage bytes ahead of the
|
||||
// Opus packet. We use a monotonic mark captured at
|
||||
// capture-loop start so timestamps are robust to mute
|
||||
// gaps and survive publisher hot-swap (the wire is
|
||||
// single-broadcast, single-encoder; timestamps are
|
||||
// codec-payload metadata, not stream-position).
|
||||
val frameStartMark: TimeMark = TimeSource.Monotonic.markNow()
|
||||
// Opus packet.
|
||||
//
|
||||
// Frame-index-derived timestamp (NOT wall clock). Each
|
||||
// captured PCM frame occupies exactly one
|
||||
// [AudioFormat.FRAME_DURATION_US] slot in the timeline,
|
||||
// and the timestamp is `nextFrameIndex *
|
||||
// FRAME_DURATION_US` snapped to that grid. This matters
|
||||
// because:
|
||||
// - `current.send(...)` suspends on transport
|
||||
// backpressure; a wall-clock timestamp captured at
|
||||
// send-time would drift forward of the actual
|
||||
// capture time of the audio sample, and the watcher's
|
||||
// WebCodecs AudioDecoder would schedule playback
|
||||
// at the wrong instant (audible offset between
|
||||
// speaker and listener).
|
||||
// - The capture loop is wall-clock-pinned anyway —
|
||||
// `capture.readFrame()` blocks until the next 20 ms
|
||||
// of PCM is ready — so frame-index advances at the
|
||||
// same rate as wall time across the whole
|
||||
// broadcast lifetime, with no codec drift relative
|
||||
// to the watcher's playback clock.
|
||||
//
|
||||
// The counter advances on EVERY captured PCM frame —
|
||||
// including encoder failures, empty-encoded frames, and
|
||||
// muted frames — so a mute gap shows up on the wire as
|
||||
// a real wall-clock gap (timestamps jump forward by the
|
||||
// muted duration) rather than collapsing the silence.
|
||||
// Survives publisher hot-swap (single-broadcast,
|
||||
// single-encoder; the new publisher inherits the
|
||||
// running counter) for the same reason the old
|
||||
// wall-clock TimeMark did: timestamps are codec-payload
|
||||
// metadata, not stream-position.
|
||||
var nextFrameIndex = 0L
|
||||
try {
|
||||
while (true) {
|
||||
val pcm = capture.readFrame() ?: break
|
||||
val timestampUs = nextFrameIndex * AudioFormat.FRAME_DURATION_US
|
||||
nextFrameIndex += 1
|
||||
val opus =
|
||||
try {
|
||||
encoder.encode(pcm)
|
||||
@@ -251,7 +278,7 @@ class NestMoqLiteBroadcaster(
|
||||
// for the production cliff this works around.
|
||||
val sendOutcome =
|
||||
runCatching {
|
||||
val tsBytes = Varint.encode(frameStartMark.elapsedNow().inWholeMicroseconds)
|
||||
val tsBytes = Varint.encode(timestampUs)
|
||||
val payload = ByteArray(tsBytes.size + opus.size)
|
||||
tsBytes.copyInto(payload, 0)
|
||||
opus.copyInto(payload, tsBytes.size)
|
||||
|
||||
Reference in New Issue
Block a user