From 361dbbe201e3c2e3b859cb6e6bab68d3a33fa60a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 15:23:29 +0000 Subject: [PATCH] =?UTF-8?q?fix(nests):=20stamp=20Opus=20frames=20at=20fram?= =?UTF-8?q?e-index=20=C3=97=20frame-duration,=20not=20wall=20clock?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The legacy-container timestamp on each audio frame was previously captured at send-time via `frameStartMark.elapsedNow()`. That mark advances on wall clock, but `current.send(...)` is a suspending call that backs off on transport backpressure — so a frame captured at T gets stamped at T+δ once the previous send drains, and the watcher's WebCodecs AudioDecoder schedules playback at the wrong instant. Audible as a slowly-growing offset between speaker and listener over the life of a session that's seen any meaningful send-side queueing. Replace with a frame-index counter pinned to the codec grid: `timestampUs = nextFrameIndex * AudioFormat.FRAME_DURATION_US`. The counter advances once per captured PCM frame BEFORE the encode/mute/ send path, so: - Encoder failures, empty-encoded frames, and muted frames each consume one slot — preserving the wall-clock gap on the wire so a 5 s mute shows up as a 5 s timestamp jump (matches the prior wall-clock behaviour for mute and silence). - Send-time stalls don't drift the timestamp: it's computed at the top of the iteration, not after the suspending send. - Survives publisher hot-swap (single-broadcast, single-encoder; the new publisher inherits the running counter) — same guarantee the prior `TimeMark` had. Also adds [AudioFormat.FRAME_DURATION_US] = 20_000 (960 / 48 kHz × 1e6) so the constant lives next to FRAME_SIZE_SAMPLES rather than being implicit in every call site. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../vitorpamplona/nestsclient/audio/Audio.kt | 9 ++++ .../audio/NestMoqLiteBroadcaster.kt | 45 +++++++++++++++---- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/Audio.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/Audio.kt index e015a49c1..c2be91b94 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/Audio.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/Audio.kt @@ -36,6 +36,15 @@ object AudioFormat { /** 20 ms at 48 kHz. */ const val FRAME_SIZE_SAMPLES: Int = 960 + /** + * Duration of one Opus frame in microseconds — derived from + * [FRAME_SIZE_SAMPLES] / [SAMPLE_RATE_HZ]. Used by the publisher + * to stamp each frame at frame-index × this value, giving hang.js's + * watcher a perfect 20 ms cadence even when the broadcaster's send + * loop suspends on transport backpressure. + */ + const val FRAME_DURATION_US: Long = (FRAME_SIZE_SAMPLES.toLong() * 1_000_000L) / SAMPLE_RATE_HZ + /** Bytes per PCM 16-bit sample. */ const val BYTES_PER_SAMPLE: Int = 2 } diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt index cf10c7a57..fadda7a28 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt @@ -27,8 +27,6 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.launch -import kotlin.time.TimeMark -import kotlin.time.TimeSource /** * Mirror of [NestBroadcaster] but driving a moq-lite @@ -207,15 +205,44 @@ class NestMoqLiteBroadcaster( // will skip the leading varint as a microsecond // timestamp; they MUST receive a real timestamp or // their decoder picks up garbage bytes ahead of the - // Opus packet. We use a monotonic mark captured at - // capture-loop start so timestamps are robust to mute - // gaps and survive publisher hot-swap (the wire is - // single-broadcast, single-encoder; timestamps are - // codec-payload metadata, not stream-position). - val frameStartMark: TimeMark = TimeSource.Monotonic.markNow() + // Opus packet. + // + // Frame-index-derived timestamp (NOT wall clock). Each + // captured PCM frame occupies exactly one + // [AudioFormat.FRAME_DURATION_US] slot in the timeline, + // and the timestamp is `nextFrameIndex * + // FRAME_DURATION_US` snapped to that grid. This matters + // because: + // - `current.send(...)` suspends on transport + // backpressure; a wall-clock timestamp captured at + // send-time would drift forward of the actual + // capture time of the audio sample, and the watcher's + // WebCodecs AudioDecoder would schedule playback + // at the wrong instant (audible offset between + // speaker and listener). + // - The capture loop is wall-clock-pinned anyway — + // `capture.readFrame()` blocks until the next 20 ms + // of PCM is ready — so frame-index advances at the + // same rate as wall time across the whole + // broadcast lifetime, with no codec drift relative + // to the watcher's playback clock. + // + // The counter advances on EVERY captured PCM frame — + // including encoder failures, empty-encoded frames, and + // muted frames — so a mute gap shows up on the wire as + // a real wall-clock gap (timestamps jump forward by the + // muted duration) rather than collapsing the silence. + // Survives publisher hot-swap (single-broadcast, + // single-encoder; the new publisher inherits the + // running counter) for the same reason the old + // wall-clock TimeMark did: timestamps are codec-payload + // metadata, not stream-position. + var nextFrameIndex = 0L try { while (true) { val pcm = capture.readFrame() ?: break + val timestampUs = nextFrameIndex * AudioFormat.FRAME_DURATION_US + nextFrameIndex += 1 val opus = try { encoder.encode(pcm) @@ -251,7 +278,7 @@ class NestMoqLiteBroadcaster( // for the production cliff this works around. val sendOutcome = runCatching { - val tsBytes = Varint.encode(frameStartMark.elapsedNow().inWholeMicroseconds) + val tsBytes = Varint.encode(timestampUs) val payload = ByteArray(tsBytes.size + opus.size) tsBytes.copyInto(payload, 0) opus.copyInto(payload, tsBytes.size)