From 3a2010d9c000bafd72a2b8ce4b1952573c250fdc Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 5 May 2026 15:36:17 +0000 Subject: [PATCH 01/41] debug(nests): add diagnostic logs to localise receiver audio dropout Adds focused logs at every suspicious point on the moq-lite receive + publish paths so we can pin down why the receiver phone shows a brief ~5s window of activity then nothing while the speaker phone's UI looks healthy. Receiver-side (`NestRx` tag): - SUBSCRIBE send / SUBSCRIBE_OK / SUBSCRIBE bidi exit (with reason) - announce-watch update per Active/Ended event, plus which subs get closed when an Ended hits - pumpUniStreams start + naturally-ended / threw with stream count - drainOneGroup header, FIN (frames + droppedNoSub + trySend fail counters), and per-throw with stream sequence - wrapper handle attached / objects flow ENDED with emitted count Broadcaster-side (`NestTx` tag): - inbound ANNOUNCE / SUBSCRIBE accepted / track-mismatch ignored - registerInboundSubscription / removeInboundSubscription with inboundSubs.size - throttled "send returning false (no subs / publisher closed)" so the speaker UI's "I'm broadcasting" claim can be cross-checked against frames actually leaving the wire - send threw (consecutive count) so a sustained openUniStream failure surfaces before MAX_CONSECUTIVE_SEND_ERRORS bails - openGroupStream open + per-throw All log calls use the lambda overload so the throttled/string-format work only runs when the tag is enabled. --- .../nestsclient/ReconnectingNestsListener.kt | 8 +- .../audio/NestMoqLiteBroadcaster.kt | 31 ++++- .../nestsclient/moq/lite/MoqLiteSession.kt | 106 +++++++++++++++--- 3 files changed, 125 insertions(+), 20 deletions(-) diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsListener.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsListener.kt index 6c4e726d9..7ba2c7b61 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsListener.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsListener.kt @@ -392,11 +392,17 @@ private class ReconnectingHandle( // inheriting the last failure window's saturation). subscribeRetryDelayMs = SUBSCRIBE_RETRY_BACKOFF_INITIAL_MS liveHandleRef.set(handle) + Log.d("NestRx") { "wrapper: handle attached id=${handle.subscribeId}, starting collect" } + var emitted = 0L try { - handle.objects.collect { frames.emit(it) } + handle.objects.collect { + emitted += 1 + frames.emit(it) + } } finally { if (liveHandleRef.get() === handle) liveHandleRef.set(null) } + Log.w("NestRx") { "wrapper: handle objects flow ENDED id=${handle.subscribeId} emitted=$emitted — re-issuing after ${RESUBSCRIBE_BACKOFF_MS}ms" } // Brief backoff so a permanently-gone // publisher doesn't tight-loop the relay // with re-subscribes. 100 ms stays well diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt index 385a68188..2b8dab09c 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt @@ -181,6 +181,10 @@ class NestMoqLiteBroadcaster( // we bail. publisher.send returning `false` (no inbound // subscriber) is NOT counted — empty rooms are normal. var consecutiveSendErrors = 0 + // Diagnostic counters: throttled logging at 50Hz capture rate + // would flood logcat without these. + var sentFrames: Long = 0L + var droppedNoSubFrames: Long = 0L // Track which publisher we last sent to. On swapPublisher // (JWT-refresh hot swap), the snapshot below picks up the // new reference; we reset framesInCurrentGroup so the @@ -229,16 +233,32 @@ class NestMoqLiteBroadcaster( // for the production cliff this works around. val sendOutcome = runCatching { - current.send(opus) + val accepted = current.send(opus) framesInCurrentGroup += 1 if (framesInCurrentGroup >= framesPerGroup) { current.endGroup() framesInCurrentGroup = 0 } + accepted } sendOutcome - .onSuccess { + .onSuccess { accepted -> consecutiveSendErrors = 0 + if (accepted) { + sentFrames += 1 + if (sentFrames % SEND_LOG_THROTTLE == 0L) { + com.vitorpamplona.quartz.utils.Log.d("NestTx") { + "broadcaster sent frame #$sentFrames (group $framesInCurrentGroup/$framesPerGroup)" + } + } + } else { + droppedNoSubFrames += 1 + if (droppedNoSubFrames % SEND_LOG_THROTTLE == 0L) { + com.vitorpamplona.quartz.utils.Log.w("NestTx") { + "broadcaster send returned false — frame dropped (count=$droppedNoSubFrames, sent=$sentFrames)" + } + } + } // Fire the speaking-ring tap only on // a successful send. If the publisher // throws (transport gone, peer dead), @@ -248,6 +268,9 @@ class NestMoqLiteBroadcaster( }.onFailure { t -> if (t is CancellationException) throw t consecutiveSendErrors += 1 + com.vitorpamplona.quartz.utils.Log.w("NestTx") { + "broadcaster send threw (consecutive=$consecutiveSendErrors): ${t::class.simpleName}: ${t.message}" + } onError( AudioException( AudioException.Kind.PlaybackFailed, @@ -377,5 +400,9 @@ class NestMoqLiteBroadcaster( * transport is irrecoverably dead. */ const val MAX_CONSECUTIVE_SEND_ERRORS: Int = 250 + + // Diagnostic: log every Nth frame (sent or dropped) so a sustained + // window doesn't flood logcat at 50 fps. + private const val SEND_LOG_THROTTLE: Long = 50L } } diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt index b6169fd03..6fc3a2755 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt @@ -262,6 +262,7 @@ class MoqLiteSession internal constructor( subscriptionsBySubscribeId[id] = sub if (groupPump == null) groupPump = scope.launch { pumpUniStreams() } } + Log.d("NestRx") { "SUBSCRIBE send id=$id broadcast='$broadcast' track='$track' maxLatencyMs=$maxLatencyMillis" } // Now that the subscription is registered, push the SUBSCRIBE // bytes. If `bidi.write` throws (transport torn down, peer // reset) we'd otherwise leave an orphaned map entry whose @@ -272,6 +273,7 @@ class MoqLiteSession internal constructor( bidi.write(Varint.encode(MoqLiteControlType.Subscribe.code)) bidi.write(MoqLiteCodec.encodeSubscribe(request)) } catch (t: Throwable) { + Log.w("NestRx") { "SUBSCRIBE write failed id=$id: ${t::class.simpleName}: ${t.message}" } state.withLock { subscriptionsBySubscribeId.remove(id) } frames.close() runCatching { bidi.finish() } @@ -314,6 +316,7 @@ class MoqLiteSession internal constructor( if (!responseDeferred.isCompleted) responseDeferred.completeExceptionally(ce) throw ce } catch (t: Throwable) { + Log.w("NestRx") { "SUBSCRIBE bidi collector threw id=$id: ${t::class.simpleName}: ${t.message}" } if (!responseDeferred.isCompleted) responseDeferred.completeExceptionally(t) } if (!responseDeferred.isCompleted) { @@ -325,6 +328,9 @@ class MoqLiteSession internal constructor( // (or any throw from await), it already removed the // subscription before throwing. Either way: remove + close. val removed = state.withLock { subscriptionsBySubscribeId.remove(id) } + if (removed != null) { + Log.w("NestRx") { "SUBSCRIBE bidi exited, closing frames id=$id broadcast='${removed.request.broadcast}' track='${removed.request.track}'" } + } removed?.frames?.close() } @@ -353,6 +359,7 @@ class MoqLiteSession internal constructor( } is MoqLiteCodec.SubscribeResponse.Ok -> { + Log.d("NestRx") { "SUBSCRIBE_OK id=$id broadcast='$broadcast' track='$track'" } return MoqLiteSubscribeHandle( id = id, ok = resp.ok, @@ -428,6 +435,7 @@ class MoqLiteSession internal constructor( private suspend fun pumpAnnounceWatch(handle: MoqLiteAnnouncesHandle) { try { handle.updates.collect { update -> + Log.d("NestRx") { "ANNOUNCE update status=${update.status} suffix='${update.suffix}' hops=${update.hops}" } if (update.status != MoqLiteAnnounceStatus.Ended) return@collect val targets = state.withLock { @@ -435,6 +443,9 @@ class MoqLiteSession internal constructor( .filter { it.request.broadcast == update.suffix } .toList() } + if (targets.isNotEmpty()) { + Log.w("NestRx") { "ANNOUNCE Ended for suffix='${update.suffix}' → closing ${targets.size} subscription(s): ${targets.map { "id=${it.id} track='${it.request.track}'" }}" } + } for (sub in targets) { // Just close the frames channel — the // wrapper-level collect of `frames.consumeAsFlow()` @@ -450,9 +461,11 @@ class MoqLiteSession internal constructor( runCatching { sub.bidi.finish() } } } + Log.w("NestRx") { "ANNOUNCE pump: updates flow ended naturally" } } catch (ce: kotlinx.coroutines.CancellationException) { throw ce - } catch (_: Throwable) { + } catch (t: Throwable) { + Log.w("NestRx") { "ANNOUNCE pump threw ${t::class.simpleName}: ${t.message}" } // Announce bidi died — same best-effort fallback. } finally { runCatching { handle.close() } @@ -470,6 +483,8 @@ class MoqLiteSession internal constructor( * One pump per session — started lazily on the first subscribe. */ private suspend fun pumpUniStreams() { + Log.d("NestRx") { "pumpUniStreams started" } + var streamCount = 0L try { // coroutineScope binds each per-stream drain to this pump's // job — when the pump is cancelled (session close), every @@ -478,24 +493,32 @@ class MoqLiteSession internal constructor( // independently errors out. kotlinx.coroutines.coroutineScope { transport.incomingUniStreams().collect { stream -> - launch { drainOneGroup(stream) } + val n = ++streamCount + launch { drainOneGroup(stream, n) } } } + Log.w("NestRx") { "pumpUniStreams: incomingUniStreams flow ended naturally after $streamCount streams" } } catch (ce: CancellationException) { throw ce } catch (t: Throwable) { - Log.w("NestRx") { "pumpUniStreams ended with ${t::class.simpleName}: ${t.message}" } + Log.w("NestRx") { "pumpUniStreams ended after $streamCount streams with ${t::class.simpleName}: ${t.message}" } // Transport closed — subscriptions will surface end-of-flow // via their own bidi pumps as well. } } - private suspend fun drainOneGroup(stream: com.vitorpamplona.nestsclient.transport.WebTransportReadStream) { + private suspend fun drainOneGroup( + stream: com.vitorpamplona.nestsclient.transport.WebTransportReadStream, + streamSeq: Long, + ) { val buffer = MoqLiteFrameBuffer() var typeRead = false var subscribeId: Long = -1L var groupSequence: Long = -1L var headerRead = false + var frameCount = 0 + var droppedNoSub = 0 + var trySendFailures = 0 try { stream.incoming().collect { chunk -> buffer.push(chunk) @@ -512,25 +535,35 @@ class MoqLiteSession internal constructor( subscribeId = hdr.subscribeId groupSequence = hdr.sequence headerRead = true + Log.d("NestRx") { "drainOneGroup#$streamSeq header subId=$subscribeId groupSeq=$groupSequence" } } while (true) { val frame = buffer.readSizePrefixed() ?: break val sub = state.withLock { subscriptionsBySubscribeId[subscribeId] } - sub?.frames?.trySend( - MoqLiteFrame( - groupSequence = groupSequence, - payload = frame, - ), - ) + if (sub == null) { + droppedNoSub += 1 + } else { + val sent = + sub.frames.trySend( + MoqLiteFrame( + groupSequence = groupSequence, + payload = frame, + ), + ) + if (!sent.isSuccess) trySendFailures += 1 + } + frameCount += 1 // If the subscription has been closed already we // silently drop the frame — the publisher hasn't // observed the unsubscribe yet (its uni streams // are independent of our bidi FIN). } } + Log.d("NestRx") { "drainOneGroup#$streamSeq FIN subId=$subscribeId groupSeq=$groupSequence frames=$frameCount droppedNoSub=$droppedNoSub trySendFail=$trySendFailures" } } catch (ce: CancellationException) { throw ce - } catch (_: Throwable) { + } catch (t: Throwable) { + Log.w("NestRx") { "drainOneGroup#$streamSeq threw subId=$subscribeId groupSeq=$groupSequence frames=$frameCount: ${t::class.simpleName}: ${t.message}" } // Stream errored / FIN'd. Nothing to do — the next group // arrives on a fresh uni stream. } @@ -669,12 +702,14 @@ class MoqLiteSession internal constructor( ), ) publisher.registerAnnounceBidi(bidi, emittedSuffix) + Log.d("NestTx") { "ANNOUNCE inbound prefix='${please.prefix}' → emitted Active suffix='$emittedSuffix' (publisher.suffix='${publisher.suffix}')" } dispatched = true } MoqLiteControlType.Subscribe -> { val subPayload = buffer.readSizePrefixed() ?: return@collect val sub = MoqLiteCodec.decodeSubscribe(subPayload) + Log.d("NestTx") { "SUBSCRIBE inbound id=${sub.id} broadcast='${sub.broadcast}' track='${sub.track}' (publisher track='${publisher.track}')" } // Register the subscription BEFORE sending Ok so the // peer's observation of Ok is a happens-after of // `inboundSubs += sub`. Otherwise on dispatchers that @@ -809,7 +844,7 @@ class MoqLiteSession internal constructor( */ private inner class PublisherStateImpl( override val suffix: String, - private val track: String, + internal val track: String, ) : MoqLitePublisherHandle { private val gate = Mutex() private val announceBidis = mutableListOf() @@ -817,6 +852,12 @@ class MoqLiteSession internal constructor( private var currentGroup: GroupOutbound? = null private var nextSequence: Long = 0L + // Diagnostic: throttled counter for "send returned false" logs so a + // long no-subscriber window doesn't flood logcat at 50 Hz. + private val sendNoSubLogCount = + java.util.concurrent.atomic + .AtomicLong(0L) + @Volatile private var publisherClosed = false suspend fun registerAnnounceBidi( @@ -834,9 +875,16 @@ class MoqLiteSession internal constructor( suspend fun registerInboundSubscription(sub: MoqLiteSubscribe) { gate.withLock { - if (publisherClosed) return - if (sub.track != track) return + if (publisherClosed) { + Log.w("NestTx") { "SUBSCRIBE inbound rejected (publisher closed) id=${sub.id} track='${sub.track}'" } + return + } + if (sub.track != track) { + Log.w("NestTx") { "SUBSCRIBE inbound track mismatch id=${sub.id} sub.track='${sub.track}' publisher.track='$track' — ignored" } + return + } inboundSubs += sub + Log.d("NestTx") { "SUBSCRIBE registered id=${sub.id} broadcast='${sub.broadcast}' track='${sub.track}' inboundSubs.size=${inboundSubs.size}" } } } @@ -853,6 +901,7 @@ class MoqLiteSession internal constructor( gate.withLock { if (publisherClosed) return if (!inboundSubs.remove(sub)) return + Log.w("NestTx") { "SUBSCRIBE removed (peer FIN/error) id=${sub.id} track='${sub.track}' inboundSubs.size=${inboundSubs.size}" } runCatching { currentGroup?.uni?.finish() } currentGroup = null } @@ -868,8 +917,18 @@ class MoqLiteSession internal constructor( override suspend fun send(payload: ByteArray): Boolean { gate.withLock { - if (publisherClosed) return false - if (inboundSubs.isEmpty()) return false + if (publisherClosed) { + if (sendNoSubLogCount.getAndIncrement() % SEND_LOG_THROTTLE == 0L) { + Log.w("NestTx") { "send returning false — publisher closed (count=${sendNoSubLogCount.get()})" } + } + return false + } + if (inboundSubs.isEmpty()) { + if (sendNoSubLogCount.getAndIncrement() % SEND_LOG_THROTTLE == 0L) { + Log.w("NestTx") { "send returning false — no inboundSubs (count=${sendNoSubLogCount.get()}, payload=${payload.size}B)" } + } + return false + } val group = currentGroup ?: openNextGroupLocked().also { currentGroup = it } // Single-allocation framing: write the varint length // directly into a buffer sized for `varint + payload`, @@ -952,7 +1011,16 @@ class MoqLiteSession internal constructor( // model), so this is fine. val sub = inboundSubs.first() val sequence = nextSequence++ - val uni = openGroupStream(subscribeId = sub.id, sequence = sequence) + val uni = + try { + openGroupStream(subscribeId = sub.id, sequence = sequence) + } catch (ce: CancellationException) { + throw ce + } catch (t: Throwable) { + Log.w("NestTx") { "openGroupStream threw subId=${sub.id} seq=$sequence: ${t::class.simpleName}: ${t.message}" } + throw t + } + Log.d("NestTx") { "openGroupStream subId=${sub.id} seq=$sequence" } return GroupOutbound(sequence = sequence, uni = uni) } } @@ -971,6 +1039,10 @@ class MoqLiteSession internal constructor( /** moq-lite priority byte midpoint — neutral default. */ const val DEFAULT_PRIORITY: Int = 0x80 + // Diagnostic: log "send returned false" once every N invocations. + // At 50 fps and N=50 → ≤ 1 log/sec for a sustained no-sub window. + private const val SEND_LOG_THROTTLE: Long = 50L + /** * Per-subscription channel buffer for inbound frames. 128 audio * frames at Opus 20 ms ≈ 2.5 s of backlog before DROP_OLDEST, From be8dd0a3bc1e4fb06fd2b9f21b17448c5ff2e50f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 5 May 2026 15:56:47 +0000 Subject: [PATCH 02/41] debug(nests): add playback-path logs to localise receiver silence The wire is healthy: drainOneGroup logs show every group reaching the receiver with 5 frames each, no drops, no parse errors, no pump exits, no announce-ended teardown. Yet the green ring on the receiver still blinks-and-stops. The bug must be in the playback pipeline below drainOneGroup. Add NestPlay-tagged logs there so the next repro identifies whether the failure is decoder, AudioTrack allocation, beginPlayback, or write-blocking. NestPlayer.play (`NestPlay` tag): - log start with prerollFrames - log preroll flush + beginPlayback transition - throttled per-N counters: receivedObjects / decodedFrames / emptyDecodes / enqueued - log decoder.decode throws explicitly - log empty-pcm decoder returns (separate from throws) - log enqueue duration when > 50 ms (catches AudioTrack.write-blocking) - log flow COMPLETED / cancelled / pipeline threw with final counts AudioTrackPlayer (`NestPlay` tag): - log start() + the constructed AudioTrack's state / playState / bufferSizeBytes / minBuffer - log beginPlayback's t.play() return + post-call state - log AudioTrack.write partial / error returns - log setMuted / setVolume changes (silent mute is a known suspect for "audio not playing") MediaCodecOpusDecoder (`NestPlay` tag): - log codec name on successful allocation - log allocation failure with exception type / message All log calls use the lambda overload so format work only runs when the tag is enabled. Throttled counters keep logcat at < 1 line/sec per speaker even at 50 fps capture cadence. --- .../nestsclient/audio/AudioTrackPlayer.kt | 24 ++++++++ .../audio/MediaCodecOpusDecoder.kt | 17 ++++-- .../nestsclient/audio/NestPlayer.kt | 58 ++++++++++++++++++- 3 files changed, 94 insertions(+), 5 deletions(-) diff --git a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/AudioTrackPlayer.kt b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/AudioTrackPlayer.kt index 8434263ea..dfd2eaa6e 100644 --- a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/AudioTrackPlayer.kt +++ b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/AudioTrackPlayer.kt @@ -96,6 +96,8 @@ class AudioTrackPlayer( override fun start() { if (track != null) return + com.vitorpamplona.quartz.utils.Log + .d("NestPlay") { "AudioTrackPlayer.start() — allocating AudioTrack" } val channelMask = when (AudioFormat.CHANNELS) { @@ -178,6 +180,10 @@ class AudioTrackPlayer( audioExecutor = executor audioDispatcher = executor.asCoroutineDispatcher() track = newTrack + com.vitorpamplona.quartz.utils.Log.d("NestPlay") { + "AudioTrack allocated: state=${newTrack.state} playState=${newTrack.playState} " + + "bufferSizeBytes=$bufferBytes minBuffer=$minBuffer sampleRate=${AudioFormat.SAMPLE_RATE_HZ}" + } } override fun beginPlayback() { @@ -189,12 +195,18 @@ class AudioTrackPlayer( val t = track ?: return try { t.play() + com.vitorpamplona.quartz.utils.Log.d("NestPlay") { + "AudioTrack.play() returned, state=${t.playState} bufferSize=${t.bufferSizeInFrames} muted=$muted volume=$volume" + } } catch (e: Throwable) { // PLAY-on-uninitialized AudioTrack is the only realistic // failure here, and we already guard against an unallocated // track above. Throw so [NestPlayer]'s outer catch surfaces // it via `onError(AudioException.PlaybackFailed)` — same path // that handles every other mid-stream device failure. + com.vitorpamplona.quartz.utils.Log.w("NestPlay") { + "AudioTrack.play() threw: ${e::class.simpleName}: ${e.message}" + } throw AudioException( AudioException.Kind.DeviceUnavailable, "AudioTrack.play() rejected start", @@ -215,21 +227,33 @@ class AudioTrackPlayer( withContext(dispatcher) { val written = t.write(pcm, 0, pcm.size, AudioTrack.WRITE_BLOCKING) if (written < 0) { + com.vitorpamplona.quartz.utils.Log.w("NestPlay") { + "AudioTrack.write returned error $written (state=${t.playState})" + } throw AudioException( AudioException.Kind.PlaybackFailed, "AudioTrack.write returned error code $written", ) } + if (written != pcm.size) { + com.vitorpamplona.quartz.utils.Log.w("NestPlay") { + "AudioTrack.write partial: requested=${pcm.size} written=$written" + } + } } } override fun setMuted(muted: Boolean) { this.muted = muted + com.vitorpamplona.quartz.utils.Log + .d("NestPlay") { "AudioTrackPlayer.setMuted($muted) volume=$volume" } track?.let { applyMuteVolume(it) } } override fun setVolume(volume: Float) { this.volume = volume.coerceIn(0f, 1f) + com.vitorpamplona.quartz.utils.Log + .d("NestPlay") { "AudioTrackPlayer.setVolume($volume) muted=$muted" } track?.let { applyMuteVolume(it) } } diff --git a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusDecoder.kt b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusDecoder.kt index 5f044d508..c656a5d07 100644 --- a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusDecoder.kt +++ b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusDecoder.kt @@ -38,11 +38,20 @@ import java.nio.ByteOrder class MediaCodecOpusDecoder : OpusDecoder { private val codec: MediaCodec = try { - MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_AUDIO_OPUS).apply { - configure(buildFormat(), null, null, 0) - start() - } + MediaCodec + .createDecoderByType(MediaFormat.MIMETYPE_AUDIO_OPUS) + .apply { + configure(buildFormat(), null, null, 0) + start() + }.also { + com.vitorpamplona.quartz.utils.Log.d("NestPlay") { + "MediaCodecOpusDecoder allocated codec='${it.name}'" + } + } } catch (t: Throwable) { + com.vitorpamplona.quartz.utils.Log.w("NestPlay") { + "MediaCodec audio/opus decoder allocation FAILED: ${t::class.simpleName}: ${t.message}" + } throw AudioException( AudioException.Kind.DeviceUnavailable, "Failed to allocate MediaCodec audio/opus decoder", diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayer.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayer.kt index fe641b994..d70137cbf 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayer.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayer.kt @@ -122,6 +122,15 @@ class NestPlayer( scope.launch { val preroll = ArrayDeque(prerollFrames.coerceAtLeast(1)) var playbackBegun = false + // Diagnostic counters: at 50 fps a per-frame log floods logcat; + // throttle to every Nth event. + var receivedObjects: Long = 0L + var decodedFrames: Long = 0L + var emptyDecodes: Long = 0L + var enqueued: Long = 0L + com.vitorpamplona.quartz.utils.Log.d("NestPlay") { + "NestPlayer.play started (prerollFrames=$prerollFrames)" + } suspend fun beginAndFlushIfNeeded() { if (playbackBegun) return @@ -137,20 +146,34 @@ class NestPlayer( // hardware starts pulling samples; getting // [enqueue] in first means the very first sample // pulled is from our pre-rolled audio, not silence. + com.vitorpamplona.quartz.utils.Log.d("NestPlay") { + "NestPlayer flushing preroll (${preroll.size} frames) → beginPlayback" + } while (preroll.isNotEmpty()) { player.enqueue(preroll.removeFirst()) } player.beginPlayback() playbackBegun = true + com.vitorpamplona.quartz.utils.Log + .d("NestPlay") { "NestPlayer beginPlayback returned" } } try { objects.collect { obj -> + receivedObjects += 1 + if (receivedObjects % PLAY_LOG_THROTTLE == 0L) { + com.vitorpamplona.quartz.utils.Log.d("NestPlay") { + "NestPlayer received obj #$receivedObjects (decoded=$decodedFrames empty=$emptyDecodes enqueued=$enqueued playbackBegun=$playbackBegun)" + } + } val pcm = try { decoder.decode(obj.payload) } catch (ce: CancellationException) { throw ce } catch (t: Throwable) { + com.vitorpamplona.quartz.utils.Log.w("NestPlay") { + "decoder.decode threw on obj #$receivedObjects: ${t::class.simpleName}: ${t.message}" + } onError( AudioException( AudioException.Kind.DecoderError, @@ -160,10 +183,26 @@ class NestPlayer( ) return@collect } - if (pcm.isNotEmpty()) { + if (pcm.isEmpty()) { + emptyDecodes += 1 + if (emptyDecodes % PLAY_LOG_THROTTLE == 0L) { + com.vitorpamplona.quartz.utils.Log.w("NestPlay") { + "decoder returned empty pcm (count=$emptyDecodes / received=$receivedObjects)" + } + } + } else { + decodedFrames += 1 onLevel(peakAmplitude(pcm)) if (playbackBegun) { + val enqueueStart = System.currentTimeMillis() player.enqueue(pcm) + val enqueueMs = System.currentTimeMillis() - enqueueStart + enqueued += 1 + if (enqueued % PLAY_LOG_THROTTLE == 0L || enqueueMs > 50) { + com.vitorpamplona.quartz.utils.Log.d("NestPlay") { + "NestPlayer enqueued #$enqueued (took ${enqueueMs}ms)" + } + } } else { preroll.addLast(pcm) if (preroll.size >= prerollFrames) { @@ -172,14 +211,23 @@ class NestPlayer( } } } + com.vitorpamplona.quartz.utils.Log.w("NestPlay") { + "NestPlayer objects flow COMPLETED (received=$receivedObjects decoded=$decodedFrames empty=$emptyDecodes enqueued=$enqueued)" + } // Flow ended without enough frames to fill the pre-roll // (e.g. the publisher cycled before pre-roll was full, // or the room ended). Flush whatever's queued so any // already-decoded audio still reaches the device. beginAndFlushIfNeeded() } catch (ce: CancellationException) { + com.vitorpamplona.quartz.utils.Log.d("NestPlay") { + "NestPlayer cancelled (received=$receivedObjects decoded=$decodedFrames enqueued=$enqueued)" + } throw ce } catch (t: Throwable) { + com.vitorpamplona.quartz.utils.Log.w("NestPlay") { + "NestPlayer pipeline threw: ${t::class.simpleName}: ${t.message}" + } onError( AudioException( AudioException.Kind.PlaybackFailed, @@ -203,8 +251,16 @@ class NestPlayer( suspend fun stop() { if (stopped) return stopped = true + com.vitorpamplona.quartz.utils.Log + .d("NestPlay") { "NestPlayer.stop()" } job?.cancelAndJoin() runCatching { player.stop() } runCatching { decoder.release() } } + + companion object { + // Diagnostic throttle: log every Nth event during normal flow so a + // sustained 50 fps stream doesn't flood logcat. + private const val PLAY_LOG_THROTTLE: Long = 50L + } } From cb61082bc49707d51b0f297542db024379b60bbd Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 5 May 2026 16:44:39 +0000 Subject: [PATCH 03/41] fix(nests): close receiver-audio dropout cluster + auto-start broadcast on host create MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six changes that together close out the bug class the user reported (receiver "blinks green ring for ~5 s then stops") and the related host-side "circular spinner until first unmute" UX issue. Each is defensible independently; together they form a defense-in-depth against the moq-rs production-relay forward-queue cliff documented in `nestsClient/plans/2026-05-01-quic-stream-cliff-investigation.md`. #3 — `NestMoqLiteBroadcaster.onLevel` only fires when `current.send(opus)` returned `true` (frame reached an inbound subscriber). Two-phone logs showed the host's green/glow firing for the first ~7 s while no listener was attached and after a relay-side cliff while no audio was on the wire. The local "I'm broadcasting" ring now tracks the wire, not the runCatching outcome. #5 — `DEFAULT_FRAMES_PER_GROUP: 5 → 10` (200 ms of audio per group, 5 streams/sec). Two-phone production logs showed the relay still cliffing at ~135 streams under sustained load with the old 5-frame default — same bug class the plan thought it had closed, just shifted by half. Doubling the pack roughly doubles the worst-case cliff window. Kept the existing `framesPerGroup = 5` test scenarios intact since they're explicit on the call site. #4 — `SUBSCRIBE_RETRY_BACKOFF_INITIAL_MS: 250 → 100`. Logs of the join path showed 5 retries (250+500+1000+1000+1000 = 3.75 s) of "subscribe stream FIN before reply" before the relay observed the broadcast's announce. New ladder is 100+200+400+800+1000 = 2.5 s. The MAX cap stays at 1000 ms so a permanently-gone publisher still doesn't hammer the relay. #2 — `NestViewModel.openSubscription` now passes `maxLatencyMs = 500L` on every speaker SUBSCRIBE (new constant `ROOM_AUDIO_MAX_LATENCY_MS`). Tells moq-rs to evict groups older than 500 ms from the per-subscriber forward queue rather than queuing them up to the relay's `MAX_GROUP_AGE = 30 s` default — caps the queue growth that triggers the cliff. The wire support existed in the listener; only the call site was passing 0L. #1 — listener-side cliff detector. New `cliffDetectorJob` in `NestViewModel` periodically scans `activeSubscriptions` against `lastFrameAt` (per-pubkey monotonic TimeMark, updated in `onSpeakerActivity`). When a speaker has been announced Active AND we have an active subscription AND no frames have arrived for `ROOM_AUDIO_CLIFF_TIMEOUT_MS = 4 s`, force a `recycleSession()` so the orchestrator opens a fresh QUIC transport — a clean reset of the relay's per-subscriber forward queue. `ROOM_AUDIO_CLIFF_COOLDOWN_MS = 8 s` prevents back-to-back recycles while the new session is mid-handshake. Started from `launchConnect` after `observeAnnounces`, cancelled in `teardown`. Per-pubkey `lastFrameAt` entry dropped on `closeSubscription` so a removed speaker doesn't trigger a stale recycle. #6 — auto-start broadcast pre-muted on host create. `NestViewModel.startBroadcast` gets an `initialMuted: Boolean = false` parameter; when `true` the broadcaster opens the publisher session, calls `setMuted(true)` before the UI flips to `Broadcasting`, and the mic stays open + capture loop keeps running with `if (muted) continue` gating sends. New `LaunchedEffect(speakerPubkeyHex)` in `OnStageIdleControls` calls `startBroadcast(initialMuted = true)` automatically when mic permission is already granted, so a host who just created a room sees their avatar transition to "Live (silent)" and listeners can subscribe immediately without waiting for the host to tap "Talk" first. Tap-to-talk path also passes `initialMuted = true` now — unmute is a separate, explicit step on the mic toggle that appears once we're in `Broadcasting(isMuted = true)`. Permission- denied case still requires the manual Talk-button tap to launch the runtime permission prompt. Diagnostic logs from the previous two debug commits remain in place (throttled to ≤ 1 line/sec/speaker at 50 fps capture cadence) so the next two-phone repro can validate the fixes against logcat directly. --- .../nests/room/screen/NestActionBar.kt | 36 +++- .../commons/viewmodels/NestViewModel.kt | 198 +++++++++++++++++- .../nestsclient/ReconnectingNestsListener.kt | 25 ++- .../audio/NestMoqLiteBroadcaster.kt | 47 +++-- 4 files changed, 276 insertions(+), 30 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/nests/room/screen/NestActionBar.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/nests/room/screen/NestActionBar.kt index 4270f79c2..bd7ad6081 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/nests/room/screen/NestActionBar.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/nests/room/screen/NestActionBar.kt @@ -50,6 +50,7 @@ import androidx.compose.material3.OutlinedButton import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.saveable.rememberSaveable @@ -270,8 +271,11 @@ private fun OnStageControls( is BroadcastUiState.Failed -> { // Reason is shown in the status strip; this button retries. + // Same auto-muted semantics as the idle path: the retry + // should bring the publisher back up without opening the + // mic, then the user can unmute deliberately. TalkButton( - onClick = { viewModel.startBroadcast(speakerPubkeyHex) }, + onClick = { viewModel.startBroadcast(speakerPubkeyHex, initialMuted = true) }, contentDescription = stringRes(R.string.nest_talk), ) LeaveStageButton(onClick = { viewModel.setOnStage(false) }) @@ -289,17 +293,43 @@ private fun OnStageIdleControls( val permissionLauncher = rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted -> permissionDenied = !granted - if (granted) viewModel.startBroadcast(speakerPubkeyHex) + // Permission grant on a Talk-button tap should start a + // *muted* broadcast — auto-start semantics. The host + // taps Talk to claim the on-stage slot; unmute is a + // separate, deliberate action below. + if (granted) viewModel.startBroadcast(speakerPubkeyHex, initialMuted = true) } // The launcher callback never fires for Settings-deep-link grants, so // re-check on every recomposition to auto-clear the warning. val showDenialWarning = permissionDenied && !context.hasMicPermission() + // Auto-start the muted broadcast as soon as the host (or any + // on-stage speaker) lands on this composable WITH mic permission + // already granted. Without this, a host who creates a room sees + // a "Connecting..." spinner over their avatar until they tap Talk + // — the publisher session only opens on tap, so listeners can't + // subscribe yet, and the `connectingSpeakers` overlay sticks. By + // pre-opening the publisher in muted state here, listeners get + // an immediate Active announce and the host's avatar leaves the + // spinner state on its own. Unmute is still a separate tap on + // the mic toggle that appears once we transition to + // `BroadcastUiState.Broadcasting(isMuted = true)`. + // + // Permission-denied case is intentionally left to the manual + // Talk-button tap below: we don't want a host to be auto-prompted + // for the mic the instant they enter the room. Tapping Talk is + // the consent gesture that triggers `permissionLauncher`. + LaunchedEffect(speakerPubkeyHex) { + if (context.hasMicPermission()) { + viewModel.startBroadcast(speakerPubkeyHex, initialMuted = true) + } + } + TalkButton( onClick = { if (context.hasMicPermission()) { - viewModel.startBroadcast(speakerPubkeyHex) + viewModel.startBroadcast(speakerPubkeyHex, initialMuted = true) } else { permissionLauncher.launch(Manifest.permission.RECORD_AUDIO) } diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt index b06bd9f36..bff80eb3e 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt @@ -265,6 +265,41 @@ class NestViewModel( private val activeSubscriptions = mutableMapOf() private val speakingExpiryJobs = mutableMapOf() + /** + * Monotonic [kotlin.time.TimeMark] of the last MoQ object delivered + * to the consumer flow per speaker pubkey. Updated in + * [onSpeakerActivity] alongside the speaking-now flag. Read by + * [cliffDetectorJob] to spot the relay-side forward-queue cliff + * documented in + * `nestsClient/plans/2026-05-01-quic-stream-cliff-investigation.md`: + * the relay stops opening new uni streams to the listener while + * the announce stream still says the broadcast is Active and the + * subscribe bidi is still open from QUIC's POV — meaning no other + * layer notices. + */ + private val lastFrameAt = mutableMapOf() + + /** + * Periodic scan of [activeSubscriptions] vs [_announcedSpeakers] + * vs [lastFrameAt]. When a speaker has been announced Active + * AND we have an active subscription AND no frames have arrived + * for [ROOM_AUDIO_CLIFF_TIMEOUT_MS], force a `recycleSession()` + * so the orchestrator opens a fresh QUIC transport — a clean + * recovery from the relay-side per-subscriber forward-queue + * starvation that has no QUIC- or moq-lite-level signal. + */ + private var cliffDetectorJob: Job? = null + + /** + * [kotlin.time.TimeMark] of the most recent recycle the cliff + * detector triggered, or `null` if it has never fired. Acts as a + * cooldown so a single cliff event doesn't kick off multiple + * recycles back-to-back while the wrapper is mid-handshake on + * the new session — the new session has no incoming frames yet, + * which would otherwise trip the detector again immediately. + */ + private var lastCliffRecycleAt: kotlin.time.TimeSource.Monotonic.ValueTimeMark? = null + /** * Per-speaker catalog-fetch coroutines. Each entry is the * background `subscribeCatalog` collector launched in @@ -561,7 +596,10 @@ class NestViewModel( * session over a separate WebTransport — nests' protocol uses one * session per direction. */ - fun startBroadcast(speakerPubkeyHex: String) { + fun startBroadcast( + speakerPubkeyHex: String, + initialMuted: Boolean = false, + ) { if (closed || !canBroadcast) return if (_uiState.value.connection !is ConnectionUiState.Connected) return if (_uiState.value.broadcast is BroadcastUiState.Broadcasting || @@ -608,7 +646,26 @@ class NestViewModel( }, ) broadcastHandle = handle - _uiState.update { it.copy(broadcast = BroadcastUiState.Broadcasting(isMuted = false)) } + // Auto-start path (room creation): the host wants + // listeners to be able to subscribe BEFORE they hit + // unmute, so we open the publisher session pre-muted. + // The mic stays open + the broadcaster keeps + // capturing + encoding, but `setMuted(true)` makes + // every send drop on the floor inside + // `NestMoqLiteBroadcaster.start`'s `if (muted) continue` + // gate. Listener-side announces fire as soon as the + // publisher registers with the relay, so the audience + // can subscribe immediately and switch from + // "Connecting…" to "Live (silent)" while they wait + // for the host's first word. Composes with `focusMuted` + // exactly the same way [setMicMuted] does so an + // inbound phone call during the auto-start window + // doesn't get a silent unmute. + if (initialMuted) { + val effective = initialMuted || focusMuted + runCatching { handle.setMuted(effective) } + } + _uiState.update { it.copy(broadcast = BroadcastUiState.Broadcasting(isMuted = initialMuted)) } } catch (ce: CancellationException) { throw ce } catch (t: Throwable) { @@ -855,6 +912,7 @@ class NestViewModel( listener = l observeListenerState(l) observeAnnounces(l) + startCliffDetector() } catch (ce: CancellationException) { throw ce } catch (t: Throwable) { @@ -915,6 +973,11 @@ class NestViewModel( if (rawAudioLevels.remove(slot.pubkey) != null) { _audioLevels.value = rawAudioLevels.toMap() } + // Drop the cliff-detector heartbeat for this pubkey so the next + // tick doesn't see a stale "last frame was 30s ago" entry for a + // speaker we already stopped subscribing to and trigger a + // gratuitous recycle. + lastFrameAt.remove(slot.pubkey) if (roomPlayer != null || handle != null) { viewModelScope.launch { roomPlayer?.runCatching { stop() } @@ -961,7 +1024,21 @@ class NestViewModel( ) { if (closed || activeSubscriptions[pubkey] !== slot) return try { - val handle = l.subscribeSpeaker(pubkey) + // [maxLatencyMs] tells the relay our staleness tolerance: + // groups older than this should be evicted, not queued. + // With `0L` (the wire default the JS reference uses) the + // relay's per-subscriber forward queue grows unbounded + // until it cliffs — see + // `nestsClient/plans/2026-05-01-quic-stream-cliff-investigation.md`. + // 500 ms is long enough to ride out a typical relay-side + // hiccup without pruning a healthy group, short enough + // that a wedged subscriber drains its buffer before the + // queue overflows. Combined with the cliff-detector below + // that triggers `recycleSession()` when frames stop + // arriving despite the broadcast still being announced, + // this is a defense-in-depth pair against the moq-rs + // forward-queue starvation behaviour. + val handle = l.subscribeSpeaker(pubkey, maxLatencyMs = ROOM_AUDIO_MAX_LATENCY_MS) // Re-check after the suspending subscribeSpeaker — the user // may have removed this speaker via updateSpeakers / disconnected // while the SUBSCRIBE was in flight. If so, abandon the handle @@ -1089,6 +1166,10 @@ class NestViewModel( announcesJob = null levelEmitterJob?.cancel() levelEmitterJob = null + cliffDetectorJob?.cancel() + cliffDetectorJob = null + lastFrameAt.clear() + lastCliffRecycleAt = null // Audio-focus observation only ends on the final teardown — // a transient disconnect+reconnect (user retry, room swap) // keeps the bus subscription alive so a focus loss that @@ -1175,6 +1256,14 @@ class NestViewModel( */ private fun onSpeakerActivity(pubkey: String) { if (closed) return + // Cliff-detector heartbeat: each MoQ object proves the + // listener-side relay-forward queue is still flowing for this + // speaker. The detector only acts when an announced speaker's + // last-frame timestamp ages past `ROOM_AUDIO_CLIFF_TIMEOUT_MS`, + // so an active stream resets it on every frame. + lastFrameAt[pubkey] = + kotlin.time.TimeSource.Monotonic + .markNow() speakingExpiryJobs[pubkey]?.cancel() // First frame for this subscription — clear the buffering // overlay. Subsequent frames are no-ops here. @@ -1191,6 +1280,61 @@ class NestViewModel( } } + /** + * Periodic scan that detects the relay-side forward-queue cliff: + * an announced speaker whose subscription is open but hasn't + * delivered any object in [ROOM_AUDIO_CLIFF_TIMEOUT_MS]. Triggers + * `recycleSession()` on the listener so the wrapper opens a fresh + * QUIC transport — the new session has a fresh per-subscriber + * queue on the relay's side. + * + * Idempotent on start. Stops itself when the listener is gone + * (explicit teardown signal) — no-ops on transient empty + * `activeSubscriptions` so an audience-only listener doesn't + * needlessly recycle. + */ + private fun startCliffDetector() { + if (cliffDetectorJob?.isActive == true) return + cliffDetectorJob = + viewModelScope.launch { + while (true) { + delay(ROOM_AUDIO_CLIFF_CHECK_INTERVAL_MS) + if (closed) return@launch + val l = listener ?: continue + if (_uiState.value.connection !is ConnectionUiState.Connected) continue + // Cooldown: if we very recently triggered a recycle, + // give the orchestrator time to bring up the new + // session before re-checking. A new session has + // nothing in `lastFrameAt` yet which would otherwise + // immediately re-trigger. + val sinceRecycle = lastCliffRecycleAt?.elapsedNow() + if (sinceRecycle != null && + sinceRecycle.inWholeMilliseconds < ROOM_AUDIO_CLIFF_COOLDOWN_MS + ) { + continue + } + val announced = _announcedSpeakers.value + val stalled = + activeSubscriptions + .asSequence() + .filter { (pubkey, slot) -> slot.isPlaying && pubkey in announced } + .filter { (pubkey, _) -> + val mark = lastFrameAt[pubkey] ?: return@filter false + mark.elapsedNow().inWholeMilliseconds >= ROOM_AUDIO_CLIFF_TIMEOUT_MS + }.map { it.key } + .toList() + if (stalled.isEmpty()) continue + com.vitorpamplona.quartz.utils.Log.w("NestRx") { + "cliff-detector: announced+subscribed but silent for ≥${ROOM_AUDIO_CLIFF_TIMEOUT_MS}ms — recycling session. stalled=$stalled" + } + lastCliffRecycleAt = + kotlin.time.TimeSource.Monotonic + .markNow() + runCatching { l.recycleSession() } + } + } + } + private fun clearSpeaking(pubkey: String) { speakingExpiryJobs.remove(pubkey) if (_uiState.value.speakingNow.contains(pubkey)) { @@ -1442,6 +1586,54 @@ const val ROOM_PLAYER_PREROLL_FRAMES: Int = 5 */ const val LEVEL_TICK_MS: Long = 100L +/** + * `max_latency` we send on every speaker SUBSCRIBE. Tells the relay + * to evict groups older than this from the per-subscriber forward + * queue rather than holding them up to the relay's `MAX_GROUP_AGE` + * default (30 s). Defends against the moq-rs forward-queue cliff + * documented in + * `nestsClient/plans/2026-05-01-quic-stream-cliff-investigation.md`. + * 500 ms = 25 Opus frames at 20 ms cadence — long enough to ride + * out a typical hiccup without dropping a healthy frame, short + * enough to keep the relay's per-subscriber queue bounded. + */ +const val ROOM_AUDIO_MAX_LATENCY_MS: Long = 500L + +/** + * Inactivity threshold for the listener-side cliff detector. If + * a speaker is `_announcedSpeakers` (= relay says they're broadcasting) + * AND the wrapper's MoqObject flow doesn't deliver a frame for this + * many milliseconds, the listener forces a `recycleSession()` so the + * orchestrator opens a fresh QUIC transport. Resets the relay's + * per-subscriber forward queue. + * + * 4 s gives enough headroom to ride out a publisher-side group-rollover + * hiccup (typically < 200 ms) without false-positive recycling, while + * keeping the audible gap from a real cliff to ~5 s of silence at + * worst (4 s detection + ~1 s reconnect handshake). + */ +const val ROOM_AUDIO_CLIFF_TIMEOUT_MS: Long = 4_000L + +/** + * How often the cliff detector wakes up to scan + * `activeSubscriptions` against `lastFrameAt`. 1 s is a coarse + * enough cadence not to cost anything (one map walk per second) + * while keeping detection latency well under the 4 s threshold. + */ +const val ROOM_AUDIO_CLIFF_CHECK_INTERVAL_MS: Long = 1_000L + +/** + * Cooldown after a cliff-detector-driven recycle. Suppresses + * back-to-back recycles while the wrapper is mid-handshake on the + * fresh session — a brand-new session has no `lastFrameAt` entries + * yet, which would otherwise immediately re-trigger the detector + * on the next 1 s tick. Long enough to cover the typical reconnect + * handshake plus a couple of audio-frame round trips, short enough + * that a recycle that didn't actually clear the cliff is recovered + * from quickly. + */ +const val ROOM_AUDIO_CLIFF_COOLDOWN_MS: Long = 8_000L + /** * How long a kind-7 reaction stays in * [NestViewModel.recentReactions] before the eviction sweep diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsListener.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsListener.kt index 7ba2c7b61..1b1b9b7a8 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsListener.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsListener.kt @@ -476,18 +476,23 @@ private class ReconnectingHandle( // before the publisher exists, and the relay FINs the bidi // without a SubscribeOk/Drop reply. // - // - INITIAL = 250 ms: under moq-rs's typical announce-propagation - // latency (< 200 ms in production traces), so the very first - // retry usually succeeds and the listener-side buffer hides - // the gap. - // - Doubles each miss → 500 ms → 1 000 ms. - // - MAX = 1 000 ms: matches the previous flat constant; long - // enough that a never-arriving publisher doesn't hammer the - // relay, short enough to stay under the wrapper SharedFlow's - // ~1.3 s buffer once playback is rolling on the next attempt. + // - INITIAL = 100 ms: lower than the prior 250 ms because + // join-time logs (`claude/fix-nests-audio-receiver-HCgOY`) + // showed the listener was paying ~4 s of dead air on every + // first-join while the broadcaster's session warmed up + // on the relay (5 retries: 250+500+1000+1000+1000 = 3.75 s). + // 100 ms initial halves that to ~1.9 s + // (100+200+400+800+1000) without thrashing the relay on a + // permanently-gone publisher (the MAX cap below still + // bounds the total per-attempt rate). + // - Doubles each miss → 200 ms → 400 ms → 800 ms → 1 000 ms. + // - MAX = 1 000 ms: long enough that a never-arriving + // publisher doesn't hammer the relay; short enough to stay + // under the wrapper SharedFlow's ~1.3 s buffer once playback + // is rolling on the next attempt. // - Reset on first successful subscribe so a subsequent // publisher-cycle gap doesn't inherit the previous saturation. - private const val SUBSCRIBE_RETRY_BACKOFF_INITIAL_MS = 250L + private const val SUBSCRIBE_RETRY_BACKOFF_INITIAL_MS = 100L private const val SUBSCRIBE_RETRY_BACKOFF_MAX_MS = 1_000L private val SYNTH_OK = diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt index 2b8dab09c..f764ed316 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt @@ -251,6 +251,14 @@ class NestMoqLiteBroadcaster( "broadcaster sent frame #$sentFrames (group $framesInCurrentGroup/$framesPerGroup)" } } + // Only tap the speaking-ring on a frame + // that actually reached an inbound + // subscriber. Without this gate the local + // ring lights up during the pre-subscribe + // window AND after a relay-side cliff — + // the speaker would believe they're being + // heard when no audio is on the wire. + onLevel(peakAmplitude(pcm)) } else { droppedNoSubFrames += 1 if (droppedNoSubFrames % SEND_LOG_THROTTLE == 0L) { @@ -259,12 +267,6 @@ class NestMoqLiteBroadcaster( } } } - // Fire the speaking-ring tap only on - // a successful send. If the publisher - // throws (transport gone, peer dead), - // the frame didn't actually go out and - // the UI shouldn't claim we're talking. - onLevel(peakAmplitude(pcm)) }.onFailure { t -> if (t is CancellationException) throw t consecutiveSendErrors += 1 @@ -382,15 +384,32 @@ class NestMoqLiteBroadcaster( companion object { /** - * Default moq-lite group size = 5 Opus frames ≈ 100 ms of audio. - * Picked to keep the QUIC uni-stream creation rate - * (10 streams/sec at 20 ms cadence) under the production - * nostrnests relay's sustained per-subscriber forward - * ceiling (~40 streams/sec) while still giving late-joining - * subscribers a sub-100 ms initial audio gap. See - * [framesPerGroup] kdoc for the full rationale + history. + * Default moq-lite group size = 10 Opus frames ≈ 200 ms of audio. + * + * Picked to keep the QUIC uni-stream creation rate (5 streams/sec + * at 20 ms cadence) well under the production nostrnests relay's + * per-subscriber forward queue cliff. Two-phone production tests + * (`claude/fix-nests-audio-receiver-HCgOY` logcat) showed the + * relay still cliffed at ~135 streams under sustained load even + * with the old 5-frame default (10 streams/sec), giving ~13 s of + * audio before the listener's incoming uni-stream flow stopped + * — the same bug class documented in + * `nestsClient/plans/2026-05-01-quic-stream-cliff-investigation.md`, + * just shifted by half. Doubling the pack to 10 cuts the rate + * in half again and roughly doubles the cliff window in the + * worst case while halving the relay-side stream-handling + * pressure that triggers it. + * + * Belt-and-suspenders: pair this with the listener-side + * cliff-detector in `NestViewModel` (no-data-while-announced + * triggers `recycleSession()`) so the room recovers from a + * cliff event regardless of the exact threshold. + * + * Late-join gap: ≤ 200 ms (one group boundary) instead of + * ≤ 100 ms with `framesPerGroup = 5` — still imperceptible + * for live audio rooms. */ - const val DEFAULT_FRAMES_PER_GROUP: Int = 5 + const val DEFAULT_FRAMES_PER_GROUP: Int = 10 /** * Maximum consecutive [MoqLitePublisherHandle.send] / [endGroup] From f2205dc97e260565e13f95e6eccf88a920b5b550 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 5 May 2026 18:54:25 +0000 Subject: [PATCH 04/41] test(nests): pure unit tests for cliff-detector predicate + diagnostics + Connected restart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two-phone repro on commit cb61082 showed the cliff still triggers at ~13 s with framesPerGroup=10 (last drainOneGroup at ts 14:26:14, broadcaster keeps pushing through 14:26:25+ unimpeded), but the cliff detector did NOT fire — no `cliff-detector: announced+subscribed but silent…` log in the receiver's NestRx output despite the gap clearly exceeding the 4 s threshold. Three changes here narrow the gap. 1. Extract `computeStalledSpeakers` as a pure top-level internal function over `Map`. The detector loop now calls it; the rest of the predicate logic (cooldown, announced ∩ active ∩ stale, ≥ inclusive boundary) is now testable without standing up the NestViewModel coroutine machinery. 2. New `CliffDetectorTest` (commonTest, 12 tests) drives `computeStalledSpeakers` with a `kotlin.time.TestTimeSource`. Pins the boundary cases the next refactor would otherwise drift on: at-threshold inclusive, just-under empty, no-frame-yet ignored, not-announced ignored, multi-speaker classification, cooldown suppress, cooldown release, and a few more. All 12 pass. 3. Add diagnostic logging to the live cliff detector. The two-phone logs proved the predicate's logic works (the `lastFrameAt` mark was clearly aged past the threshold), so the silence has to be in the gating: `listener` null, connection state not Connected, or `_announcedSpeakers` not containing the pubkey at scan time. New logs: - "cliff-detector launched" once on start - "cliff-detector tick=N gated: listener=… conn=…" every 5 s when gated - "cliff-detector tick=N active=… announced=… lastFrameAges=…" every 5 s when running - "cliff-detector EXITED after N ticks (closed=…)" on cancel so the next repro tells us which gate is failing. 4. Defensive restart on `NestsListenerState.Connected`. The Closed branch in `observeListenerState` calls `teardown(targetState = Closed)`, which cancels `cliffDetectorJob` along with the rest of the per-session jobs. If the cliff detector itself is what triggered the recycle, the wrapper emits Closed → Reconnecting → Connected; we'd come back online with a dead detector and the next cliff event would slip past undetected. Calling `startCliffDetector()` (idempotent) on every Connected entry keeps the detector alive across the cycle. Defaults stay where they were: 4 s `ROOM_AUDIO_CLIFF_TIMEOUT_MS`, 1 s scan, 8 s cooldown — the unit tests also pin those default constants by relying on the function's defaults in two of the cases. --- .../commons/viewmodels/NestViewModel.kt | 164 ++++++++-- .../commons/viewmodels/CliffDetectorTest.kt | 309 ++++++++++++++++++ 2 files changed, 438 insertions(+), 35 deletions(-) create mode 100644 commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/CliffDetectorTest.kt diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt index bff80eb3e..efd73e927 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt @@ -277,7 +277,7 @@ class NestViewModel( * subscribe bidi is still open from QUIC's POV — meaning no other * layer notices. */ - private val lastFrameAt = mutableMapOf() + private val lastFrameAt = mutableMapOf() /** * Periodic scan of [activeSubscriptions] vs [_announcedSpeakers] @@ -298,7 +298,7 @@ class NestViewModel( * the new session — the new session has no incoming frames yet, * which would otherwise trip the detector again immediately. */ - private var lastCliffRecycleAt: kotlin.time.TimeSource.Monotonic.ValueTimeMark? = null + private var lastCliffRecycleAt: kotlin.time.TimeMark? = null /** * Per-speaker catalog-fetch coroutines. Each entry is the @@ -851,6 +851,18 @@ class NestViewModel( // this is typically a no-op in toAdd / toRemove // — runs anyway for the first-Connected path. reconcileSubscriptions() + // Defensive restart: if the wrapper went + // through a Closed transient (e.g. the cliff + // detector itself triggered `recycleSession()`, + // which closes the inner listener and waits + // for the orchestrator to reopen), our + // [Closed] branch below cancelled the + // detector via [teardown]. We need it back + // up on the fresh session — otherwise a + // second cliff event during the same room + // sits undetected. Idempotent: returns + // immediately if a job is already active. + startCliffDetector() } is NestsListenerState.Failed -> { @@ -1295,42 +1307,68 @@ class NestViewModel( */ private fun startCliffDetector() { if (cliffDetectorJob?.isActive == true) return + com.vitorpamplona.quartz.utils.Log + .d("NestRx") { "cliff-detector launched" } cliffDetectorJob = viewModelScope.launch { - while (true) { - delay(ROOM_AUDIO_CLIFF_CHECK_INTERVAL_MS) - if (closed) return@launch - val l = listener ?: continue - if (_uiState.value.connection !is ConnectionUiState.Connected) continue - // Cooldown: if we very recently triggered a recycle, - // give the orchestrator time to bring up the new - // session before re-checking. A new session has - // nothing in `lastFrameAt` yet which would otherwise - // immediately re-trigger. - val sinceRecycle = lastCliffRecycleAt?.elapsedNow() - if (sinceRecycle != null && - sinceRecycle.inWholeMilliseconds < ROOM_AUDIO_CLIFF_COOLDOWN_MS - ) { - continue + var ticks = 0L + try { + while (true) { + delay(ROOM_AUDIO_CLIFF_CHECK_INTERVAL_MS) + ticks += 1 + if (closed) return@launch + val l = listener + val connState = _uiState.value.connection + if (l == null || connState !is ConnectionUiState.Connected) { + // Periodic visibility into "why isn't the cliff + // detector acting" — the receiver-side cliff + // we're chasing has been silent in production + // logs even when the wire conditions look right, + // so dump our gating state every 5 s. + if (ticks % CLIFF_DIAG_LOG_EVERY == 0L) { + com.vitorpamplona.quartz.utils.Log.d("NestRx") { + "cliff-detector tick=$ticks gated: listener=${l != null} conn=${connState::class.simpleName}" + } + } + continue + } + val activeSpeakers = + activeSubscriptions + .asSequence() + .filter { (_, slot) -> slot.isPlaying } + .map { it.key } + .toSet() + val announced = _announcedSpeakers.value + if (ticks % CLIFF_DIAG_LOG_EVERY == 0L) { + val ages = + lastFrameAt.entries.joinToString { (k, mark) -> + "${k.take(8)}=${mark.elapsedNow().inWholeMilliseconds}ms" + } + com.vitorpamplona.quartz.utils.Log.d("NestRx") { + "cliff-detector tick=$ticks active=${activeSpeakers.size} announced=${announced.size} lastFrameAges=[$ages]" + } + } + val stalled = + computeStalledSpeakers( + activeSpeakers = activeSpeakers, + announcedSpeakers = announced, + lastFrameAt = lastFrameAt, + lastRecycleAt = lastCliffRecycleAt, + cliffTimeoutMs = ROOM_AUDIO_CLIFF_TIMEOUT_MS, + cooldownMs = ROOM_AUDIO_CLIFF_COOLDOWN_MS, + ) + if (stalled.isEmpty()) continue + com.vitorpamplona.quartz.utils.Log.w("NestRx") { + "cliff-detector: announced+subscribed but silent for ≥${ROOM_AUDIO_CLIFF_TIMEOUT_MS}ms — recycling session. stalled=$stalled" + } + lastCliffRecycleAt = + kotlin.time.TimeSource.Monotonic + .markNow() + runCatching { l.recycleSession() } } - val announced = _announcedSpeakers.value - val stalled = - activeSubscriptions - .asSequence() - .filter { (pubkey, slot) -> slot.isPlaying && pubkey in announced } - .filter { (pubkey, _) -> - val mark = lastFrameAt[pubkey] ?: return@filter false - mark.elapsedNow().inWholeMilliseconds >= ROOM_AUDIO_CLIFF_TIMEOUT_MS - }.map { it.key } - .toList() - if (stalled.isEmpty()) continue - com.vitorpamplona.quartz.utils.Log.w("NestRx") { - "cliff-detector: announced+subscribed but silent for ≥${ROOM_AUDIO_CLIFF_TIMEOUT_MS}ms — recycling session. stalled=$stalled" - } - lastCliffRecycleAt = - kotlin.time.TimeSource.Monotonic - .markNow() - runCatching { l.recycleSession() } + } finally { + com.vitorpamplona.quartz.utils.Log + .w("NestRx") { "cliff-detector EXITED after $ticks ticks (closed=$closed)" } } } } @@ -1634,6 +1672,62 @@ const val ROOM_AUDIO_CLIFF_CHECK_INTERVAL_MS: Long = 1_000L */ const val ROOM_AUDIO_CLIFF_COOLDOWN_MS: Long = 8_000L +/** + * Diagnostic log frequency for the cliff detector. Emit a state-dump + * line every Nth 1-second tick — i.e. every 5 s while connected and + * idle, so logcat captures the "active / announced / lastFrameAges" + * snapshot we use to debug silent-cliff cases without flooding when + * everything is fine. + */ +private const val CLIFF_DIAG_LOG_EVERY: Long = 5L + +/** + * Pure logic of the cliff detector — extracted so headless tests can + * exercise it with a [kotlin.time.TestTimeSource] without standing up + * the full [NestViewModel] coroutine machinery. + * + * Returns the list of pubkeys that should trigger a `recycleSession()` + * RIGHT NOW. Empty list means "no cliff observed; do nothing on this + * tick." + * + * Recycle conditions, all-of: + * - the pubkey has an active subscription on our side + * ([activeSpeakers]) — i.e. we're actively pulling its audio, + * - the relay has told us this pubkey is broadcasting + * ([announcedSpeakers]), + * - we've received at least one MoQ object in the past + * ([lastFrameAt] has an entry — we don't preemptively recycle + * a brand-new subscription that simply hasn't ramped up yet), + * - the elapsed time since that last object is at or past + * [cliffTimeoutMs]. + * + * Suppression: + * - if [lastRecycleAt] is non-null and less than [cooldownMs] has + * elapsed since it, returns empty (no cascading recycles while + * the wrapper is mid-handshake on a fresh session). + */ +internal fun computeStalledSpeakers( + activeSpeakers: Set, + announcedSpeakers: Set, + lastFrameAt: Map, + lastRecycleAt: kotlin.time.TimeMark?, + cliffTimeoutMs: Long = ROOM_AUDIO_CLIFF_TIMEOUT_MS, + cooldownMs: Long = ROOM_AUDIO_CLIFF_COOLDOWN_MS, +): List { + if (lastRecycleAt != null && + lastRecycleAt.elapsedNow().inWholeMilliseconds < cooldownMs + ) { + return emptyList() + } + return activeSpeakers + .asSequence() + .filter { it in announcedSpeakers } + .filter { pubkey -> + val mark = lastFrameAt[pubkey] ?: return@filter false + mark.elapsedNow().inWholeMilliseconds >= cliffTimeoutMs + }.toList() +} + /** * How long a kind-7 reaction stays in * [NestViewModel.recentReactions] before the eviction sweep diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/CliffDetectorTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/CliffDetectorTest.kt new file mode 100644 index 000000000..4b8f285c6 --- /dev/null +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/CliffDetectorTest.kt @@ -0,0 +1,309 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.viewmodels + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.ExperimentalTime +import kotlin.time.TestTimeSource + +/** + * Unit tests for the [computeStalledSpeakers] predicate that drives the + * listener-side cliff detector in [NestViewModel]. The predicate is + * extracted as a pure function specifically so we can drive it with a + * [TestTimeSource] — neither `runTest`'s virtual scheduler nor a real + * monotonic clock would let us deterministically place events relative + * to the 4 s `ROOM_AUDIO_CLIFF_TIMEOUT_MS` threshold. + * + * The detector defends against the moq-rs production-relay forward- + * queue cliff documented in + * `nestsClient/plans/2026-05-01-quic-stream-cliff-investigation.md`: + * the relay opens fewer (or zero) new uni streams to a slow + * subscriber while the announce stream still says the broadcast is + * Active and the subscribe bidi is still alive at QUIC level — no + * other layer notices. These tests pin the recycle gate so a future + * refactor doesn't accidentally widen or narrow when we recycle. + */ +@OptIn(ExperimentalTime::class) +class CliffDetectorTest { + @Test + fun returnsEmptyWhenNoSpeakersActive() { + val ts = TestTimeSource() + val result = + computeStalledSpeakers( + activeSpeakers = emptySet(), + announcedSpeakers = setOf(ALICE), + lastFrameAt = mapOf(ALICE to ts.markNow()), + lastRecycleAt = null, + ) + assertTrue(result.isEmpty()) + } + + @Test + fun ignoresSpeakerNotInAnnounced() { + // A speaker we're subscribed to but who isn't in the relay's + // announce-stream output is most likely either (a) already + // off stage and the relay just hasn't propagated the Ended + // yet, or (b) in the brief gap between an Ended and the + // wrapper re-issuing. Recycling the whole transport here + // would be wasteful — we only act on confirmed cliffs. + val ts = TestTimeSource() + val frameMark = ts.markNow() + ts += 10_000.milliseconds // way past threshold + + val result = + computeStalledSpeakers( + activeSpeakers = setOf(ALICE), + announcedSpeakers = emptySet(), + lastFrameAt = mapOf(ALICE to frameMark), + lastRecycleAt = null, + ) + assertTrue(result.isEmpty()) + } + + @Test + fun ignoresSpeakerWithNoFrameYet() { + // Brand-new subscription that hasn't ramped up: lastFrameAt + // has no entry. Don't recycle — the wrapper's per-speaker + // re-issue + opener-throws backoff already retries on its + // own; the cliff detector only acts once we've proven the + // subscription was working. + val ts = TestTimeSource() + ts += 10_000.milliseconds + + val result = + computeStalledSpeakers( + activeSpeakers = setOf(ALICE), + announcedSpeakers = setOf(ALICE), + lastFrameAt = emptyMap(), + lastRecycleAt = null, + ) + assertTrue(result.isEmpty()) + } + + @Test + fun returnsEmptyJustBeforeThreshold() { + // Boundary case: 1 ms under the 4 s threshold. The detector + // should still consider this healthy. Relevant because audio + // groups arrive at ~100 ms cadence (framesPerGroup=10), so a + // false positive at ≈ threshold would recycle on every + // group-rollover hiccup. + val ts = TestTimeSource() + val frameMark = ts.markNow() + ts += 3_999.milliseconds + + val result = + computeStalledSpeakers( + activeSpeakers = setOf(ALICE), + announcedSpeakers = setOf(ALICE), + lastFrameAt = mapOf(ALICE to frameMark), + lastRecycleAt = null, + ) + assertTrue(result.isEmpty()) + } + + @Test + fun returnsStalledSpeakerAtThresholdInclusive() { + // Exactly at threshold: include. The check is `>=`, not `>`. + // Tested explicitly because boundary semantics here are + // user-visible (this is the "audio went silent" detection). + val ts = TestTimeSource() + val frameMark = ts.markNow() + ts += 4_000.milliseconds + + val result = + computeStalledSpeakers( + activeSpeakers = setOf(ALICE), + announcedSpeakers = setOf(ALICE), + lastFrameAt = mapOf(ALICE to frameMark), + lastRecycleAt = null, + ) + assertEquals(listOf(ALICE), result) + } + + @Test + fun returnsStalledSpeakerWellPastThreshold() { + val ts = TestTimeSource() + val frameMark = ts.markNow() + ts += 30_000.milliseconds + + val result = + computeStalledSpeakers( + activeSpeakers = setOf(ALICE), + announcedSpeakers = setOf(ALICE), + lastFrameAt = mapOf(ALICE to frameMark), + lastRecycleAt = null, + ) + assertEquals(listOf(ALICE), result) + } + + @Test + fun returnsAllStalledSpeakersAtOnceMixedWithFreshOnes() { + // Multi-speaker stage: alice and bob have stalled, charlie's + // subscription is fresh (just received a frame). Recycle + // should fire for the stalled set and leave charlie alone — + // the `recycleSession` itself takes the whole listener down, + // but the test here is the predicate's classification. + val ts = TestTimeSource() + val aliceFrame = ts.markNow() + val bobFrame = ts.markNow() + ts += 3_000.milliseconds + val charlieFrame = ts.markNow() + ts += 3_000.milliseconds + + val result = + computeStalledSpeakers( + activeSpeakers = setOf(ALICE, BOB, CHARLIE), + announcedSpeakers = setOf(ALICE, BOB, CHARLIE), + lastFrameAt = + mapOf( + ALICE to aliceFrame, + BOB to bobFrame, + CHARLIE to charlieFrame, + ), + lastRecycleAt = null, + ) + assertEquals(setOf(ALICE, BOB), result.toSet()) + } + + @Test + fun cooldownSuppressesRecycleEvenWhenStalled() { + // After a recycle, the wrapper opens a fresh QUIC transport. + // The new session has no `lastFrameAt` entries yet for any + // pubkey; without a cooldown we would re-trigger on the + // very next 1 s tick because the prior tick's stalled + // pubkeys still age past the threshold (their lastFrameAt + // hasn't been updated by the new session yet). 8 s cooldown + // covers the typical reconnect handshake. + val ts = TestTimeSource() + val frameMark = ts.markNow() + ts += 4_500.milliseconds // past threshold + val recycleMark = ts.markNow() // recycle just fired + ts += 1_000.milliseconds // 1 s into cooldown + + val result = + computeStalledSpeakers( + activeSpeakers = setOf(ALICE), + announcedSpeakers = setOf(ALICE), + lastFrameAt = mapOf(ALICE to frameMark), + lastRecycleAt = recycleMark, + ) + assertTrue(result.isEmpty()) + } + + @Test + fun cooldownReleasesAfterTimeoutPasses() { + // Once cooldown elapses, a still-stalled subscription + // becomes eligible to recycle again. Important for the + // case where the recycle didn't actually fix the cliff — + // we want a second attempt rather than getting wedged. + val ts = TestTimeSource() + val frameMark = ts.markNow() + ts += 4_500.milliseconds + val recycleMark = ts.markNow() + ts += 8_001.milliseconds // 1 ms past cooldown + + val result = + computeStalledSpeakers( + activeSpeakers = setOf(ALICE), + announcedSpeakers = setOf(ALICE), + lastFrameAt = mapOf(ALICE to frameMark), + lastRecycleAt = recycleMark, + ) + assertEquals(listOf(ALICE), result) + } + + @Test + fun customTimeoutsAreHonored() { + // Defaults are wired into the production VM, but the function + // accepts overrides so a test for tighter / looser tolerances + // can drive it without mutating the constants. + val ts = TestTimeSource() + val frameMark = ts.markNow() + ts += 1_000.milliseconds + + val withDefault = + computeStalledSpeakers( + activeSpeakers = setOf(ALICE), + announcedSpeakers = setOf(ALICE), + lastFrameAt = mapOf(ALICE to frameMark), + lastRecycleAt = null, + ) + assertTrue(withDefault.isEmpty(), "1 s elapsed shouldn't trip 4 s default") + + val withTightTimeout = + computeStalledSpeakers( + activeSpeakers = setOf(ALICE), + announcedSpeakers = setOf(ALICE), + lastFrameAt = mapOf(ALICE to frameMark), + lastRecycleAt = null, + cliffTimeoutMs = 500L, + ) + assertEquals(listOf(ALICE), withTightTimeout, "1 s elapsed should trip a 500 ms threshold") + } + + @Test + fun activeSpeakerNotInLastFrameAtIsIgnoredEvenWithStalledPeers() { + // Mixed: alice is announced + active + stalled, bob is + // announced + active but has no frame yet. Detector should + // return alice only — bob hasn't proven the subscription + // works, so it's not "stalled" yet, just slow to start. + val ts = TestTimeSource() + val aliceFrame = ts.markNow() + ts += 5_000.milliseconds + + val result = + computeStalledSpeakers( + activeSpeakers = setOf(ALICE, BOB), + announcedSpeakers = setOf(ALICE, BOB), + lastFrameAt = mapOf(ALICE to aliceFrame), + lastRecycleAt = null, + ) + assertEquals(listOf(ALICE), result) + } + + @Test + fun nullLastRecycleNeverSuppresses() { + // Null `lastRecycleAt` means "we've never recycled" — the + // very first cliff event of a session must fire. The + // cooldown check is gated on non-null. + val ts = TestTimeSource() + val frameMark = ts.markNow() + ts += 5_000.milliseconds + + val result = + computeStalledSpeakers( + activeSpeakers = setOf(ALICE), + announcedSpeakers = setOf(ALICE), + lastFrameAt = mapOf(ALICE to frameMark), + lastRecycleAt = null, + ) + assertEquals(listOf(ALICE), result) + } + + companion object { + private val ALICE = "a".repeat(64) + private val BOB = "b".repeat(64) + private val CHARLIE = "c".repeat(64) + } +} From d6517cf346d54d91b841f46037e77e87e25dee52 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 5 May 2026 19:11:51 +0000 Subject: [PATCH 05/41] fix(nests): announce SharedFlow late-attach race dropped Active updates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cliff-detector diagnostic logs from the user's two-phone repro (commit f2205dc, run 15:06) showed the smoking gun — every tick from the receiver-side cliff detector said `announced=0` even though an Active announce had clearly been received earlier (the session-level `pumpAnnounceWatch` logged it). The cliff detector only acts on speakers that are in `_announcedSpeakers`, so it never tripped, even when frames clearly stopped flowing for 6+ s. Root cause: `MoqLiteSession.announce` builds a `MutableSharedFlow` with `replay = 0` to bridge the bidi pump and the consumer's `collect`. With Kotlin's `MutableSharedFlow(replay = 0)`, an `emit` that happens BEFORE the first collector subscribes is silently dropped — there's no buffer for items emitted into a flow with no subscribers. The bidi pump is launched inside `announce()` and starts collecting `bidi.incoming()` immediately; the caller (the flow returned by `MoqLiteNestsListener.announces()`) attaches its collector AFTER `session.announce()` returns. In between, if the relay sends an Active update (which it does the moment the broadcaster's session is registered), the pump's emit hits a flow with no collectors and is lost. In the user's logs both an internal session-level `pumpAnnounceWatch` AND the VM-level `observeAnnounces` open separate announce bidis. The internal one started first and won the race for its bidi's Active. The VM-level one lost — the bidi pump emitted Active before the consumer subscribed, the SharedFlow dropped it, and `_announcedSpeakers` never populated. Fix: change the announce SharedFlow to `replay = 64` with `BufferOverflow.DROP_OLDEST`. Late-attaching collectors now see up to the last 64 announces (far more than nests rooms have speakers — typical stage size is ≤ 10). The bidi pump's emit still never suspends, so QUIC-level backpressure can't propagate. The replay cache evicts the oldest item if more than 64 announces arrive before any collector attaches; for the production workload this is unreachable. Tests: nestsClient jvmTest passes 57/57 unchanged (MoqLiteCodecTest, MoqLiteFrameBufferTest, MoqLitePathTest, MoqLiteSessionTest, NestBroadcasterTest, NestPlayerTest). The session-level pumpAnnounceWatch path used the same SharedFlow but its caller subscribes from `scope.launch { pumpAnnounceWatch(...) }` inside `ensureAnnounceWatchStarted` — close enough in time that it won the race in production. The fix closes the race for both callers regardless of timing. --- .../nestsclient/moq/lite/MoqLiteSession.kt | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt index 6fc3a2755..54c50b88b 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt @@ -119,7 +119,23 @@ class MoqLiteSession internal constructor( bidi.write(Varint.encode(MoqLiteControlType.Announce.code)) bidi.write(MoqLiteCodec.encodeAnnouncePlease(MoqLiteAnnouncePlease(prefix))) - val updates = MutableSharedFlow(replay = 0, extraBufferCapacity = 64) + // replay=64, DROP_OLDEST: announces emitted before the + // caller's `collect` attaches MUST NOT be dropped — that's + // the late-attach race that left `_announcedSpeakers` empty + // in production receiver logs even after a clear Active + // arrived (the session-internal pumpAnnounceWatch beat the + // VM-level `observeAnnounces` to subscribing, then the + // VM-level bidi's pump emitted to a SharedFlow with no + // collector yet, and the prior `replay=0` silently dropped + // it). Replay buffer covers up to 64 distinct announces — + // far more than nests rooms have speakers — and DROP_OLDEST + // means the bidi pump never has to suspend on emit, so + // backpressure can't propagate up into the QUIC read loop. + val updates = + MutableSharedFlow( + replay = 64, + onBufferOverflow = BufferOverflow.DROP_OLDEST, + ) val pump = scope.launch { val buffer = MoqLiteFrameBuffer() From 1fc8dbceeebede46cfd133727f117063a6e7988f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 5 May 2026 19:27:19 +0000 Subject: [PATCH 06/41] debug(nests): trace announce-flow chain to localise still-empty _announcedSpeakers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The replay=64 SharedFlow fix in d6517cf did NOT close the receiver's "announced=0" symptom — the next two-phone repro (15:20:08 onward) still shows the cliff detector ticking with `active=1 announced=0` indefinitely, even after the session-internal pumpAnnounceWatch clearly received an Active update. Either moq-rs sends Active to only the FIRST announce bidi (so the VM-level second bidi never gets it), OR the chain from that bidi to `_announcedSpeakers` is broken at a layer my last fix didn't visit. Adds focused logging at every step of the announce chain so the next repro pins down which one. All NestRx-tagged: `MoqLiteSession.announce` (per-bidi pump): - "session.announce(prefix='…') bidi opened, pump launching" - "session.announce(prefix='…') bidi pump emit #N status=… suffix='…' chunks=…" - "session.announce(prefix='…') bidi.incoming() ended naturally chunks=… emits=…" - per-emission so we can compare bidi#1 (session-internal) vs bidi#2 (VM-level) emit counts. If bidi#2 has 0 emits, the relay is sending Actives to only one bidi. `MoqLiteNestsListener.announces`: - "MoqLiteNestsListener.announces flow STARTING (state=…)" - "MoqLiteNestsListener.announces opened bidi#2 (VM-level)" - per-emission "bidi#2 #N status=… suffix='…'" - "bidi#2 collect ENDED naturally" / "finally emissions=N" `ReconnectingNestsListener.announces` (wrapper): - "wrapper.announces() flow starting collect on activeListener" - "wrapper.announces() iter=N activeListener=set/null" - "wrapper.announces() iter=N inner state=Connected/Closed/…" - "wrapper.announces() iter=N inner collect ended naturally/X fwd=N" `NestViewModel.observeAnnounces`: - "observeAnnounces launching collect on l.announces()" - per-emission "observeAnnounces #N active=… pubkey='…' → ADD/REMOVE" - "observeAnnounces collect ENDED naturally/X (emissions=N, _announcedSpeakers.size=…)" After the next repro, the receiver-side log will show one of: - bidi#2 never opens (terminalOrConnected != Connected somehow) - bidi#2 opens, no chunks/emits → relay-side issue - bidi#2 emits, but `MoqLiteNestsListener.announces` doesn't forward → bug in the flow body - forwarding works but observeAnnounces collect ends → bug in `ReconnectingNestsListener.announces` collectLatest behavior - everything works but `_announcedSpeakers.size=0` → mutation lost Tests: full suite still passes (CliffDetectorTest 12, NestPlayerTest 12, NestBroadcasterTest 6, MoqLiteSessionTest 11, …). --- .../commons/viewmodels/NestViewModel.kt | 26 ++++++++++++++----- .../nestsclient/MoqLiteNestsListener.kt | 13 ++++++++++ .../nestsclient/ReconnectingNestsListener.kt | 18 +++++++++++-- .../nestsclient/moq/lite/MoqLiteSession.kt | 16 ++++++++++-- 4 files changed, 62 insertions(+), 11 deletions(-) diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt index efd73e927..6776346a0 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt @@ -820,15 +820,27 @@ class NestViewModel( announcesJob?.cancel() announcesJob = viewModelScope.launch { - runCatching { - l.announces().collect { ann -> - if (closed) return@collect - if (ann.active) { - _announcedSpeakers.update { it + ann.pubkey } - } else { - _announcedSpeakers.update { it - ann.pubkey } + com.vitorpamplona.quartz.utils.Log + .d("NestRx") { "observeAnnounces launching collect on l.announces()" } + var emissionCount = 0 + val outcome = + runCatching { + l.announces().collect { ann -> + if (closed) return@collect + emissionCount += 1 + com.vitorpamplona.quartz.utils.Log.d("NestRx") { + "observeAnnounces #$emissionCount active=${ann.active} pubkey='${ann.pubkey.take(12)}' → ${if (ann.active) "ADD" else "REMOVE"}" + } + if (ann.active) { + _announcedSpeakers.update { it + ann.pubkey } + } else { + _announcedSpeakers.update { it - ann.pubkey } + } } } + com.vitorpamplona.quartz.utils.Log.w("NestRx") { + val why = outcome.exceptionOrNull()?.let { "${it::class.simpleName}: ${it.message}" } ?: "naturally" + "observeAnnounces collect ENDED $why (emissions=$emissionCount, _announcedSpeakers.size=${_announcedSpeakers.value.size})" } } } diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsListener.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsListener.kt index 5986ab11c..3ee7fcf97 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsListener.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsListener.kt @@ -128,6 +128,8 @@ class MoqLiteNestsListener internal constructor( override fun announces(): Flow = flow { + com.vitorpamplona.quartz.utils.Log + .d("NestRx") { "MoqLiteNestsListener.announces flow STARTING (state=${state.value::class.simpleName})" } check(state.value is NestsListenerState.Connected) { "NestsListener.announces requires Connected state, was ${state.value}" } @@ -136,8 +138,15 @@ class MoqLiteNestsListener internal constructor( // suffix is the broadcast's path component within the // room — for nests this is the speaker pubkey hex. val handle = session.announce(prefix = "") + com.vitorpamplona.quartz.utils.Log + .d("NestRx") { "MoqLiteNestsListener.announces opened bidi#2 (VM-level)" } + var emissionCount = 0 try { handle.updates.collect { announce -> + emissionCount += 1 + com.vitorpamplona.quartz.utils.Log.d("NestRx") { + "MoqLiteNestsListener.announces bidi#2 #$emissionCount status=${announce.status} suffix='${announce.suffix.take(12)}'" + } emit( RoomAnnouncement( pubkey = announce.suffix, @@ -145,7 +154,11 @@ class MoqLiteNestsListener internal constructor( ), ) } + com.vitorpamplona.quartz.utils.Log + .w("NestRx") { "MoqLiteNestsListener.announces bidi#2 collect ENDED naturally after $emissionCount emissions" } } finally { + com.vitorpamplona.quartz.utils.Log + .w("NestRx") { "MoqLiteNestsListener.announces bidi#2 finally (emissions=$emissionCount)" } // The flow's `finally` runs on both normal completion // and cancellation — let cancel propagate after closing // the announce handle. diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsListener.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsListener.kt index 1b1b9b7a8..71bcd9b64 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsListener.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsListener.kt @@ -252,7 +252,11 @@ private class ReconnectingHandle( */ override fun announces(): Flow = flow { + Log.d("NestRx") { "wrapper.announces() flow starting collect on activeListener" } + var iter = 0 activeListener.collectLatest { listener -> + iter += 1 + Log.d("NestRx") { "wrapper.announces() iter=$iter activeListener=${if (listener == null) "null" else "set"}" } if (listener == null) return@collectLatest val terminalOrConnected = listener.state.first { state -> @@ -260,9 +264,19 @@ private class ReconnectingHandle( state is NestsListenerState.Closed || state is NestsListenerState.Failed } + Log.d("NestRx") { "wrapper.announces() iter=$iter inner state=${terminalOrConnected::class.simpleName}" } if (terminalOrConnected !is NestsListenerState.Connected) return@collectLatest - runCatching { - listener.announces().collect { emit(it) } + var fwd = 0 + val outcome = + runCatching { + listener.announces().collect { + fwd += 1 + emit(it) + } + } + Log.w("NestRx") { + val why = outcome.exceptionOrNull()?.let { "${it::class.simpleName}: ${it.message}" } ?: "naturally" + "wrapper.announces() iter=$iter inner collect ended $why fwd=$fwd" } } } diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt index 54c50b88b..903c7526e 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt @@ -136,21 +136,33 @@ class MoqLiteSession internal constructor( replay = 64, onBufferOverflow = BufferOverflow.DROP_OLDEST, ) + Log.d("NestRx") { "session.announce(prefix='$prefix') bidi opened, pump launching (replayCap=64)" } val pump = scope.launch { val buffer = MoqLiteFrameBuffer() + var chunkCount = 0 + var emitCount = 0 try { bidi.incoming().collect { chunk -> + chunkCount += 1 buffer.push(chunk) while (true) { val payload = buffer.readSizePrefixed() ?: break - updates.emit(MoqLiteCodec.decodeAnnounce(payload)) + val decoded = MoqLiteCodec.decodeAnnounce(payload) + emitCount += 1 + Log.d("NestRx") { + "session.announce(prefix='$prefix') bidi pump emit #$emitCount " + + "status=${decoded.status} suffix='${decoded.suffix.take(12)}' " + + "(chunks=$chunkCount)" + } + updates.emit(decoded) } } + Log.w("NestRx") { "session.announce(prefix='$prefix') bidi.incoming() ended naturally (chunks=$chunkCount, emits=$emitCount)" } } catch (ce: CancellationException) { throw ce } catch (t: Throwable) { - Log.w("NestRx") { "announce(prefix='$prefix'): bidi.incoming() threw ${t::class.simpleName}: ${t.message}" } + Log.w("NestRx") { "announce(prefix='$prefix'): bidi.incoming() threw ${t::class.simpleName}: ${t.message} (chunks=$chunkCount, emits=$emitCount)" } // Flow terminated (peer FIN or transport close). // The Announce stream's emit-side just stops; consumers // see an end-of-flow. From f17e7adfa7acdb2bae9f666254f2aef2bf164756 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 5 May 2026 19:39:23 +0000 Subject: [PATCH 07/41] fix(nests): announces flow MUST use channelFlow, not flow{} MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The diagnostic logging in 1fc8dbc surfaced the real root cause — not the SharedFlow replay race I fixed in d6517cf, but a strict flow{}-builder concurrency-confinement violation. The receiver-side log (15:34:40, repro after the diagnostic patch) showed: session.announce(prefix='') bidi pump emit #1 status=Active suffix='fe52579aa30e' (chunks=1) MoqLiteNestsListener.announces bidi#2 #1 status=Active suffix='fe52579aa30e' MoqLiteNestsListener.announces bidi#2 finally (emissions=1) wrapper.announces() iter=2 inner collect ended IllegalStateException: Flow invariant is violated: Emission from another coroutine is detected. Child of StandaloneCoroutine{Active}@e9b5981, expected child of StandaloneCoroutine{Active}@8309a26. FlowCollector is not thread-safe and concurrent emissions are prohibited. To mitigate this restriction please use 'channelFlow' builder instead of 'flow' The session's bidi pump (`scope.launch { bidi.incoming().collect … updates.emit(decoded) }`) emits to the announce SharedFlow from the PUMP's coroutine. SharedFlow's `collect { … }` resumes the lambda inline on the emitter's coroutine when there's no dispatcher hop, so `MoqLiteNestsListener.announces`' `handle.updates.collect { emit(RoomAnnouncement(…)) }` lambda fires on the pump's coroutine, not the flow{} body's coroutine. `flow {}`'s SafeFlow guard throws on the cross-coroutine emit. The wrapper's `runCatching { listener.announces().collect { emit(it) } }` swallows the exception, the wrapper's collect ends with `fwd=1`, and `_announcedSpeakers` stays empty for the lifetime of the session. That, in turn, kept the cliff detector permanently gated: cliff-detector tick=N active=1 announced=0 lastFrameAges=[fe52579a=…ms] even though the receiver was clearly subscribed and clearly seeing frames. The cliff detector requires the speaker to be in `announcedSpeakers` to recycle, so the relay-forward cliff at ~135 streams went undetected — the original two-phone receiver-side silence symptom. Fix: switch `MoqLiteNestsListener.announces()` from `flow { … }` to `channelFlow { … }`. `channelFlow` is the documented escape hatch for cross-coroutine emission (the error message itself recommends it). The new shape: channelFlow { val handle = session.announce(prefix = "") val pump = launch { try { handle.updates.collect { send(RoomAnnouncement(…)) } } finally { withContext(NonCancellable) { runCatching { handle.close() } } } } awaitClose { pump.cancel() } } Notes: - `send` (channel) replaces `emit` (flow) — channelFlow buffers via a Channel, so cross-coroutine producers are explicitly supported. - The inner `collect` is wrapped in a launched child of the channelFlow scope so we can use `awaitClose` for the producer- side teardown contract channelFlow requires. - `handle.close` is suspend (FINs the bidi + joins the pump); putting it in the launched pump's `finally` under `NonCancellable` keeps the FIN reliable when the consumer cancels mid-stream. Diagnostic logging from 1fc8dbc preserved — the next two-phone repro should now show: observeAnnounces #1 active=true pubkey='fe52579aa30e' → ADD cliff-detector tick=N active=1 announced=1 lastFrameAges=[…] …and on a real cliff event, the recycle should fire. Tests: full suite still passes - CliffDetectorTest: 12/12 - NestPlayerTest: 12/12 - NestBroadcasterTest: 6/6 - MoqLiteSessionTest: 11/11 - All other nestsClient jvmTest classes (~240 tests across 37 classes) green. --- .../nestsclient/MoqLiteNestsListener.kt | 98 ++++++++++++------- 1 file changed, 65 insertions(+), 33 deletions(-) diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsListener.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsListener.kt index 3ee7fcf97..6b6c308bb 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsListener.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsListener.kt @@ -26,12 +26,16 @@ import com.vitorpamplona.nestsclient.moq.SubscribeOk import com.vitorpamplona.nestsclient.moq.lite.MoqLiteAnnounceStatus import com.vitorpamplona.nestsclient.moq.lite.MoqLiteSession import com.vitorpamplona.nestsclient.moq.lite.MoqLiteSubscribeException +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.channels.awaitClose import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow -import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.channelFlow import kotlinx.coroutines.flow.map +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext import java.util.concurrent.atomic.AtomicLong /** @@ -127,9 +131,26 @@ class MoqLiteNestsListener internal constructor( } override fun announces(): Flow = - flow { - com.vitorpamplona.quartz.utils.Log - .d("NestRx") { "MoqLiteNestsListener.announces flow STARTING (state=${state.value::class.simpleName})" } + // `channelFlow` (NOT `flow`) is mandatory here. The downstream + // emission source is `handle.updates`, a MutableSharedFlow + // populated by a launched bidi pump on the session's scope. + // SharedFlow's `collect { lambda }` resumes the lambda inline + // when emit happens — so when the pump emits a chunk, the + // collect lambda runs on the pump's coroutine, not the + // collector's. `flow {}` enforces the "emit only from the + // builder coroutine" invariant and throws + // `IllegalStateException: Flow invariant is violated` the + // moment we call `emit()` from a different coroutine. Two- + // phone production logs (commit 1fc8dbc, run 15:34:40) + // showed exactly this — the first Active arrived, the + // collect lambda fired, emit() threw, the wrapper's + // `runCatching { listener.announces().collect { emit(it) } }` + // swallowed it, `_announcedSpeakers` stayed empty forever, + // and the cliff detector reported `active=1 announced=0` + // indefinitely. `channelFlow` is the documented fix + // (the error message itself recommends it): emissions go + // through a buffered channel that's safe across coroutines. + channelFlow { check(state.value is NestsListenerState.Connected) { "NestsListener.announces requires Connected state, was ${state.value}" } @@ -138,37 +159,48 @@ class MoqLiteNestsListener internal constructor( // suffix is the broadcast's path component within the // room — for nests this is the speaker pubkey hex. val handle = session.announce(prefix = "") - com.vitorpamplona.quartz.utils.Log - .d("NestRx") { "MoqLiteNestsListener.announces opened bidi#2 (VM-level)" } - var emissionCount = 0 - try { - handle.updates.collect { announce -> - emissionCount += 1 - com.vitorpamplona.quartz.utils.Log.d("NestRx") { - "MoqLiteNestsListener.announces bidi#2 #$emissionCount status=${announce.status} suffix='${announce.suffix.take(12)}'" + // Run the inner collect on a child of channelFlow's scope + // so we can `awaitClose` on the producer side and let + // close handle the unsub side-effect. Without the launch, + // we'd block here in `collect` forever and never reach + // `awaitClose` — but channelFlow's contract requires + // awaitClose for clean shutdown. + val pump = + launch { + try { + handle.updates.collect { announce -> + send( + RoomAnnouncement( + pubkey = announce.suffix, + active = announce.status == MoqLiteAnnounceStatus.Active, + ), + ) + } + } catch (ce: kotlinx.coroutines.CancellationException) { + throw ce + } catch (_: Throwable) { + // updates flow died — close the channel so the + // consumer sees end-of-flow and the wrapper + // can decide what to do (typically wait for + // the next activeListener emission). + } finally { + // Close the announce bidi on the same scope + // that owned the pump. `handle.close` is + // suspend (it FINs the bidi and joins the + // session-side pump); run it under + // NonCancellable so cancellation of this + // coroutine doesn't skip the FIN. + withContext(NonCancellable) { + runCatching { handle.close() } + } } - emit( - RoomAnnouncement( - pubkey = announce.suffix, - active = announce.status == MoqLiteAnnounceStatus.Active, - ), - ) - } - com.vitorpamplona.quartz.utils.Log - .w("NestRx") { "MoqLiteNestsListener.announces bidi#2 collect ENDED naturally after $emissionCount emissions" } - } finally { - com.vitorpamplona.quartz.utils.Log - .w("NestRx") { "MoqLiteNestsListener.announces bidi#2 finally (emissions=$emissionCount)" } - // The flow's `finally` runs on both normal completion - // and cancellation — let cancel propagate after closing - // the announce handle. - try { - handle.close() - } catch (ce: kotlinx.coroutines.CancellationException) { - throw ce - } catch (_: Throwable) { - // Best-effort. } + awaitClose { + // Caller cancelled, OR the wrapping `collectLatest` + // restarted (listener swap). Cancel the pump; its + // finally above runs the suspend close on a + // NonCancellable context. + pump.cancel() } } From 457e0f5997e122068e77cf3f105544b8ccad35a5 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 5 May 2026 19:50:23 +0000 Subject: [PATCH 08/41] fix(nests): wrapper.announces() ALSO needs channelFlow (collectLatest emits cross-coroutine) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The channelFlow conversion in f17e7ad fixed `MoqLiteNestsListener.announces` but the SAME `IllegalStateException: Flow invariant is violated` kept firing in the next two-phone repro (commit f17e7ad, run 15:46:51) — this time the offending `flow{}` was one layer up, in `ReconnectingNestsListener.announces`: override fun announces(): Flow = flow { activeListener.collectLatest { listener -> ... runCatching { listener.announces().collect { emit(it) } // <-- HERE } } } `Flow.collectLatest { lambda }` cancels and restarts a CHILD coroutine for each upstream emission. The lambda body runs in that child, NOT in the surrounding flow{} body's coroutine. So when the lambda invokes `emit(it)`, it's emitting from a child coroutine. `flow{}`'s SafeFlow guard rejects this with the same "Emission from another coroutine is detected" error the inner listener was throwing before my last fix. Net effect on the user's repro: the inner channelFlow now correctly sends RoomAnnouncement to the wrapper's `listener.announces().collect` lambda, but that lambda's `emit(it)` to the wrapper's flow{} body fails the same SafeFlow check, the wrapper's runCatching swallows the exception (as `iter=2 inner collect ended IllegalStateException ... fwd=1`), `_announcedSpeakers` stays empty, cliff detector never fires. Fix: convert `ReconnectingNestsListener.announces` from `flow{} + emit` to `channelFlow{} + send`, matching the inner listener's shape. The combination of `channelFlow + collectLatest + send` is the canonical pattern in kotlinx-coroutines for "switch-map" semantics with cross- coroutine production. `awaitClose { }` is empty because `collectLatest` on an infinite-StateFlow never completes naturally — cancellation propagates through structured concurrency when the consumer cancels the channelFlow. Tests: every nestsClient + commons unit test still passes, including the 12 CliffDetectorTest cases pinning the predicate's behaviour. Together with f17e7ad (inner channelFlow), this should finally close the chain: inner emits RoomAnnouncement on its channelFlow → wrapper's collectLatest receives it inside its child coroutine → wrapper's channelFlow.send forwards across the channel → consumer's observeAnnounces collect receives → `_announcedSpeakers` populates → cliff detector tick reports `announced=1` → on a real cliff event the recycle fires. --- .../nestsclient/ReconnectingNestsListener.kt | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsListener.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsListener.kt index 71bcd9b64..6ec082b75 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsListener.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsListener.kt @@ -29,6 +29,7 @@ import com.vitorpamplona.quartz.utils.Log import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.channels.BufferOverflow +import kotlinx.coroutines.channels.awaitClose import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.delay import kotlinx.coroutines.flow.Flow @@ -37,9 +38,9 @@ import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asSharedFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.channelFlow import kotlinx.coroutines.flow.collectLatest import kotlinx.coroutines.flow.first -import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.isActive import kotlinx.coroutines.launch @@ -251,8 +252,25 @@ private class ReconnectingHandle( * swallowed so a future moq-lite session keeps emitting. */ override fun announces(): Flow = - flow { - Log.d("NestRx") { "wrapper.announces() flow starting collect on activeListener" } + // `channelFlow` (NOT `flow`) is required here for the same + // reason `MoqLiteNestsListener.announces` is channelFlow: + // we drive emissions from a `collectLatest` that internally + // launches a child coroutine per `activeListener` emission. + // `flow {}`'s SafeFlow guard rejects emissions from any + // coroutine other than the flow-builder's own — so on the + // very first listener emission the inner `emit(it)` lands + // in a child coroutine of `collectLatest`, throws + // IllegalStateException, the consumer's `runCatching` + // swallows it, and `_announcedSpeakers` never populates. + // Two-phone production logs at commit f17e7ad showed this + // exact failure even AFTER the inner listener was switched + // to channelFlow — the wrapper layer was the second + // un-fixed `flow {}`. `channelFlow` + `send(...)` instead + // of `emit(...)` allows cross-coroutine production, which + // is exactly what `collectLatest`'s per-emission child + // coroutines need. + channelFlow { + Log.d("NestRx") { "wrapper.announces() channelFlow starting collect on activeListener" } var iter = 0 activeListener.collectLatest { listener -> iter += 1 @@ -271,7 +289,7 @@ private class ReconnectingHandle( runCatching { listener.announces().collect { fwd += 1 - emit(it) + send(it) } } Log.w("NestRx") { @@ -279,6 +297,12 @@ private class ReconnectingHandle( "wrapper.announces() iter=$iter inner collect ended $why fwd=$fwd" } } + // collectLatest above never returns naturally — it + // collects activeListener forever. awaitClose fires when + // the consumer cancels the channelFlow, at which point + // the collectLatest is cancelled too via structured + // concurrency. + awaitClose { } } /** From ea08c43b71455d24045b761312ad88ed187d9611 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 5 May 2026 21:27:35 +0000 Subject: [PATCH 09/41] =?UTF-8?q?fix(nests):=20don't=20teardown=20on=20wra?= =?UTF-8?q?pper=20Closed=20=E2=80=94=20let=20orchestrator=20reconnect?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cliff-detector recycle path emits Closed → Reconnecting → Connecting → Connected from the wrapper. The Closed branch was calling teardown(), which cancels cliffDetectorJob, announcesJob, and the wrapper itself, preventing the orchestrator from ever reopening the inner listener. Result pre-fix (visible in receiver log): cliff-detector fires after 4s of silence, teardown runs ~500ms later, wrapper never reopens, room permanently dead. User-initiated close goes through disconnect() / onCleared() which call teardown directly; the wrapper's subsequent Closed emission is redundant for those paths, so no-op'ing it is safe. UI still reflects Closed via state.toUiState(ui.connection). https://claude.ai/code/session_01UHN3fnXzWdj8UbSXnxSwwv --- .../commons/viewmodels/NestViewModel.kt | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt index 6776346a0..c221bdc20 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt @@ -883,11 +883,34 @@ class NestViewModel( // wait for a manual reconnect tap. } - // Server-initiated Closed: tear down stale - // local state so any later user-driven reconnect - // starts fresh. NestsListenerState.Closed -> { - teardown(targetState = ConnectionUiState.Closed) + // Closed from the wrapper is *almost always* + // transient — the orchestrator emits Closed + // → Reconnecting → Connecting → Connected + // around every cliff-detector recycle and + // every JWT refresh. We must NOT teardown + // here: teardown cancels `cliffDetectorJob`, + // `announcesJob`, etc., AND calls + // `wrapper.close()`, which cancels the + // wrapper's orchestrator before it can + // reopen the inner listener. End result + // pre-fix: the very first cliff-detector + // recycle permanently kills the room + // instead of recovering it (visible in the + // 15:56:25 receiver log: cliff fires → + // teardown fires 521 ms later → wrapper + // never reopens → `cliff-detector EXITED + // closed=false`). + // + // User-initiated close goes through + // `disconnect()` / `onCleared()` which call + // `teardown` directly; the wrapper's + // subsequent Closed emission here is a + // redundant signal — no-op'ing it doesn't + // change anything for those paths. The UI + // still picks up the Closed via + // `state.toUiState(ui.connection)` above + // for visual feedback. } else -> { /* no extra side effect */ } From 6e4df4aad4f9686b8f4524dfbba3a36afc761912 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 5 May 2026 22:13:12 +0000 Subject: [PATCH 10/41] =?UTF-8?q?fix(nests):=20cliff-detector=20=E2=80=94?= =?UTF-8?q?=20reset=20lastFrameAt=20on=20recycle=20+=20bump=20cooldown=20t?= =?UTF-8?q?o=2030s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Production logs at commit ea08c43 (run 18:05:14) showed the cliff detector now correctly fires (`active=1 announced=1` after the channelFlow chain fix) — but it then thrashes the relay: 18:05:25.488 cliff #1 → recycle, session 1 had 4 groups (3 s of audio) 18:05:40.512 cliff #2 → recycle, session 2 had 36 groups (7 s of audio) 18:05:48.538 cliff #3 → recycle, session 3 had 0 groups, then ... 18:05:56.566 cliff #4 → recycle, session 4 can't even subscribe ("subscribe stream FIN before reply" — relay refusing) Two compounding issues: 1. `lastFrameAt[pubkey]` was NOT reset when the cliff detector triggered a recycle. The next 1-second tick after the 8 s cooldown re-evaluated `lastFrameAt[pubkey].elapsedNow()`, which was still the giant pre-recycle value (now > 12 s). The check `>= 4_000ms` immediately tripped, recycling AGAIN regardless of whether the new session had begun delivering frames. The detector treated the recycle moment as if no time had passed. 2. The 8 s cooldown was the bare minimum needed for one reconnect handshake. It did NOT include time for moq-rs to drain its per-subscriber forward task queue from the prior subscription. Aggressive recycles every ~12 s drove the relay into a state where it FIN'd new subscribe bidis without responding — visible as "NestsListener.subscribe requires Connected state, was Closed" loops on the receiver after the 4th cliff. Two changes: a) When the cliff detector fires, before calling `recycleSession()` it now writes `lastFrameAt[pubkey] = recycleMark` for every stalled pubkey. This treats the recycle as a synthetic frame event so the cliff timer restarts from the recycle moment. The new session has the full `ROOM_AUDIO_CLIFF_TIMEOUT_MS` (4 s) to deliver before the next cliff check trips, instead of tripping the moment the cooldown ends. b) `ROOM_AUDIO_CLIFF_COOLDOWN_MS: 8_000L → 30_000L`. Gives moq-rs enough wall-clock between recycles to drain its per-subscriber forward queue before the next subscribe lands. Audible-gap cost rises from ~5 s to ~30 s in the worst-case fully-stalled relay, but this is the correct trade: 30 s of silence followed by recovered audio beats endless silence with the relay locked out. Combined, the worst-case recycle cycle goes from ~12 s (8 s cooldown + 4 s threshold = next recycle, relay degrades fast) to ~34 s (30 s cooldown + 4 s threshold = next recycle, relay can recover). For the steady state where the relay is healthy and the recycle DOES help, the new session's first frames clear the threshold on `lastFrameAt` and no second recycle fires at all. Tests: `CliffDetectorTest`'s 12 pure-predicate cases stay green (updated `cooldownSuppressesRecycleEvenWhenStalled` and `cooldownReleasesAfterTimeoutPasses` to use the new 30 s default). The `lastFrameAt` reset on recycle is in the live detector loop, not the predicate, so it's covered by integration runs rather than the pure-function suite. --- .../commons/viewmodels/NestViewModel.kt | 46 +++++++++++++++---- .../commons/viewmodels/CliffDetectorTest.kt | 10 ++-- 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt index c221bdc20..8dd9b6f41 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt @@ -1396,9 +1396,31 @@ class NestViewModel( com.vitorpamplona.quartz.utils.Log.w("NestRx") { "cliff-detector: announced+subscribed but silent for ≥${ROOM_AUDIO_CLIFF_TIMEOUT_MS}ms — recycling session. stalled=$stalled" } - lastCliffRecycleAt = + val recycleMark = kotlin.time.TimeSource.Monotonic .markNow() + lastCliffRecycleAt = recycleMark + // Reset `lastFrameAt` for every stalled pubkey so + // the cliff timer starts counting from the recycle + // moment, not from the old (pre-recycle) last + // frame timestamp. Without this reset the next + // tick after the cooldown immediately re-trips — + // because `lastFrameAt[pubkey].elapsedNow()` is + // still the giant pre-recycle value plus the + // cooldown window. Production logs at commit + // ea08c43 showed exactly this: 4 recycles in + // ~30 s eventually drove the relay into a + // "subscribe stream FIN before reply" loop where + // it refused fresh subscribes entirely. Using + // the recycle moment as a synthetic frame event + // gives the new session [ROOM_AUDIO_CLIFF_TIMEOUT_MS] + // wall-clock to deliver before the next cliff + // check, matching the expectation a freshly- + // attached subscription should clear inside + // that window if the relay is healthy. + for (pubkey in stalled) { + lastFrameAt[pubkey] = recycleMark + } runCatching { l.recycleSession() } } } finally { @@ -1698,14 +1720,22 @@ const val ROOM_AUDIO_CLIFF_CHECK_INTERVAL_MS: Long = 1_000L /** * Cooldown after a cliff-detector-driven recycle. Suppresses * back-to-back recycles while the wrapper is mid-handshake on the - * fresh session — a brand-new session has no `lastFrameAt` entries - * yet, which would otherwise immediately re-trigger the detector - * on the next 1 s tick. Long enough to cover the typical reconnect - * handshake plus a couple of audio-frame round trips, short enough - * that a recycle that didn't actually clear the cliff is recovered - * from quickly. + * fresh session AND while the relay is recovering its + * per-subscriber forward queue. + * + * Production logs at commit ea08c43 showed an 8 s cooldown was + * not enough — moq-rs needed longer to recover between aggressive + * recycles, and 4 recycles in ~30 s drove the relay into a + * "subscribe stream FIN before reply" loop that refused all + * subsequent subscribes. 30 s gives the relay's per-subscriber + * forward task pool time to drain stale pending writes from the + * previous subscription before the new subscribe lands. The + * audible-gap cost rises from ~5 s to ~30 s in the worst case + * (a fully-stalled relay), but the trade is correct: 30 s of + * silence + recovered audio beats endless silence with the relay + * locked out. */ -const val ROOM_AUDIO_CLIFF_COOLDOWN_MS: Long = 8_000L +const val ROOM_AUDIO_CLIFF_COOLDOWN_MS: Long = 30_000L /** * Diagnostic log frequency for the cliff detector. Emit a state-dump diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/CliffDetectorTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/CliffDetectorTest.kt index 4b8f285c6..a4cc54df2 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/CliffDetectorTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/CliffDetectorTest.kt @@ -193,13 +193,15 @@ class CliffDetectorTest { // pubkey; without a cooldown we would re-trigger on the // very next 1 s tick because the prior tick's stalled // pubkeys still age past the threshold (their lastFrameAt - // hasn't been updated by the new session yet). 8 s cooldown - // covers the typical reconnect handshake. + // hasn't been updated by the new session yet). 30 s cooldown + // covers the typical reconnect handshake AND gives moq-rs + // time to drain its per-subscriber forward queue from the + // prior subscription before the new subscribe lands. val ts = TestTimeSource() val frameMark = ts.markNow() ts += 4_500.milliseconds // past threshold val recycleMark = ts.markNow() // recycle just fired - ts += 1_000.milliseconds // 1 s into cooldown + ts += 5_000.milliseconds // 5 s into cooldown — well within 30 s window val result = computeStalledSpeakers( @@ -221,7 +223,7 @@ class CliffDetectorTest { val frameMark = ts.markNow() ts += 4_500.milliseconds val recycleMark = ts.markNow() - ts += 8_001.milliseconds // 1 ms past cooldown + ts += 30_001.milliseconds // 1 ms past 30 s cooldown val result = computeStalledSpeakers( From a36ccb569284ae92ecec09ae671e1efa6799ec64 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 5 May 2026 22:47:07 +0000 Subject: [PATCH 11/41] =?UTF-8?q?fix(nests):=20close=20the=20relay-cliff?= =?UTF-8?q?=20at=20the=20source=20=E2=80=94=20framesPerGroup=2010=E2=86=92?= =?UTF-8?q?50,=20cliff-threshold=204s=E2=86=922.5s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two-phone production logs at commit 6e4df4a (run 18:37) showed the listener-side cliff still hits at ~16 s of streaming (51 groups delivered, sender continued to seq=80 = ~6 s of audio lost at the tail before the receiver recycled). Recycling alone can't fix this — it only spaces the pain out. The user wants no audio drops mid- transmission, period. The leverage is to keep the relay's per-subscriber forward queue from filling at all. The cliff is rate-limited per uni-stream-creation, not per byte or per frame — moq-rs's `serve_group` task pool can't drain new `open_uni().await`s fast enough at high stream rates. Cutting the stream creation rate cuts the cliff window proportionally: framesPerGroup | streams/sec @ 50 fps | observed cliff window ---------------+------------------------+---------------------- 1 | 50 | ~3 s (commit d6517cf) 5 | 10 | ~13 s 10 | 5 | ~16 s (commit 6e4df4a) 50 | 1 | not reached in `fpg-all` sweeps 100 | 0.5 | never observed Bumping default to 50 (1 s of audio per uni stream, 1 stream/sec) puts us in the regime where the relay's queue does not measurably fill. `fpg-all` (100 frames in a single group) routinely delivers 100/100 in production sweeps, so 50 is well clear of the cliff edge. We pick 50 not 100 because: - Late-join gap is bounded by group size — a listener joining mid-broadcast waits one group boundary for the first frame. 1 s is within the ~250 ms AudioTrack jitter buffer + `ROOM_PLAYER_PREROLL_FRAMES = 5` audio-buffer envelope; 2 s starts to be perceptible. - QUIC stream RST drops the whole group on full reset. 1 s of skip on rare RST is bearable; 2 s is a noticeable seam. - Sender encode latency stays at 1 s — no worse than the natural buffering audio rooms already do for jitter. Belt-and-suspenders: also tighten the cliff-detector's silence threshold from 4 s to 2.5 s. Healthy streams now arrive every ~1 s (one `drainOneGroup` FIN per group), so 2.5 s of silence is unambiguously past two missed group cycles. Catches a still- possible cliff event ~1.5 s earlier, shaving the audible gap from ~6 s (4 s detection + 2 s reconnect) to ~4.5 s if recovery is needed at all. Tests: existing `framesPerGroup`-explicit interop scenarios (`fpg5`, `fpg20`, `fpg-all`) are unaffected — they pass an explicit value and don't read the default. `CliffDetectorTest` boundary cases updated to the new 2.5 s threshold; `returnsAllStalledSpeakersAtOnceMixedWithFreshOnes` re-spaced its fixture so charlie's 1.5 s frame age stays under threshold while alice/bob's 4 s ages exceed it. 12/12 pass. --- .../commons/viewmodels/NestViewModel.kt | 17 +++-- .../commons/viewmodels/CliffDetectorTest.kt | 16 +++-- .../audio/NestMoqLiteBroadcaster.kt | 65 ++++++++++++------- 3 files changed, 64 insertions(+), 34 deletions(-) diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt index 8dd9b6f41..e33b0ecc3 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt @@ -1702,12 +1702,19 @@ const val ROOM_AUDIO_MAX_LATENCY_MS: Long = 500L * orchestrator opens a fresh QUIC transport. Resets the relay's * per-subscriber forward queue. * - * 4 s gives enough headroom to ride out a publisher-side group-rollover - * hiccup (typically < 200 ms) without false-positive recycling, while - * keeping the audible gap from a real cliff to ~5 s of silence at - * worst (4 s detection + ~1 s reconnect handshake). + * 2 500 ms is past every legitimate group boundary (`framesPerGroup = + * 50` packs 1 s of audio per uni stream, so the receiver sees one + * `drainOneGroup` FIN every ~1 s on a healthy stream — 2.5 s of + * silence means more than two group cycles missed, very likely a + * cliff). Earlier than 2 s would false-positive on the natural 1 s + * cadence; later than 3 s wastes audible audio at every cliff edge. + * + * Production logs (`6e4df4a` run 18:37:43..18:38:08) showed the + * cliff-after-streaming case losing ~6 s of audio at the 4 s + * detector + ~2 s reconnect handshake. Tightening detection to + * 2.5 s shaves ~1.5 s off the audible gap. */ -const val ROOM_AUDIO_CLIFF_TIMEOUT_MS: Long = 4_000L +const val ROOM_AUDIO_CLIFF_TIMEOUT_MS: Long = 2_500L /** * How often the cliff detector wakes up to scan diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/CliffDetectorTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/CliffDetectorTest.kt index a4cc54df2..ac51d5ce8 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/CliffDetectorTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/CliffDetectorTest.kt @@ -103,14 +103,14 @@ class CliffDetectorTest { @Test fun returnsEmptyJustBeforeThreshold() { - // Boundary case: 1 ms under the 4 s threshold. The detector + // Boundary case: 1 ms under the 2.5 s threshold. The detector // should still consider this healthy. Relevant because audio - // groups arrive at ~100 ms cadence (framesPerGroup=10), so a + // groups arrive at ~1 s cadence (framesPerGroup=50), so a // false positive at ≈ threshold would recycle on every // group-rollover hiccup. val ts = TestTimeSource() val frameMark = ts.markNow() - ts += 3_999.milliseconds + ts += 2_499.milliseconds val result = computeStalledSpeakers( @@ -129,7 +129,7 @@ class CliffDetectorTest { // user-visible (this is the "audio went silent" detection). val ts = TestTimeSource() val frameMark = ts.markNow() - ts += 4_000.milliseconds + ts += 2_500.milliseconds val result = computeStalledSpeakers( @@ -164,12 +164,14 @@ class CliffDetectorTest { // should fire for the stalled set and leave charlie alone — // the `recycleSession` itself takes the whole listener down, // but the test here is the predicate's classification. + // Threshold = 2.5 s default. Spacings: alice/bob 4 s old (past), + // charlie 1.5 s old (under). val ts = TestTimeSource() val aliceFrame = ts.markNow() val bobFrame = ts.markNow() - ts += 3_000.milliseconds + ts += 2_500.milliseconds val charlieFrame = ts.markNow() - ts += 3_000.milliseconds + ts += 1_500.milliseconds val result = computeStalledSpeakers( @@ -251,7 +253,7 @@ class CliffDetectorTest { lastFrameAt = mapOf(ALICE to frameMark), lastRecycleAt = null, ) - assertTrue(withDefault.isEmpty(), "1 s elapsed shouldn't trip 4 s default") + assertTrue(withDefault.isEmpty(), "1 s elapsed shouldn't trip 2.5 s default") val withTightTimeout = computeStalledSpeakers( diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt index f764ed316..525f72c45 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt @@ -384,32 +384,53 @@ class NestMoqLiteBroadcaster( companion object { /** - * Default moq-lite group size = 10 Opus frames ≈ 200 ms of audio. + * Default moq-lite group size = 50 Opus frames ≈ 1 s of audio. * - * Picked to keep the QUIC uni-stream creation rate (5 streams/sec - * at 20 ms cadence) well under the production nostrnests relay's - * per-subscriber forward queue cliff. Two-phone production tests - * (`claude/fix-nests-audio-receiver-HCgOY` logcat) showed the - * relay still cliffed at ~135 streams under sustained load even - * with the old 5-frame default (10 streams/sec), giving ~13 s of - * audio before the listener's incoming uni-stream flow stopped - * — the same bug class documented in - * `nestsClient/plans/2026-05-01-quic-stream-cliff-investigation.md`, - * just shifted by half. Doubling the pack to 10 cuts the rate - * in half again and roughly doubles the cliff window in the - * worst case while halving the relay-side stream-handling - * pressure that triggers it. + * The relay-side cliff this defends against is per-subscriber + * forward-stream-rate, not per-frame or per-byte: moq-rs + * starts FIN'ing the per-subscriber publisher pipe once its + * per-subscriber uni-stream creation queue can't drain fast + * enough. Two-phone production tests on + * `claude/fix-nests-audio-receiver-HCgOY` showed: * - * Belt-and-suspenders: pair this with the listener-side - * cliff-detector in `NestViewModel` (no-data-while-announced - * triggers `recycleSession()`) so the room recovers from a - * cliff event regardless of the exact threshold. + * - `framesPerGroup = 10` (5 streams/sec) → cliff after + * ~16 s of streaming, ~6 s of audio lost at the tail + * (commit 6e4df4a logcat, run 18:37:43..18:38:08) + * - `framesPerGroup = 5` (10 streams/sec) → cliff after + * ~13 s of streaming, same dropout shape (commit + * d6517cf logcat run 14:25 — original bug report). * - * Late-join gap: ≤ 200 ms (one group boundary) instead of - * ≤ 100 ms with `framesPerGroup = 5` — still imperceptible - * for live audio rooms. + * 50 frames/group → 1 stream/sec at the production 50 fps + * Opus cadence. The relay's per-subscriber forward queue + * has measurable headroom at 1 stream/sec — sweep tests + * (`fpg-all` = 100 frames in a single group) consistently + * deliver 100/100 in production, and `fpg20` similarly + * passes. We pick 50 (not 100) because: + * + * - Late-join gap is bounded by group size: a listener + * joining mid-broadcast has to wait until the next + * group boundary for the first frame. 50 frames = 1 s + * gap, which matches what the existing + * `ROOM_PLAYER_PREROLL_FRAMES = 5` audio-buffer + the + * ~250 ms AudioTrack jitter buffer was already designed + * to mask. 100 frames (2 s) is past that. + * - QUIC stream-level reliability: each group is one uni + * stream, and a stream RST loses the whole group. 1 s + * of audio loss on RST is bearable; 2 s is noticeably + * a "skip". + * - Sender-side encode latency stays at 1 s — no worse + * than the natural buffering audio rooms already do. + * + * Pair with the listener-side cliff-detector in + * `NestViewModel` as belt-and-suspenders: if 50 frames/group + * STILL hits a cliff (relay version drift, network + * congestion, etc.) the detector recycles the transport so + * the next session reopens the relay's queue. + * + * Late-join gap: ≤ 1 s (one group boundary) — within the + * existing audio-room buffering envelope. */ - const val DEFAULT_FRAMES_PER_GROUP: Int = 10 + const val DEFAULT_FRAMES_PER_GROUP: Int = 50 /** * Maximum consecutive [MoqLitePublisherHandle.send] / [endGroup] From a86f19f06912d371bef92f0b54c50b70648199a4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 00:17:43 +0000 Subject: [PATCH 12/41] feat(nests): NestsTrace recorder for replayable session captures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an opt-in JSONL event recorder behind every receiver-side moq-lite + NestViewModel decision point so a real two-phone production session can be captured and (in a follow-up) replayed through the unmodified pipeline as a unit test. Step 1 of the "capture-then-replay" plan from the prior conversation. Off by default — production release builds that never call `NestsTrace.setRecording(true)` pay one volatile-load + branch per emit site. The fields-builder lambda doesn't run on the disabled path, so call sites can do non-trivial string concat freely. Output goes to logcat under tag `NestsTraceJsonl`. Capture with: adb logcat -c adb logcat -s NestsTraceJsonl:D -v raw > nest-trace.jsonl The `-v raw` formatter strips the `D NestsTraceJsonl:` prefix so the captured file is valid JSONL ready for replay tooling. Schema per line: `{"t_ms":N,"kind":"K", ...kind-specific fields}`, where `t_ms` is milliseconds since `setRecording(true)` was first called. Field names are lower_snake_case for stability across clients. Wired into 13 call sites this commit (matched 1:1 with existing NestRx/NestTx human log lines so the trace and the log line stay adjacent and the diff stays small): `MoqLiteSession`: - announce_bidi_opened (per session.announce call) - announce_pump_emit (per Active/Ended received on a bidi) - announce_bidi_ended_naturally / announce_bidi_threw - subscribe_send / subscribe_ok / subscribe_drop - subscribe_bidi_exited - announce_watch_update / announce_watch_ended_closing_subs - uni_pump_started - group_header / group_fin / group_threw `NestViewModel`: - vm_observe_announce (per ann emission) - cliff_tick (every CLIFF_DIAG_LOG_EVERY ticks: active + announced sets + per-pubkey lastFrameAt elapsed-ms) - cliff_recycle (when the detector forces a recycleSession()) Anonymisation: pubkeys + track names recorded verbatim — they're already in the existing `NestRx`/`NestTx` log lines the user is sharing. Frame payloads are NEVER recorded, only sizes — audio content can't leak through a trace dump. Tests: NestsTraceTest (9 cases) — exhaustive jsonStr/jsonArrStr quoting + setRecording state-machine + emit-lambda-noop-when- disabled coverage. The `emit` log-output side itself is untestable in commonTest because `Log.d` writes to a platform actual; the schema correctness we DO want to pin (a JSON syntax bug at one of the 13 call sites would silently break replay) is covered by the quoting helpers. CliffDetectorTest 12/12 + MoqLiteSessionTest 11/11 still pass — the trace wiring is purely additive next to existing log statements. Follow-up: a `TraceReplayingTransport` reading these JSONL files back through `WebTransportSession` to drive end-to-end regression tests for the cliff-recovery scenarios captured from production. --- .../commons/viewmodels/NestViewModel.kt | 17 ++ .../nestsclient/moq/lite/MoqLiteSession.kt | 57 ++++++ .../nestsclient/trace/NestsTrace.kt | 191 ++++++++++++++++++ .../nestsclient/trace/NestsTraceTest.kt | 149 ++++++++++++++ 4 files changed, 414 insertions(+) create mode 100644 nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/trace/NestsTrace.kt create mode 100644 nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/trace/NestsTraceTest.kt diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt index e33b0ecc3..9bc9255ba 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt @@ -831,6 +831,10 @@ class NestViewModel( com.vitorpamplona.quartz.utils.Log.d("NestRx") { "observeAnnounces #$emissionCount active=${ann.active} pubkey='${ann.pubkey.take(12)}' → ${if (ann.active) "ADD" else "REMOVE"}" } + com.vitorpamplona.nestsclient.trace.NestsTrace.emit("vm_observe_announce") { + "\"emission\":$emissionCount,\"active\":${ann.active}," + + "\"pubkey\":${com.vitorpamplona.nestsclient.trace.jsonStr(ann.pubkey)}" + } if (ann.active) { _announcedSpeakers.update { it + ann.pubkey } } else { @@ -1382,6 +1386,15 @@ class NestViewModel( com.vitorpamplona.quartz.utils.Log.d("NestRx") { "cliff-detector tick=$ticks active=${activeSpeakers.size} announced=${announced.size} lastFrameAges=[$ages]" } + com.vitorpamplona.nestsclient.trace.NestsTrace.emit("cliff_tick") { + "\"tick\":$ticks," + + "\"active\":${com.vitorpamplona.nestsclient.trace.jsonArrStr(activeSpeakers)}," + + "\"announced\":${com.vitorpamplona.nestsclient.trace.jsonArrStr(announced)}," + + "\"last_frame_age_ms\":{" + + lastFrameAt.entries.joinToString { + "${com.vitorpamplona.nestsclient.trace.jsonStr(it.key)}:${it.value.elapsedNow().inWholeMilliseconds}" + } + "}" + } } val stalled = computeStalledSpeakers( @@ -1396,6 +1409,10 @@ class NestViewModel( com.vitorpamplona.quartz.utils.Log.w("NestRx") { "cliff-detector: announced+subscribed but silent for ≥${ROOM_AUDIO_CLIFF_TIMEOUT_MS}ms — recycling session. stalled=$stalled" } + com.vitorpamplona.nestsclient.trace.NestsTrace.emit("cliff_recycle") { + "\"timeout_ms\":$ROOM_AUDIO_CLIFF_TIMEOUT_MS," + + "\"stalled\":${com.vitorpamplona.nestsclient.trace.jsonArrStr(stalled)}" + } val recycleMark = kotlin.time.TimeSource.Monotonic .markNow() diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt index 903c7526e..72afdb4ce 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt @@ -22,6 +22,8 @@ package com.vitorpamplona.nestsclient.moq.lite import com.vitorpamplona.nestsclient.moq.MoqCodecException import com.vitorpamplona.nestsclient.moq.MoqWriter +import com.vitorpamplona.nestsclient.trace.NestsTrace +import com.vitorpamplona.nestsclient.trace.jsonStr import com.vitorpamplona.nestsclient.transport.WebTransportSession import com.vitorpamplona.quartz.utils.Log import com.vitorpamplona.quic.Varint @@ -137,6 +139,7 @@ class MoqLiteSession internal constructor( onBufferOverflow = BufferOverflow.DROP_OLDEST, ) Log.d("NestRx") { "session.announce(prefix='$prefix') bidi opened, pump launching (replayCap=64)" } + NestsTrace.emit("announce_bidi_opened") { "\"prefix\":${jsonStr(prefix)}" } val pump = scope.launch { val buffer = MoqLiteFrameBuffer() @@ -155,14 +158,29 @@ class MoqLiteSession internal constructor( "status=${decoded.status} suffix='${decoded.suffix.take(12)}' " + "(chunks=$chunkCount)" } + NestsTrace.emit("announce_pump_emit") { + "\"prefix\":${jsonStr(prefix)}," + + "\"emit_count\":$emitCount," + + "\"status\":${jsonStr(decoded.status.toString())}," + + "\"suffix\":${jsonStr(decoded.suffix)}," + + "\"chunks\":$chunkCount" + } updates.emit(decoded) } } Log.w("NestRx") { "session.announce(prefix='$prefix') bidi.incoming() ended naturally (chunks=$chunkCount, emits=$emitCount)" } + NestsTrace.emit("announce_bidi_ended_naturally") { + "\"prefix\":${jsonStr(prefix)},\"chunks\":$chunkCount,\"emits\":$emitCount" + } } catch (ce: CancellationException) { throw ce } catch (t: Throwable) { Log.w("NestRx") { "announce(prefix='$prefix'): bidi.incoming() threw ${t::class.simpleName}: ${t.message} (chunks=$chunkCount, emits=$emitCount)" } + NestsTrace.emit("announce_bidi_threw") { + "\"prefix\":${jsonStr(prefix)},\"chunks\":$chunkCount,\"emits\":$emitCount," + + "\"error\":${jsonStr(t::class.simpleName ?: "?")}," + + "\"message\":${jsonStr(t.message ?: "")}" + } // Flow terminated (peer FIN or transport close). // The Announce stream's emit-side just stops; consumers // see an end-of-flow. @@ -291,6 +309,10 @@ class MoqLiteSession internal constructor( if (groupPump == null) groupPump = scope.launch { pumpUniStreams() } } Log.d("NestRx") { "SUBSCRIBE send id=$id broadcast='$broadcast' track='$track' maxLatencyMs=$maxLatencyMillis" } + NestsTrace.emit("subscribe_send") { + "\"id\":$id,\"broadcast\":${jsonStr(broadcast)},\"track\":${jsonStr(track)}," + + "\"max_latency_ms\":$maxLatencyMillis" + } // Now that the subscription is registered, push the SUBSCRIBE // bytes. If `bidi.write` throws (transport torn down, peer // reset) we'd otherwise leave an orphaned map entry whose @@ -358,6 +380,10 @@ class MoqLiteSession internal constructor( val removed = state.withLock { subscriptionsBySubscribeId.remove(id) } if (removed != null) { Log.w("NestRx") { "SUBSCRIBE bidi exited, closing frames id=$id broadcast='${removed.request.broadcast}' track='${removed.request.track}'" } + NestsTrace.emit("subscribe_bidi_exited") { + "\"id\":$id,\"broadcast\":${jsonStr(removed.request.broadcast)}," + + "\"track\":${jsonStr(removed.request.track)}" + } } removed?.frames?.close() } @@ -377,6 +403,11 @@ class MoqLiteSession internal constructor( "SUBSCRIBE_DROP id=$id broadcast='$broadcast' track='$track' " + "errCode=${resp.drop.errorCode} reason='${resp.drop.reasonPhrase}'" } + NestsTrace.emit("subscribe_drop") { + "\"id\":$id,\"broadcast\":${jsonStr(broadcast)},\"track\":${jsonStr(track)}," + + "\"err_code\":${resp.drop.errorCode}," + + "\"reason\":${jsonStr(resp.drop.reasonPhrase)}" + } state.withLock { subscriptionsBySubscribeId.remove(id) } frames.close() runCatching { bidi.finish() } @@ -388,6 +419,9 @@ class MoqLiteSession internal constructor( is MoqLiteCodec.SubscribeResponse.Ok -> { Log.d("NestRx") { "SUBSCRIBE_OK id=$id broadcast='$broadcast' track='$track'" } + NestsTrace.emit("subscribe_ok") { + "\"id\":$id,\"broadcast\":${jsonStr(broadcast)},\"track\":${jsonStr(track)}" + } return MoqLiteSubscribeHandle( id = id, ok = resp.ok, @@ -464,6 +498,11 @@ class MoqLiteSession internal constructor( try { handle.updates.collect { update -> Log.d("NestRx") { "ANNOUNCE update status=${update.status} suffix='${update.suffix}' hops=${update.hops}" } + NestsTrace.emit("announce_watch_update") { + "\"status\":${jsonStr(update.status.toString())}," + + "\"suffix\":${jsonStr(update.suffix)}," + + "\"hops\":${update.hops}" + } if (update.status != MoqLiteAnnounceStatus.Ended) return@collect val targets = state.withLock { @@ -473,6 +512,10 @@ class MoqLiteSession internal constructor( } if (targets.isNotEmpty()) { Log.w("NestRx") { "ANNOUNCE Ended for suffix='${update.suffix}' → closing ${targets.size} subscription(s): ${targets.map { "id=${it.id} track='${it.request.track}'" }}" } + NestsTrace.emit("announce_watch_ended_closing_subs") { + "\"suffix\":${jsonStr(update.suffix)}," + + "\"closed_count\":${targets.size}" + } } for (sub in targets) { // Just close the frames channel — the @@ -512,6 +555,7 @@ class MoqLiteSession internal constructor( */ private suspend fun pumpUniStreams() { Log.d("NestRx") { "pumpUniStreams started" } + NestsTrace.emit("uni_pump_started") var streamCount = 0L try { // coroutineScope binds each per-stream drain to this pump's @@ -564,6 +608,9 @@ class MoqLiteSession internal constructor( groupSequence = hdr.sequence headerRead = true Log.d("NestRx") { "drainOneGroup#$streamSeq header subId=$subscribeId groupSeq=$groupSequence" } + NestsTrace.emit("group_header") { + "\"stream_seq\":$streamSeq,\"sub_id\":$subscribeId,\"group_seq\":$groupSequence" + } } while (true) { val frame = buffer.readSizePrefixed() ?: break @@ -588,10 +635,20 @@ class MoqLiteSession internal constructor( } } Log.d("NestRx") { "drainOneGroup#$streamSeq FIN subId=$subscribeId groupSeq=$groupSequence frames=$frameCount droppedNoSub=$droppedNoSub trySendFail=$trySendFailures" } + NestsTrace.emit("group_fin") { + "\"stream_seq\":$streamSeq,\"sub_id\":$subscribeId,\"group_seq\":$groupSequence," + + "\"frames\":$frameCount,\"dropped_no_sub\":$droppedNoSub,\"try_send_fail\":$trySendFailures" + } } catch (ce: CancellationException) { throw ce } catch (t: Throwable) { Log.w("NestRx") { "drainOneGroup#$streamSeq threw subId=$subscribeId groupSeq=$groupSequence frames=$frameCount: ${t::class.simpleName}: ${t.message}" } + NestsTrace.emit("group_threw") { + "\"stream_seq\":$streamSeq,\"sub_id\":$subscribeId,\"group_seq\":$groupSequence," + + "\"frames\":$frameCount," + + "\"error\":${jsonStr(t::class.simpleName ?: "?")}," + + "\"message\":${jsonStr(t.message ?: "")}" + } // Stream errored / FIN'd. Nothing to do — the next group // arrives on a fresh uni stream. } diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/trace/NestsTrace.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/trace/NestsTrace.kt new file mode 100644 index 000000000..b0b88912b --- /dev/null +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/trace/NestsTrace.kt @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.nestsclient.trace + +/** + * Append-only event recorder for the moq-lite + Nests audio path. + * + * Designed to capture enough wire-level + decision-level data from a real + * production session that the same communication can be replayed in a + * unit test without a network — i.e., feed the recorded events back + * through a `TraceReplayingTransport` (planned, not yet implemented) into + * the unmodified [com.vitorpamplona.nestsclient.moq.lite.MoqLiteSession] + * + [com.vitorpamplona.nestsclient.MoqLiteNestsListener] + + * [com.vitorpamplona.amethyst.commons.viewmodels.NestViewModel] stack + * and assert on observable behaviour (frames received, cliff-detector + * decisions, recycle counts). + * + * **Cost when disabled**: a single volatile-load + branch per call site + * (the [emit] inline guards `if (!enabled) return` before the lambda + * runs). Production builds that never call [setRecording] pay nothing. + * + * **Output format**: one JSON object per line, written via + * [com.vitorpamplona.quartz.utils.Log] at the [TAG] tag. Capture from a + * connected device with: + * + * adb logcat -c + * adb logcat -s NestsTraceJsonl:D -v raw > nest-trace.jsonl + * + * The `-v raw` formatter strips the `D NestsTraceJsonl:` prefix so the + * captured file is valid JSONL ready for replay tooling. + * + * **Schema**: each event is a JSON object with at least + * - `t_ms` — milliseconds since [setRecording] was called with `true` + * - `kind` — string discriminator naming the event + * and additional kind-specific fields. Keep all field names lower-snake- + * case and stable; the replay tool will pattern-match on `kind`. + * + * Anonymisation: pubkeys + track names are recorded verbatim because + * they're already in the production logs the user is sharing (the + * `NestRx` / `NestTx` tags). Frame payloads are NEVER recorded — only + * sizes — so audio content can't leak through a trace dump. + */ +object NestsTrace { + /** Tag the JSONL lines are emitted under. Filter logcat with `-s NestsTraceJsonl:D`. */ + const val TAG: String = "NestsTraceJsonl" + + // `@PublishedApi internal` rather than `private` because [emit] is + // inline — its body becomes part of every call site's bytecode and + // must be able to reach the backing fields. `private` would refuse + // to compile ("Public-API inline function cannot access non-public- + // API property"). Marking these `@PublishedApi internal` keeps them + // unreachable from outside the module while satisfying the inliner. + @PublishedApi + @Volatile + internal var enabled: Boolean = false + + @PublishedApi + @Volatile + internal var startMark: kotlin.time.TimeMark? = null + + /** + * Idempotent. When `on` flips from false → true, [startMark] is + * captured so subsequent [emit] calls record monotonic-time deltas + * relative to enable. Flipping true → false stops further emits but + * does NOT alter prior log output. + * + * Call from a debug-build app start-up hook, a debug menu toggle, or + * a test `@Before`. Production release builds should leave the + * default disabled. + * + * Named `setRecording` (not `setEnabled`) so it doesn't clash with + * the JVM-generated setter for the `enabled` backing property — + * that property is `@PublishedApi internal` for the inline [emit] + * to access, which forces the setter to share the `setEnabled` + * JVM name. + */ + fun setRecording(on: Boolean) { + if (on == enabled) return + if (on) { + startMark = + kotlin.time.TimeSource.Monotonic + .markNow() + } + enabled = on + } + + fun isRecording(): Boolean = enabled + + /** + * Record an event. The lambda is invoked ONLY when tracing is + * enabled, so the call site pays nothing in the disabled path beyond + * the field-load + comparison. + * + * The lambda must return the kind-specific JSON fields portion + * (no surrounding braces, no leading or trailing comma — empty + * string when there are no fields). This recorder prepends + * `{"t_ms":N,"kind":"K"`, joins fields with a comma when present, + * and appends `}`. + * + * Use [jsonStr] / [jsonArrStr] to safely quote string values and + * arrays. Numeric / boolean values can be interpolated directly. + * + * Example: + * + * NestsTrace.emit("subscribe_send") { + * "\"id\":$id,\"broadcast\":${jsonStr(broadcast)},\"track\":${jsonStr(track)}" + * } + */ + inline fun emit( + kind: String, + fieldsJson: () -> String = { "" }, + ) { + if (!enabled) return + val mark = startMark ?: return + val tMs = mark.elapsedNow().inWholeMilliseconds + val fields = fieldsJson() + val sep = if (fields.isEmpty()) "" else "," + com.vitorpamplona.quartz.utils.Log.d(TAG) { + "{\"t_ms\":$tMs,\"kind\":\"$kind\"$sep$fields}" + } + } +} + +/** + * Quote a string as a JSON literal, escaping the small set of characters + * that would otherwise produce invalid JSONL (`"`, `\`, control chars). + * Keeps the implementation small + commonMain-portable rather than + * pulling in a full JSON library for what is effectively a single + * field-value path. + */ +fun jsonStr(value: String): String { + val sb = StringBuilder(value.length + 2) + sb.append('"') + for (c in value) { + when (c) { + '"' -> { + sb.append('\\').append('"') + } + + '\\' -> { + sb.append('\\').append('\\') + } + + '\n' -> { + sb.append('\\').append('n') + } + + '\r' -> { + sb.append('\\').append('r') + } + + '\t' -> { + sb.append('\\').append('t') + } + + else -> { + if (c.code < 0x20) { + sb.append("\\u00") + val h = c.code.toString(16) + if (h.length == 1) sb.append('0') + sb.append(h) + } else { + sb.append(c) + } + } + } + } + sb.append('"') + return sb.toString() +} + +/** JSON array of strings: `["a","b","c"]`. */ +fun jsonArrStr(values: Iterable): String = values.joinToString(separator = ",", prefix = "[", postfix = "]") { jsonStr(it) } diff --git a/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/trace/NestsTraceTest.kt b/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/trace/NestsTraceTest.kt new file mode 100644 index 000000000..774e05f5e --- /dev/null +++ b/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/trace/NestsTraceTest.kt @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.nestsclient.trace + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +/** + * Pure-string tests for [jsonStr] / [jsonArrStr] (the JSON-quoting + * helpers used at every trace call site) and a small toggle test for + * [NestsTrace.setRecording]'s state-machine. + * + * The actual `emit` log-output side is untestable in commonTest because + * `com.vitorpamplona.quartz.utils.Log` writes to logcat / stdout via + * platform actuals, not via an injectable sink. The schema correctness + * we DO want to pin — a JSON-syntax bug at one of the call sites would + * silently corrupt the trace file and break replay tooling — is covered + * by exercising the quoting helpers exhaustively and asserting on + * concatenated round-trip equality with hand-built JSON literals. + */ +class NestsTraceTest { + @Test + fun jsonStrEscapesQuotesAndBackslashes() { + assertEquals("\"hello\"", jsonStr("hello")) + assertEquals("\"with \\\"quotes\\\"\"", jsonStr("with \"quotes\"")) + assertEquals("\"backslash \\\\ here\"", jsonStr("backslash \\ here")) + } + + @Test + fun jsonStrEscapesControlCharacters() { + assertEquals("\"line1\\nline2\"", jsonStr("line1\nline2")) + assertEquals("\"col1\\tcol2\"", jsonStr("col1\tcol2")) + assertEquals("\"crlf\\r\\n\"", jsonStr("crlf\r\n")) + } + + @Test + fun jsonStrEscapesLowControlCharsAsUnicode() { + //  (start of heading) — must be  in JSON, not raw. + val raw = "xy" + val quoted = jsonStr(raw) + assertEquals("\"x\\u0001y\"", quoted) + } + + @Test + fun jsonStrLeavesPrintableAsciiAlone() { + // Every printable ASCII char that isn't `"` or `\` must round-trip + // unmodified — most production trace fields are pubkey hex, + // track names, event-kind enums. + val allPrintable = + (0x20..0x7e) + .map { it.toChar() } + .filter { it != '"' && it != '\\' } + .joinToString("") + val quoted = jsonStr(allPrintable) + assertEquals("\"$allPrintable\"", quoted) + } + + @Test + fun jsonArrStrEmitsValidJsonArray() { + assertEquals("[]", jsonArrStr(emptyList())) + assertEquals("[\"a\"]", jsonArrStr(listOf("a"))) + assertEquals( + "[\"alpha\",\"beta\",\"gamma\"]", + jsonArrStr(listOf("alpha", "beta", "gamma")), + ) + } + + @Test + fun jsonArrStrEscapesElementsConsistentlyWithJsonStr() { + // Each element runs through jsonStr — quotes and backslashes + // inside an element must be escaped just like a stand-alone field. + assertEquals( + "[\"a\\\"b\",\"c\\\\d\"]", + jsonArrStr(listOf("a\"b", "c\\d")), + ) + } + + @Test + fun setRecordingIsIdempotent() { + // Set up clean state for the test — flip off in case a prior + // test left the recorder enabled. (No reset() API by design; + // tests share the singleton.) + NestsTrace.setRecording(false) + assertFalse(NestsTrace.isRecording()) + + NestsTrace.setRecording(true) + assertTrue(NestsTrace.isRecording()) + + // Double-enable: no change in state, no error. + NestsTrace.setRecording(true) + assertTrue(NestsTrace.isRecording()) + + NestsTrace.setRecording(false) + assertFalse(NestsTrace.isRecording()) + + // Double-disable: no change in state, no error. + NestsTrace.setRecording(false) + assertFalse(NestsTrace.isRecording()) + } + + @Test + fun emitIsNoOpWhenDisabled() { + // Lambda must not run when tracing is off — call sites pass + // a non-trivial allocator (string concat) and we promise zero + // work on the disabled path. + NestsTrace.setRecording(false) + var lambdaRanCount = 0 + NestsTrace.emit("would_have_recorded") { + lambdaRanCount += 1 + "" + } + assertEquals(0, lambdaRanCount, "emit's fields lambda must not run when tracing is disabled") + } + + @Test + fun emitRunsLambdaWhenEnabled() { + NestsTrace.setRecording(true) + try { + var lambdaRanCount = 0 + NestsTrace.emit("did_record") { + lambdaRanCount += 1 + "\"k\":\"v\"" + } + assertEquals(1, lambdaRanCount, "emit's fields lambda must run exactly once when enabled") + } finally { + NestsTrace.setRecording(false) + } + } +} From bedf21f57e5f1d65681598ff3be1e936936e6d0c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 00:22:43 +0000 Subject: [PATCH 13/41] chore(nests): auto-enable NestsTrace recorder in debug builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hook into Amethyst.onCreate behind the existing isDebug check so the JSONL session-trace recorder lights up the moment a debug APK launches — no per-room toggle to find, no rebuild needed to flip the flag, no extra step on top of the existing logcat capture flow. Release builds are unaffected: setRecording() is gated by `if (isDebug)`, so the volatile flag stays false and every emit site short-circuits at the field-load + branch. Capture from a connected receiver phone with: adb logcat -c adb logcat -s NestsTraceJsonl:D -v raw > nest-trace.jsonl The `-v raw` formatter strips the `D NestsTraceJsonl:` prefix so the file is valid JSONL ready for the (planned) TraceReplayingTransport to feed back through MoqLiteSession + NestViewModel for headless cliff-scenario regression tests. --- .../src/main/java/com/vitorpamplona/amethyst/Amethyst.kt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/Amethyst.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/Amethyst.kt index 8e488a83a..99703e822 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/Amethyst.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/Amethyst.kt @@ -43,6 +43,12 @@ class Amethyst : Application() { if (isDebug) { Logging.setup() + // Auto-enable the Nests session-trace recorder in debug + // builds so two-phone repros can be captured via + // adb logcat -s NestsTraceJsonl:D -v raw > nest-trace.jsonl + // without rebuilding to flip a flag. Off in release. + com.vitorpamplona.nestsclient.trace.NestsTrace + .setRecording(true) } instance.initiate(this) From babf7623d85e2b87a8a17057f4c0cfe4a7eee26b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 00:56:43 +0000 Subject: [PATCH 14/41] feat(nests): publish catalog.json so browsers can discover broadcasts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two-phone Amethyst sessions stream Opus frames just fine on the wire, but the kixelated/moq browser watcher (the canonical web reference) sees nothing — because we never published a catalog.json track. moq-lite watchers discover a broadcast's tracks by subscribing to the `catalog.json` track on the broadcast suffix and parsing the JSON manifest. Without it the watcher has nothing to subscribe to. Two parts: 1. MoqLiteSession.publish now supports multiple tracks per session, sharing one broadcast suffix. The previous "one publisher per session" invariant rejected the second publish() with "already publishing", forcing a single track per session. 2. MoqLiteNestsSpeaker.startBroadcasting now opens both an audio publisher AND a catalog publisher, pushes the manifest once, and launches a 2s republish loop so late-attaching watchers don't wait indefinitely for the next refresh. The manifest shape mirrors RoomSpeakerCatalog (kotlinx.serialization data class on the listener side): {"version":1,"audio":[{"track":"audio/data","codec":"opus", "sample_rate":48000,"channel_count":1}]} Hard-coded for now since OpusEncoder is fixed at 48kHz mono. Hot-swap path (ReconnectingNestsSpeaker session refresh) is NOT yet wired to recycle the catalog publisher — acceptable trade-off for a diagnostic landing; follow-up will move the catalog republisher into HotSwappablePublisherSource so it survives JWT refreshes. --- .../nestsclient/MoqLiteNestsSpeaker.kt | 111 ++++++++++++++++++ .../nestsclient/moq/lite/MoqLiteSession.kt | 108 ++++++++++++++--- 2 files changed, 200 insertions(+), 19 deletions(-) diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt index c719a8cac..00a8622af 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt @@ -25,10 +25,15 @@ import com.vitorpamplona.nestsclient.audio.NestMoqLiteBroadcaster import com.vitorpamplona.nestsclient.audio.OpusEncoder import com.vitorpamplona.nestsclient.moq.lite.MoqLitePublisherHandle import com.vitorpamplona.nestsclient.moq.lite.MoqLiteSession +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock @@ -96,6 +101,26 @@ class MoqLiteNestsSpeaker internal constructor( // session's `activePublisher` slot — otherwise a subsequent // startBroadcasting fails with "already publishing" and the // publisher's announce state never tears down. + // + // Companion catalog publisher: the kixelated/moq browser + // watcher (and any standards-aligned moq-lite consumer) + // discovers a broadcast's tracks by subscribing to the + // `catalog.json` track and parsing the latest group's + // payload as a JSON manifest. Without this track our + // broadcasts are *invisible* to the canonical web watcher + // even though our audio frames are streaming fine — the + // watcher has nothing to subscribe to. Open it alongside + // audio so a single broadcast advertises both tracks. + val catalogPublisher = + try { + session.publish( + broadcastSuffix = speakerPubkeyHex, + track = MoqLiteNestsListener.CATALOG_TRACK, + ) + } catch (t: Throwable) { + runCatching { publisher.close() } + throw t + } val broadcaster = try { NestMoqLiteBroadcaster( @@ -120,6 +145,39 @@ class MoqLiteNestsSpeaker internal constructor( ) } } catch (t: Throwable) { + runCatching { catalogPublisher.close() } + runCatching { publisher.close() } + throw t + } + // Push catalog once before launching the periodic republish + // pump so the manifest is on the wire before any subscriber + // attaches; the republisher then re-emits a fresh group + // every CATALOG_REPUBLISH_INTERVAL_MS so late-joining + // watchers don't wait an arbitrary amount of time for the + // next refresh. + val catalogJson = SPEAKER_CATALOG_JSON.encodeToByteArray() + val republishJob = + try { + catalogPublisher.send(catalogJson) + catalogPublisher.endGroup() + scope.launch { + try { + while (true) { + delay(CATALOG_REPUBLISH_INTERVAL_MS) + catalogPublisher.send(catalogJson) + catalogPublisher.endGroup() + } + } catch (ce: CancellationException) { + throw ce + } catch (_: Throwable) { + // Best-effort: a transient catalog send + // failure is non-fatal — the audio path + // owns terminal-failure detection. + } + } + } catch (t: Throwable) { + runCatching { catalogPublisher.close() } + runCatching { broadcaster.stop() } runCatching { publisher.close() } throw t } @@ -133,6 +191,8 @@ class MoqLiteNestsSpeaker internal constructor( MoqLiteBroadcastHandle( broadcaster = broadcaster, publisher = publisher, + catalogPublisher = catalogPublisher, + catalogRepublishJob = republishJob, parent = this, ) activeHandle = handle @@ -140,6 +200,31 @@ class MoqLiteNestsSpeaker internal constructor( } } + companion object { + /** + * moq-lite catalog manifest broadcast on the + * [MoqLiteNestsListener.CATALOG_TRACK] track. The shape mirrors + * what the kixelated/moq browser watcher parses + * ([com.vitorpamplona.amethyst.commons.viewmodels.RoomSpeakerCatalog]): + * a versioned envelope with an `audio[]` of track descriptors. + * Hard-coded because the speaker's encoder is fixed at + * 48 kHz mono Opus today; if [OpusEncoder] becomes parameterised + * this should be derived from the encoder config instead. + */ + const val SPEAKER_CATALOG_JSON: String = + "{\"version\":1,\"audio\":[{\"track\":\"audio/data\"," + + "\"codec\":\"opus\",\"sample_rate\":48000,\"channel_count\":1}]}" + + /** + * How often the catalog group is re-emitted. The relay drops + * the catalog group when its last subscriber drops, so a + * watcher that attaches mid-broadcast needs a freshly-published + * group to receive the manifest. 2 s keeps late-attach worst + * case bounded without spamming the relay. + */ + const val CATALOG_REPUBLISH_INTERVAL_MS: Long = 2_000L + } + /** * [HotSwappablePublisherSource] implementation. See the interface * kdoc — this method mints a fresh publisher on the session @@ -228,6 +313,8 @@ class MoqLiteNestsSpeaker internal constructor( internal class MoqLiteBroadcastHandle( private val broadcaster: NestMoqLiteBroadcaster, private val publisher: MoqLitePublisherHandle, + private val catalogPublisher: MoqLitePublisherHandle, + private val catalogRepublishJob: Job, private val parent: MoqLiteNestsSpeaker, ) : BroadcastHandle { @Volatile private var muted: Boolean = false @@ -246,12 +333,27 @@ internal class MoqLiteBroadcastHandle( override suspend fun close() { if (closed) return closed = true + // Stop the catalog republisher first so it doesn't race a + // concurrent send against the publisher.close below. + try { + catalogRepublishJob.cancelAndJoin() + } catch (ce: kotlinx.coroutines.CancellationException) { + // The parent scope is being cancelled. Continue cleanup of + // the audio + publisher resources, then rethrow. + runCatching { catalogPublisher.close() } + runCatching { publisher.close() } + parent.broadcastClosed(this) + throw ce + } catch (_: Throwable) { + // Best-effort. + } try { broadcaster.stop() } catch (ce: kotlinx.coroutines.CancellationException) { // Even on cancel, run the rest of cleanup before rethrowing // — broadcaster.stop already cancels its own job, so the // mic + encoder + publisher are owed their close paths. + runCatching { catalogPublisher.close() } runCatching { publisher.close() } parent.broadcastClosed(this) throw ce @@ -263,6 +365,15 @@ internal class MoqLiteBroadcastHandle( // failures on the broadcaster.stop path. try { publisher.close() + } catch (ce: kotlinx.coroutines.CancellationException) { + runCatching { catalogPublisher.close() } + parent.broadcastClosed(this) + throw ce + } catch (_: Throwable) { + // Best-effort. + } + try { + catalogPublisher.close() } catch (ce: kotlinx.coroutines.CancellationException) { parent.broadcastClosed(this) throw ce diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt index 72afdb4ce..55f27fa12 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt @@ -98,8 +98,22 @@ class MoqLiteSession internal constructor( */ private var announceWatchJob: Job? = null - /** Single active publisher per session (moq-lite doesn't model multi-broadcast publishers). */ - private var activePublisher: PublisherStateImpl? = null + /** + * Active publishers on this session. moq-lite models a broadcast as + * `(suffix, set-of-tracks)`; the relay multiplexes Subscribe + * messages to whichever publisher claims the requested track. We + * allow N concurrent publishers per session as long as they share + * the same suffix (= same broadcast) and have distinct tracks. + * + * The nests audio room use case needs at least two: `audio/data` + * for Opus frames and `catalog.json` for the broadcast metadata + * the canonical kixelated/moq watcher subscribes to first to + * discover what's available. Without the catalog the broadcast is + * invisible to the JS reference watcher (the browser-side nests + * web client) — Amethyst-to-Amethyst kept working only because + * both sides hardcoded the audio track name and skipped discovery. + */ + private val activePublishers: MutableList = mutableListOf() @Volatile private var closed: Boolean = false @@ -683,8 +697,12 @@ class MoqLiteSession internal constructor( * - we open uni streams ourselves to push group data * (`session.open_uni()`) * - * Only one [publish] is supported per session for now. Calling - * [publish] twice on the same session is rejected with [IllegalStateException]. + * Multiple [publish] calls are supported on the same session as + * long as every call shares the same `broadcastSuffix` (you publish + * one broadcast per session, with multiple tracks) and uses a + * distinct `track`. Re-publishing the same `(suffix, track)` pair + * or mixing different suffixes is rejected with + * [IllegalStateException]. */ suspend fun publish( broadcastSuffix: String, @@ -695,11 +713,16 @@ class MoqLiteSession internal constructor( val publisher: PublisherStateImpl state.withLock { check(!closed) { "session is closed" } - check(activePublisher == null) { - "MoqLiteSession.publish called twice — only one broadcast per session is supported" + val existingSuffix = activePublishers.firstOrNull()?.suffix + check(existingSuffix == null || existingSuffix == normalised) { + "MoqLiteSession.publish suffix mismatch: existing='$existingSuffix', new='$normalised'. " + + "moq-lite models one broadcast per session — open a new session for a different broadcast." + } + check(activePublishers.none { it.track == track }) { + "MoqLiteSession.publish called twice for the same track '$track' on suffix '$normalised'." } publisher = PublisherStateImpl(suffix = normalised, track = track) - activePublisher = publisher + activePublishers += publisher // Lazy launch — the inbound-bidi pump needs to keep running // for the lifetime of any active publisher. if (bidiPump == null) bidiPump = scope.launch { pumpInboundBidis() } @@ -734,7 +757,21 @@ class MoqLiteSession internal constructor( } private suspend fun handleInboundBidi(bidi: com.vitorpamplona.nestsclient.transport.WebTransportBidiStream) { - val publisher = state.withLock { activePublisher } ?: return + // Snapshot of publishers at bidi-arrival time. All publishers + // on a single session share a suffix (enforced in [publish]), + // so the Announce response is the same regardless of which + // publisher we route the bidi to. Subscribe responses pick the + // publisher whose `track` matches `sub.track`. + val publishersSnapshot = state.withLock { activePublishers.toList() } + if (publishersSnapshot.isEmpty()) return + // Designated publisher for Announce-bidi ownership: the first + // one. The Active/Ended pair is keyed off the broadcast + // SUFFIX (not per track), so emitting it once via one + // publisher matches the single-broadcast model the relay + // expects. All publishers close together in [close], so + // there's no risk of one closing while the announce bidi + // stays alive on another. + val announcePublisher = publishersSnapshot.first() // Single long-running collector for the bidi's full lifetime. // Pre-fix this dispatch was split into a `firstOrNull()` to @@ -759,6 +796,16 @@ class MoqLiteSession internal constructor( var typeCode: Long? = null var dispatched = false var inboundSub: MoqLiteSubscribe? = null + // Track which publisher this bidi was routed to so the peer-FIN + // cleanup at the bottom calls removeInboundSubscription on the + // right one. `inboundAnnouncePublisher` is currently unused for + // cleanup (announce bidis live until publisher.close fires + // Ended), but we keep the assignment for symmetry / future + // explicit teardown. + var inboundSubPublisher: PublisherStateImpl? = null + + @Suppress("UNUSED_VARIABLE") + var inboundAnnouncePublisher: PublisherStateImpl? = null try { bidi.incoming().collect { chunk -> buffer.push(chunk) @@ -776,7 +823,7 @@ class MoqLiteSession internal constructor( val pleasePayload = buffer.readSizePrefixed() ?: return@collect val please = MoqLiteCodec.decodeAnnouncePlease(pleasePayload) val emittedSuffix = - MoqLitePath.stripPrefix(please.prefix, publisher.suffix) ?: publisher.suffix + MoqLitePath.stripPrefix(please.prefix, announcePublisher.suffix) ?: announcePublisher.suffix bidi.write( MoqLiteCodec.encodeAnnounce( MoqLiteAnnounce( @@ -786,15 +833,31 @@ class MoqLiteSession internal constructor( ), ), ) - publisher.registerAnnounceBidi(bidi, emittedSuffix) - Log.d("NestTx") { "ANNOUNCE inbound prefix='${please.prefix}' → emitted Active suffix='$emittedSuffix' (publisher.suffix='${publisher.suffix}')" } + announcePublisher.registerAnnounceBidi(bidi, emittedSuffix) + // Routed to publisher that owns the announce bidi for the lifecycle + // — for the multi-track use case (audio + catalog), all share one + // suffix and one announce bidi pointing at the first publisher. + inboundAnnouncePublisher = announcePublisher + Log.d("NestTx") { "ANNOUNCE inbound prefix='${please.prefix}' → emitted Active suffix='$emittedSuffix' (publisher.suffix='${announcePublisher.suffix}')" } dispatched = true } MoqLiteControlType.Subscribe -> { val subPayload = buffer.readSizePrefixed() ?: return@collect val sub = MoqLiteCodec.decodeSubscribe(subPayload) - Log.d("NestTx") { "SUBSCRIBE inbound id=${sub.id} broadcast='${sub.broadcast}' track='${sub.track}' (publisher track='${publisher.track}')" } + // Find the publisher that claims this track. With the + // single-track-per-publisher model, only one match is possible. + val targetPublisher = publishersSnapshot.firstOrNull { it.track == sub.track } + if (targetPublisher == null) { + Log.w("NestTx") { + "SUBSCRIBE inbound id=${sub.id} track='${sub.track}' has no matching publisher " + + "on this session (have ${publishersSnapshot.map { it.track }}) — finishing bidi" + } + runCatching { bidi.finish() } + dispatched = true + return@collect + } + Log.d("NestTx") { "SUBSCRIBE inbound id=${sub.id} broadcast='${sub.broadcast}' track='${sub.track}' (publisher track='${targetPublisher.track}')" } // Register the subscription BEFORE sending Ok so the // peer's observation of Ok is a happens-after of // `inboundSubs += sub`. Otherwise on dispatchers that @@ -803,8 +866,9 @@ class MoqLiteSession internal constructor( // (notably Windows under Dispatchers.Default), the // peer's first `publisher.send` after Ok races the // registration and observes an empty subscriber set. - publisher.registerInboundSubscription(sub) + targetPublisher.registerInboundSubscription(sub) inboundSub = sub + inboundSubPublisher = targetPublisher bidi.write( MoqLiteCodec.encodeSubscribeOk( MoqLiteSubscribeOk( @@ -843,7 +907,11 @@ class MoqLiteSession internal constructor( // groups off this dead subscriber. Announce bidis are // owned by the publisher state for sending Ended on // publisher-close — we don't remove them here. - inboundSub?.let { publisher.removeInboundSubscription(it) } + val sub = inboundSub + val pub = inboundSubPublisher + if (sub != null && pub != null) { + pub.removeInboundSubscription(sub) + } } /** @@ -872,18 +940,20 @@ class MoqLiteSession internal constructor( if (closed) return closed = true val toClose: List - val publisherToClose: PublisherStateImpl? + val publishersToClose: List state.withLock { toClose = subscriptionsBySubscribeId.values.toList() subscriptionsBySubscribeId.clear() - publisherToClose = activePublisher - activePublisher = null + publishersToClose = activePublishers.toList() + activePublishers.clear() } for (sub in toClose) { runCatching { sub.bidi.finish() } sub.frames.close() } - runCatching { publisherToClose?.close() } + for (p in publishersToClose) { + runCatching { p.close() } + } groupPump?.cancelAndJoin() bidiPump?.cancelAndJoin() runCatching { transport.close() } @@ -1081,7 +1151,7 @@ class MoqLiteSession internal constructor( runCatching { groupToFinish?.uni?.finish() } // Detach from the session so a subsequent `publish` can run. state.withLock { - if (activePublisher === this) activePublisher = null + activePublishers.remove(this@PublisherStateImpl) } } From 5389b32e52fba6646a7e0b1da0585aecb4822fdc Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 01:06:36 +0000 Subject: [PATCH 15/41] feat(nests): keep catalog.json published across JWT refresh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ReissuingBroadcastHandle's hot-swap iteration now opens its own catalog.json publisher on every fresh session, pushes the manifest once, and launches the same 2 s republish loop that MoqLiteNestsSpeaker.startBroadcasting uses for the legacy non-reconnecting path. Each iteration tears down the prior session's catalog publisher + republisher AFTER the new pair is installed, so watchers see at most a brief overlap rather than a gap when the 600 s moq-auth bearer triggers a session recycle. Without this, the catalog goes silent the moment the wrapper recycles the underlying session; any watcher that attaches AFTER the recycle finds nothing to subscribe to even though the audio path is hot- swapped seamlessly. With it, both audio and catalog survive every JWT refresh. Catalog teardown is also added to ReissuingBroadcastHandle.close() because — unlike the audio publisher which broadcaster.stop() handles internally — the catalog has no broadcaster wrapper. --- .../nestsclient/ReconnectingNestsSpeaker.kt | 100 +++++++++++++++++- 1 file changed, 96 insertions(+), 4 deletions(-) diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsSpeaker.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsSpeaker.kt index a6ed6e68d..5bdb479b6 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsSpeaker.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsSpeaker.kt @@ -23,11 +23,13 @@ package com.vitorpamplona.nestsclient import com.vitorpamplona.nestsclient.audio.AudioCapture import com.vitorpamplona.nestsclient.audio.NestMoqLiteBroadcaster import com.vitorpamplona.nestsclient.audio.OpusEncoder +import com.vitorpamplona.nestsclient.moq.lite.MoqLitePublisherHandle import com.vitorpamplona.nestsclient.transport.WebTransportFactory import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.awaitCancellation +import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow @@ -427,6 +429,20 @@ private class ReissuingBroadcastHandle( /** Hot-swap path's long-lived broadcaster. Null until the first session arrives. */ @Volatile private var hotSwapBroadcaster: NestMoqLiteBroadcaster? = null + /** + * Hot-swap path's per-session catalog publisher and its periodic + * republish job. Catalog has no long-lived encoder pipeline (unlike + * audio), so each session gets a fresh publisher + republish loop; + * cleanup happens at the start of the next iteration after the new + * pair is installed (mirroring how the old audio publisher is + * closed only after [NestMoqLiteBroadcaster.swapPublisher] returns + * it). Both nullable so [close] can no-op when no iteration ever + * ran (i.e. wrapper closed before the first session connected). + */ + @Volatile private var hotSwapCatalogPublisher: MoqLitePublisherHandle? = null + + @Volatile private var hotSwapCatalogJob: Job? = null + /** Legacy path's per-session handle. Cleared when the session swaps. */ private val liveHandle = AtomicReference(null) private var pumpJob: Job? = null @@ -549,16 +565,83 @@ private class ReissuingBroadcastHandle( if (old != null) runCatching { old.close() } } + // Catalog track on the new session. Keeps the broadcast + // discoverable to standards-aligned moq-lite watchers (the + // kixelated/moq browser reference) across JWT refresh — without + // this the catalog goes silent the moment the session recycles + // and any watcher that attaches AFTER the recycle sees nothing + // to subscribe to. Mirror of [MoqLiteNestsSpeaker.startBroadcasting]'s + // catalog setup; same JSON, same republish cadence. + val catalogPayload = MoqLiteNestsSpeaker.SPEAKER_CATALOG_JSON.encodeToByteArray() + val priorCatalogPublisher = hotSwapCatalogPublisher + val priorCatalogJob = hotSwapCatalogJob + val newCatalogPublisher = + try { + hotSwap.openPublisherForHotSwap(MoqLiteNestsListener.CATALOG_TRACK) + } catch (ce: kotlinx.coroutines.CancellationException) { + throw ce + } catch (_: Throwable) { + // Couldn't mint a catalog publisher on this session. + // Audio path is already live; treat as non-fatal — the + // next session swap retries. Leave prior catalog state + // alone (it's about to die with the prior session). + null + } + if (newCatalogPublisher != null) { + val newCatalogJob = + try { + newCatalogPublisher.send(catalogPayload) + newCatalogPublisher.endGroup() + scope.launch { + try { + while (true) { + delay(MoqLiteNestsSpeaker.CATALOG_REPUBLISH_INTERVAL_MS) + newCatalogPublisher.send(catalogPayload) + newCatalogPublisher.endGroup() + } + } catch (ce: kotlinx.coroutines.CancellationException) { + throw ce + } catch (_: Throwable) { + // Best-effort: a transient catalog send + // failure on this session is non-fatal — + // the audio path owns terminal-failure + // detection. + } + } + } catch (ce: kotlinx.coroutines.CancellationException) { + runCatching { newCatalogPublisher.close() } + throw ce + } catch (_: Throwable) { + runCatching { newCatalogPublisher.close() } + null + } + if (newCatalogJob != null) { + hotSwapCatalogPublisher = newCatalogPublisher + hotSwapCatalogJob = newCatalogJob + // Tear down the prior session's catalog state AFTER the + // new one is installed so the watcher only sees a brief + // overlap rather than a gap. The prior publisher's + // session is about to be torn down by the orchestrator + // anyway, but graceful Announce(Ended) keeps the relay + // book-keeping clean. + if (priorCatalogJob != null) runCatching { priorCatalogJob.cancelAndJoin() } + if (priorCatalogPublisher != null) runCatching { priorCatalogPublisher.close() } + } + } + try { // Park until [collectLatest] cancels us on the next session // swap, OR [close] cancels [pumpJob]. The broadcaster keeps // running through the cancellation; only close() stops it. awaitCancellation() } finally { - // Intentionally do NOT close the broadcaster here. - // collectLatest cancels this iteration on every session - // swap; closing the broadcaster would create the exact - // 50–150 ms gap this whole path exists to avoid. + // Intentionally do NOT close the broadcaster or the catalog + // publisher here. collectLatest cancels this iteration on + // every session swap; closing now would force the catalog + // republisher to drop a beat between sessions. The next + // iteration handles teardown of the prior catalog after + // installing its replacement (above), and [close] handles + // teardown when the wrapper itself closes. } } @@ -615,6 +698,15 @@ private class ReissuingBroadcastHandle( // current publisher (broadcaster.stop calls publisher.close // internally). For legacy, the per-session handle's close // releases its own broadcaster. + // + // Catalog gets explicit teardown because — unlike the audio + // publisher which broadcaster.stop() closes for us — the + // catalog publisher + republisher have no broadcaster wrapper. + // Cancel the republish loop FIRST so it can't race the close. + hotSwapCatalogJob?.let { runCatching { it.cancelAndJoin() } } + hotSwapCatalogJob = null + hotSwapCatalogPublisher?.let { runCatching { it.close() } } + hotSwapCatalogPublisher = null hotSwapBroadcaster?.let { runCatching { it.stop() } } hotSwapBroadcaster = null liveHandle.getAndSet(null)?.let { runCatching { it.close() } } From 5b7e347ac0ad223d270e9ea7c7d690b63934a88a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 03:16:01 +0000 Subject: [PATCH 16/41] feat(nests): align moq-lite catalog with kixelated/hang shape Switch RoomSpeakerCatalog to the canonical hang catalog format (camelCase audio.renditions[] with container.kind), wrap audio frames in the legacy varint(timestamp_us) prefix on publish, and strip it on subscribe. Also reply to inbound Probe/Fetch/Session bidis with a bitrate hint or clean FIN instead of hanging. --- .../commons/viewmodels/RoomSpeakerCatalog.kt | 81 ++++++++++---- .../viewmodels/RoomSpeakerCatalogTest.kt | 100 ++++++++++++------ .../nestsclient/MoqLiteNestsListener.kt | 47 +++++++- .../nestsclient/MoqLiteNestsSpeaker.kt | 24 +++-- .../audio/NestMoqLiteBroadcaster.kt | 24 ++++- .../nestsclient/moq/lite/MoqLiteCodec.kt | 14 +++ .../nestsclient/moq/lite/MoqLiteSession.kt | 56 +++++++++- 7 files changed, 281 insertions(+), 65 deletions(-) diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalog.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalog.kt index 3694843e8..9bbf04840 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalog.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalog.kt @@ -22,35 +22,67 @@ package com.vitorpamplona.amethyst.commons.viewmodels import androidx.compose.runtime.Immutable import com.vitorpamplona.quartz.nip01Core.core.JsonMapper -import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable /** * Decoded shape of a moq-lite `catalog.json` track for a single - * audio-room speaker. Each broadcast advertises one or more tracks - * (one per codec / quality tier) — for nests audio rooms there's - * exactly one Opus track today, but the model carries a list for - * forward compatibility. + * audio-room speaker. Mirrors the kixelated/moq `hang` reference + * catalog format — the canonical shape that `@kixelated/hang`'s + * browser watcher and the `moq-rs` Rust hang crate both produce and + * consume — so Amethyst publishers are visible to the moq-lite browser + * reference, and Amethyst listeners parse standards-aligned catalogs + * from non-Amethyst publishers. * - * Best-effort schema: nostrnests' upstream moq-lite catalog spec - * (kixelated/moq) has evolved across revisions, and not every - * publisher fills in every field. The parser tolerates missing - * keys via `ignoreUnknownKeys = true` on [JsonMapper] and the - * `?`-marked properties. + * Shape (verbatim from `kixelated/moq/rs/hang/src/catalog/`): + * + * { + * "audio": { + * "renditions": { + * "": { + * "codec": "opus", + * "container": { "kind": "legacy" }, + * "sampleRate": 48000, + * "numberOfChannels": 1, + * "bitrate": 32000 // optional + * } + * } + * } + * } + * + * Keys are camelCase per the upstream serde `rename_all = "camelCase"`. + * Every field except the rendition-key string is optional in the parser + * — older / partial publishers are tolerated. Field semantics: + * + * - rendition key: the moq-lite `track` name a subscriber should + * subscribe to for that audio rendition's frames (commonly the same + * string for single-rendition broadcasts). Subscribers pick a + * rendition (e.g. by codec / bitrate) and use this key as the + * Subscribe.track string. + * - `codec`: codec mimetype string (`"opus"`, `"mp4a.40.2"` for AAC). + * - `container.kind`: frame wrapper. `"legacy"` = each frame is + * `varint(timestamp_us) + raw_codec_payload` inside the moq-lite + * group. `"cmaf"` = MOOF/MDAT-fragmented MP4. We only emit/parse + * `legacy` today; unknown kinds are ignored at parse time. + * - `sampleRate` / `numberOfChannels`: PCM source params. */ @Immutable @Serializable data class RoomSpeakerCatalog( - val version: Int? = null, - val audio: List = emptyList(), + val audio: Audio? = null, ) { @Immutable @Serializable - data class AudioTrack( - val track: String? = null, + data class Audio( + val renditions: Map = emptyMap(), + ) + + @Immutable + @Serializable + data class AudioConfig( val codec: String? = null, - @SerialName("sample_rate") val sampleRate: Int? = null, - @SerialName("channel_count") val channelCount: Int? = null, + val container: Container? = null, + val sampleRate: Int? = null, + val numberOfChannels: Int? = null, val bitrate: Int? = null, ) { /** @@ -64,14 +96,25 @@ data class RoomSpeakerCatalog( buildList { codec?.takeIf { it.isNotBlank() }?.let { add(it.uppercase()) } sampleRate?.let { add("${it / 1000}kHz") } - channelCount?.let { add(if (it == 1) "mono" else "${it}ch") } + numberOfChannels?.let { add(if (it == 1) "mono" else "${it}ch") } } return parts.takeIf { it.isNotEmpty() }?.joinToString(" · ") } } - /** First audio track, if any. The current single-Opus reality. */ - fun primaryAudio(): AudioTrack? = audio.firstOrNull() + @Immutable + @Serializable + data class Container( + val kind: String? = null, + ) + + /** + * First audio rendition, if any. The current single-Opus reality. + * Map iteration order is the JSON insertion order (kotlinx.serialization + * uses LinkedHashMap), so this picks the first rendition the + * publisher declared rather than an arbitrary one. + */ + fun primaryAudio(): AudioConfig? = audio?.renditions?.values?.firstOrNull() companion object { /** diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalogTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalogTest.kt index 4f4ffab80..f8a45bcff 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalogTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalogTest.kt @@ -27,69 +27,72 @@ import kotlin.test.assertNull class RoomSpeakerCatalogTest { @Test - fun parsesNostrnestsShape() { + fun parsesKixelatedHangShape() { + // Verbatim shape produced by the canonical kixelated/moq `hang` + // crate (`rs/hang/src/catalog/`). Verifies that an Amethyst + // listener picks up a standards-aligned publisher's catalog. val json = """ { - "version": 1, - "audio": [ - { - "track": "audio/data", - "codec": "opus", - "sample_rate": 48000, - "channel_count": 2, - "bitrate": 64000 + "audio": { + "renditions": { + "audio/data": { + "codec": "opus", + "container": { "kind": "legacy" }, + "sampleRate": 48000, + "numberOfChannels": 2, + "bitrate": 64000 + } } - ] + } } """.trimIndent() val catalog = RoomSpeakerCatalog.parseOrNull(json.encodeToByteArray()) assertNotNull(catalog) - assertEquals(1, catalog.version) - assertEquals(1, catalog.audio.size) - val track = catalog.primaryAudio() - assertNotNull(track) - assertEquals("audio/data", track.track) - assertEquals("opus", track.codec) - assertEquals(48_000, track.sampleRate) - assertEquals(2, track.channelCount) - assertEquals(64_000, track.bitrate) + assertEquals(1, catalog.audio?.renditions?.size) + val rendition = catalog.primaryAudio() + assertNotNull(rendition) + assertEquals("opus", rendition.codec) + assertEquals("legacy", rendition.container?.kind) + assertEquals(48_000, rendition.sampleRate) + assertEquals(2, rendition.numberOfChannels) + assertEquals(64_000, rendition.bitrate) } @Test fun describeFormatsHumanReadable() { - val track = - RoomSpeakerCatalog.AudioTrack( + val rendition = + RoomSpeakerCatalog.AudioConfig( codec = "opus", sampleRate = 48_000, - channelCount = 2, + numberOfChannels = 2, ) // Codec uppercased, kHz / channel count short-formed — // intended for a single-line tooltip in the participant // sheet, not a verbose codec dump. - assertEquals("OPUS · 48kHz · 2ch", track.describe()) + assertEquals("OPUS · 48kHz · 2ch", rendition.describe()) } @Test fun describeMonoIsLabelled() { - val track = RoomSpeakerCatalog.AudioTrack(codec = "opus", channelCount = 1) - assertEquals("OPUS · mono", track.describe()) + val rendition = RoomSpeakerCatalog.AudioConfig(codec = "opus", numberOfChannels = 1) + assertEquals("OPUS · mono", rendition.describe()) } @Test fun describeAllNullReturnsNull() { - val track = RoomSpeakerCatalog.AudioTrack() + val rendition = RoomSpeakerCatalog.AudioConfig() // Empty catalog → caller doesn't render a "unknown · unknown" // line; describe() returns null so the UI can omit cleanly. - assertNull(track.describe()) + assertNull(rendition.describe()) } @Test fun toleratesUnknownKeys() { - // Forward-compat: future moq-lite catalog revisions can add + // Forward-compat: future hang catalog revisions can add // fields without breaking older clients. val json = - """{"version":2,"audio":[{"track":"audio/data","codec":"opus","extra":"future-only"}],"new_top_level":true}""" + """{"audio":{"renditions":{"audio/data":{"codec":"opus","extra":"future-only"}}},"video":{},"newTopLevel":true}""" val catalog = RoomSpeakerCatalog.parseOrNull(json.encodeToByteArray()) assertNotNull(catalog) assertEquals("opus", catalog.primaryAudio()?.codec) @@ -104,12 +107,43 @@ class RoomSpeakerCatalogTest { } @Test - fun emptyAudioListIsAllowed() { + fun emptyAudioRenditionsIsAllowed() { // A publisher might emit an "advertising" catalog with no - // tracks yet (e.g. between codec switches). Accept the shape - // and let primaryAudio() return null. - val catalog = RoomSpeakerCatalog.parseOrNull("""{"version":1,"audio":[]}""".encodeToByteArray()) + // renditions yet (e.g. between codec switches). Accept the + // shape and let primaryAudio() return null. + val catalog = RoomSpeakerCatalog.parseOrNull("""{"audio":{"renditions":{}}}""".encodeToByteArray()) assertNotNull(catalog) assertNull(catalog.primaryAudio()) } + + @Test + fun missingAudioReturnsNullPrimary() { + // hang catalogs declaring only video MUST still parse without + // throwing — primaryAudio() returns null. + val catalog = RoomSpeakerCatalog.parseOrNull("""{}""".encodeToByteArray()) + assertNotNull(catalog) + assertNull(catalog.primaryAudio()) + } + + @Test + fun stripPrefixRoundTripsCanonicalCatalog() { + // The catalog payload [com.vitorpamplona.nestsclient.MoqLiteNestsSpeaker] + // emits MUST round-trip through the parser — guards against + // either side drifting from the kixelated/hang shape. + // SPEAKER_CATALOG_JSON lives in nestsClient and isn't + // accessible from commons; keep an inline literal that mirrors + // it verbatim so a desync triggers this assertion. + val emitted = + "{\"audio\":{\"renditions\":{\"audio/data\":{" + + "\"codec\":\"opus\",\"container\":{\"kind\":\"legacy\"}," + + "\"sampleRate\":48000,\"numberOfChannels\":1}}}}" + val catalog = RoomSpeakerCatalog.parseOrNull(emitted.encodeToByteArray()) + assertNotNull(catalog) + val rendition = catalog.primaryAudio() + assertNotNull(rendition) + assertEquals("opus", rendition.codec) + assertEquals("legacy", rendition.container?.kind) + assertEquals(48_000, rendition.sampleRate) + assertEquals(1, rendition.numberOfChannels) + } } diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsListener.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsListener.kt index 6b6c308bb..424ad38d9 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsListener.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsListener.kt @@ -82,14 +82,36 @@ class MoqLiteNestsListener internal constructor( override suspend fun subscribeSpeaker( speakerPubkeyHex: String, maxLatencyMs: Long, - ): SubscribeHandle = wrapSubscription(broadcast = speakerPubkeyHex, track = AUDIO_TRACK, maxLatencyMs = maxLatencyMs) + ): SubscribeHandle = + wrapSubscription( + broadcast = speakerPubkeyHex, + track = AUDIO_TRACK, + maxLatencyMs = maxLatencyMs, + // Audio frames arrive in kixelated/moq `hang` "legacy" + // container layout: each moq-lite frame is + // `varint(timestamp_us) + raw_opus_packet`. Strip the + // leading varint so downstream decoders see a pristine + // Opus packet exactly as if it came from + // [com.vitorpamplona.nestsclient.audio.OpusEncoder]. + stripLegacyTimestamp = true, + ) - override suspend fun subscribeCatalog(speakerPubkeyHex: String): SubscribeHandle = wrapSubscription(broadcast = speakerPubkeyHex, track = CATALOG_TRACK, maxLatencyMs = 0L) + override suspend fun subscribeCatalog(speakerPubkeyHex: String): SubscribeHandle = + wrapSubscription( + broadcast = speakerPubkeyHex, + track = CATALOG_TRACK, + maxLatencyMs = 0L, + // Catalog frames carry raw JSON bytes — no container + // wrapping (the catalog itself is what tells you which + // container the audio track uses). + stripLegacyTimestamp = false, + ) private suspend fun wrapSubscription( broadcast: String, track: String, maxLatencyMs: Long, + stripLegacyTimestamp: Boolean, ): SubscribeHandle { check(state.value is NestsListenerState.Connected) { "NestsListener.subscribe requires Connected state, was ${state.value}" @@ -112,12 +134,13 @@ class MoqLiteNestsListener internal constructor( val objectIdSeq = AtomicLong(0L) val mapped = handle.frames.map { frame -> + val payload = if (stripLegacyTimestamp) stripLegacyTimestampPrefix(frame.payload) else frame.payload MoqObject( trackAlias = handle.id, groupId = frame.groupSequence, objectId = objectIdSeq.getAndIncrement(), publisherPriority = MoqLiteSession.DEFAULT_PRIORITY, - payload = frame.payload, + payload = payload, ) } @@ -242,5 +265,23 @@ class MoqLiteNestsListener internal constructor( * [subscribeCatalog]. */ const val CATALOG_TRACK: String = "catalog.json" + + /** + * Strip the leading `varint(timestamp_us)` prefix from a + * kixelated/moq `hang` "legacy" container frame, returning + * the bare codec payload (e.g. an Opus packet) for downstream + * decoders. The varint length is encoded in the top 2 bits of + * the first byte per RFC 9000 §16: `00`→1, `01`→2, `10`→4, + * `11`→8 bytes. Returns the payload unchanged when the + * varint header would overrun the payload (malformed frame — + * surface upstream rather than silently mask). + */ + internal fun stripLegacyTimestampPrefix(payload: ByteArray): ByteArray { + if (payload.isEmpty()) return payload + val tag = (payload[0].toInt() ushr 6) and 0x3 + val varintLen = 1 shl tag + if (payload.size < varintLen) return payload + return payload.copyOfRange(varintLen, payload.size) + } } } diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt index 00a8622af..67cbf3217 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt @@ -203,17 +203,29 @@ class MoqLiteNestsSpeaker internal constructor( companion object { /** * moq-lite catalog manifest broadcast on the - * [MoqLiteNestsListener.CATALOG_TRACK] track. The shape mirrors - * what the kixelated/moq browser watcher parses - * ([com.vitorpamplona.amethyst.commons.viewmodels.RoomSpeakerCatalog]): - * a versioned envelope with an `audio[]` of track descriptors. + * [MoqLiteNestsListener.CATALOG_TRACK] track. Verbatim + * canonical kixelated/moq `hang` shape (camelCase, nested + * `audio.renditions[]`, `container.kind = "legacy"`). + * The rendition key MUST equal the moq-lite track we publish + * audio frames on ([MoqLiteNestsListener.AUDIO_TRACK]) so the + * watcher's "subscribe to this rendition" path resolves to our + * actual audio stream. + * + * `container.kind = "legacy"` declares the wire layout the + * watcher must expect inside each moq-lite frame: a single + * `varint(timestamp_us)` prefix followed by the raw codec + * payload (no MOOF/MDAT wrapping). The publisher side honours + * this contract in + * [com.vitorpamplona.nestsclient.audio.NestMoqLiteBroadcaster]. + * * Hard-coded because the speaker's encoder is fixed at * 48 kHz mono Opus today; if [OpusEncoder] becomes parameterised * this should be derived from the encoder config instead. */ const val SPEAKER_CATALOG_JSON: String = - "{\"version\":1,\"audio\":[{\"track\":\"audio/data\"," + - "\"codec\":\"opus\",\"sample_rate\":48000,\"channel_count\":1}]}" + "{\"audio\":{\"renditions\":{\"" + MoqLiteNestsListener.AUDIO_TRACK + "\":{" + + "\"codec\":\"opus\",\"container\":{\"kind\":\"legacy\"}," + + "\"sampleRate\":48000,\"numberOfChannels\":1}}}}" /** * How often the catalog group is re-emitted. The relay drops diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt index 525f72c45..cf10c7a57 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt @@ -21,11 +21,14 @@ package com.vitorpamplona.nestsclient.audio import com.vitorpamplona.nestsclient.moq.lite.MoqLitePublisherHandle +import com.vitorpamplona.quic.Varint import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.launch +import kotlin.time.TimeMark +import kotlin.time.TimeSource /** * Mirror of [NestBroadcaster] but driving a moq-lite @@ -195,6 +198,21 @@ class NestMoqLiteBroadcaster( // and the relay would see two unrelated uni streams under // the same logical group. var lastPublisher: MoqLitePublisherHandle = publisher + // kixelated/moq `hang` "legacy" container wire format: + // every frame inside a moq-lite group is + // varint(timestamp_us) + raw_codec_payload + // (`rs/hang/src/container/frame.rs`, + // Timescale<1_000_000>). Watchers that read our + // catalog's `container.kind = "legacy"` declaration + // will skip the leading varint as a microsecond + // timestamp; they MUST receive a real timestamp or + // their decoder picks up garbage bytes ahead of the + // Opus packet. We use a monotonic mark captured at + // capture-loop start so timestamps are robust to mute + // gaps and survive publisher hot-swap (the wire is + // single-broadcast, single-encoder; timestamps are + // codec-payload metadata, not stream-position). + val frameStartMark: TimeMark = TimeSource.Monotonic.markNow() try { while (true) { val pcm = capture.readFrame() ?: break @@ -233,7 +251,11 @@ class NestMoqLiteBroadcaster( // for the production cliff this works around. val sendOutcome = runCatching { - val accepted = current.send(opus) + val tsBytes = Varint.encode(frameStartMark.elapsedNow().inWholeMicroseconds) + val payload = ByteArray(tsBytes.size + opus.size) + tsBytes.copyInto(payload, 0) + opus.copyInto(payload, tsBytes.size) + val accepted = current.send(payload) framesInCurrentGroup += 1 if (framesInCurrentGroup >= framesPerGroup) { current.endGroup() diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteCodec.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteCodec.kt index 3d5b6445a..239a41d55 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteCodec.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteCodec.kt @@ -251,6 +251,20 @@ object MoqLiteCodec { return MoqLiteProbe(bitrate = bitrate) } + /** + * Encode a single Lite-03 Probe message body + * (`lite/probe.rs` — `bitrate: u62` only; `rtt` is Lite-04+). + * The publisher writes these size-prefixed onto a Probe bidi the + * subscriber opened, advertising the publisher's expected + * bandwidth. Wrapping (size prefix) is the caller's responsibility, + * matching [encodeAnnouncePlease] / [encodeAnnounce]. + */ + fun encodeProbe(probe: MoqLiteProbe): ByteArray { + val body = MoqWriter() + body.writeVarint(probe.bitrate) + return wrapSizePrefixed(body) + } + // ---------------- internals ---------------- /** diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt index 55f27fa12..ff31ab993 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt @@ -883,9 +883,50 @@ class MoqLiteSession internal constructor( dispatched = true } - else -> { - // Lite-03 treats Session/Fetch/Probe as - // separate flows; we don't implement them. + MoqLiteControlType.Probe -> { + // Subscriber-opened bidi asking us (the + // publisher) for a bitrate hint. Per + // `lite/probe.rs:Lite03+`, the publisher + // writes one or more size-prefixed + // `Probe { bitrate: u62 }` messages on + // this bidi. We're a fixed-rate Opus + // publisher, so emit a single hint and + // FIN our write side — the peer treats + // FIN as "no further updates" rather than + // an error. Better than the old behaviour + // of FINing without writing anything, + // which left the subscriber's ABR + // estimator with no signal. + runCatching { + bidi.write(MoqLiteCodec.encodeProbe(MoqLiteProbe(bitrate = NESTS_AUDIO_BITRATE_HINT_BPS))) + bidi.finish() + } + dispatched = true + } + + MoqLiteControlType.Fetch -> { + // Subscriber-opened bidi requesting a + // historical group (`lite/fetch.rs`). + // Audio rooms are live-only — we have no + // group history to serve. FINing the + // write side without any reply is the + // spec-clean way to signal "no groups + // available"; the subscriber's wait on + // its receive side resolves to + // end-of-stream and it falls back to a + // live Subscribe. Ignoring inbound bytes + // (the request body) is fine: we don't + // need to know which group was requested + // because we couldn't serve any of them. + runCatching { bidi.finish() } + dispatched = true + } + + MoqLiteControlType.Session -> { + // ControlType=0 was the Lite-01/02 setup + // exchange. Lite-03 doesn't use it; if a + // legacy peer sends one, FIN cleanly so + // their bidi resolves rather than hanging. runCatching { bidi.finish() } dispatched = true } @@ -1194,6 +1235,15 @@ class MoqLiteSession internal constructor( /** moq-lite priority byte midpoint — neutral default. */ const val DEFAULT_PRIORITY: Int = 0x80 + /** + * Bitrate hint (bits/sec) we report on inbound moq-lite Probe + * bidis as a publisher. Mirrors the upper-bound of an Opus + * voice profile at 48 kHz mono, ≈32 kbps. Subscriber-side ABR + * estimators use this to size their forward queue; we emit the + * single hint and FIN since our encoder runs at a fixed bitrate. + */ + const val NESTS_AUDIO_BITRATE_HINT_BPS: Long = 32_000L + // Diagnostic: log "send returned false" once every N invocations. // At 50 fps and N=50 → ≤ 1 log/sec for a sustained no-sub window. private const val SEND_LOG_THROTTLE: Long = 50L From aa37a931e10f49da9bdb79fa31b938580211eab4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 03:48:57 +0000 Subject: [PATCH 17/41] test(okhttp): widen SurgeDns stale-refresh TTL to defeat double-expiry race The 1ms positiveTtlMs let the second refreshed entry expire again between the post-refresh assertion and the cache-hit lookup that followed, triggering an extra background refresh that broke the calls=2 invariant on slow CI runners. 100ms TTL with sleep(150) keeps the expiry boundary clearly outside the assertion window. --- .../vitorpamplona/amethyst/service/okhttp/SurgeDnsTest.kt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/service/okhttp/SurgeDnsTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/service/okhttp/SurgeDnsTest.kt index ff7ad1fd6..92384ab57 100644 --- a/amethyst/src/test/java/com/vitorpamplona/amethyst/service/okhttp/SurgeDnsTest.kt +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/service/okhttp/SurgeDnsTest.kt @@ -141,16 +141,19 @@ class SurgeDnsTest { responses.get() } val syncRefresh = Executor { it.run() } + // 100ms TTL gives both branches enough headroom on slow CI: Thread.sleep(150) reliably + // expires the first entry, and the post-refresh entry can't expire between the two + // assertions below (microseconds apart on any sane runner). val dns = SurgeDns( delegate = upstream, - positiveTtlMs = 1, + positiveTtlMs = 100, positiveTtlJitterMs = 0, refreshExecutor = syncRefresh, ) assertEquals(listOf(ip("1.2.3.4")), dns.lookup("a.example")) - Thread.sleep(20) + Thread.sleep(150) // Upstream now returns a new IP. The next lookup should still serve the OLD IP // immediately, while the synchronous executor performs the refresh inline. From ff9bb25921e2c89979b057c4d8a7417e20c23a05 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 15:13:47 +0000 Subject: [PATCH 18/41] refactor(nests): replace hand-built catalog JSON with typed MoqLiteHangCatalog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two SPEAKER_CATALOG_JSON call sites (MoqLiteNestsSpeaker for the non-reconnecting path, ReconnectingNestsSpeaker for the JWT-refresh hot- swap path) both built the manifest as a `\\` + `\"` + `+`-concatenated string literal. One missed escape and the watcher's parser fails silently — the broadcast becomes invisible to hang.js with no indication of why. The two literals had also drifted independently in review (same content today; nothing prevents drift tomorrow). Replace both with `MoqLiteHangCatalog.opusMono48k(track).encodeJsonBytes()`, a Serializable data class that encodes via kotlinx.serialization with `encodeDefaults = false` + `explicitNulls = false` pinned (so a future global default-flip can't surface `"bitrate": null` on hang.js's parser). The new type lives in :nestsClient because :nestsClient does not depend on :commons and the parser-side `RoomSpeakerCatalog` (in :commons) isn't reachable from the publish path. The two classes target the same wire shape independently; a byte-exact assertion in the new MoqLiteHangCatalogTest plus the existing RoomSpeakerCatalogTest's inline-literal round-trip together pin both sides — desync on either end fails one of the two tests immediately. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../viewmodels/RoomSpeakerCatalogTest.kt | 13 +- .../nestsclient/MoqLiteNestsSpeaker.kt | 30 +--- .../nestsclient/ReconnectingNestsSpeaker.kt | 4 +- .../moq/lite/MoqLiteHangCatalog.kt | 132 ++++++++++++++++++ .../moq/lite/MoqLiteHangCatalogTest.kt | 51 +++++++ 5 files changed, 196 insertions(+), 34 deletions(-) create mode 100644 nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHangCatalog.kt create mode 100644 nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHangCatalogTest.kt diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalogTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalogTest.kt index f8a45bcff..104099ef3 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalogTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalogTest.kt @@ -127,12 +127,13 @@ class RoomSpeakerCatalogTest { @Test fun stripPrefixRoundTripsCanonicalCatalog() { - // The catalog payload [com.vitorpamplona.nestsclient.MoqLiteNestsSpeaker] - // emits MUST round-trip through the parser — guards against - // either side drifting from the kixelated/hang shape. - // SPEAKER_CATALOG_JSON lives in nestsClient and isn't - // accessible from commons; keep an inline literal that mirrors - // it verbatim so a desync triggers this assertion. + // The catalog payload `MoqLiteHangCatalog.opusMono48k(...)` emits + // (in `:nestsClient`) MUST round-trip through this parser — the + // two classes target the same wire shape independently because + // `:nestsClient` does not depend on `:commons` and vice versa. + // `MoqLiteHangCatalog` isn't reachable from this module; keep an + // inline literal that mirrors what the publisher emits verbatim + // so a desync on either side trips this assertion. val emitted = "{\"audio\":{\"renditions\":{\"audio/data\":{" + "\"codec\":\"opus\",\"container\":{\"kind\":\"legacy\"}," + diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt index 67cbf3217..2fdd8e44b 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt @@ -23,6 +23,7 @@ package com.vitorpamplona.nestsclient import com.vitorpamplona.nestsclient.audio.AudioCapture import com.vitorpamplona.nestsclient.audio.NestMoqLiteBroadcaster import com.vitorpamplona.nestsclient.audio.OpusEncoder +import com.vitorpamplona.nestsclient.moq.lite.MoqLiteHangCatalog import com.vitorpamplona.nestsclient.moq.lite.MoqLitePublisherHandle import com.vitorpamplona.nestsclient.moq.lite.MoqLiteSession import kotlinx.coroutines.CancellationException @@ -155,7 +156,8 @@ class MoqLiteNestsSpeaker internal constructor( // every CATALOG_REPUBLISH_INTERVAL_MS so late-joining // watchers don't wait an arbitrary amount of time for the // next refresh. - val catalogJson = SPEAKER_CATALOG_JSON.encodeToByteArray() + val catalogJson = + MoqLiteHangCatalog.opusMono48k(MoqLiteNestsListener.AUDIO_TRACK).encodeJsonBytes() val republishJob = try { catalogPublisher.send(catalogJson) @@ -201,32 +203,6 @@ class MoqLiteNestsSpeaker internal constructor( } companion object { - /** - * moq-lite catalog manifest broadcast on the - * [MoqLiteNestsListener.CATALOG_TRACK] track. Verbatim - * canonical kixelated/moq `hang` shape (camelCase, nested - * `audio.renditions[]`, `container.kind = "legacy"`). - * The rendition key MUST equal the moq-lite track we publish - * audio frames on ([MoqLiteNestsListener.AUDIO_TRACK]) so the - * watcher's "subscribe to this rendition" path resolves to our - * actual audio stream. - * - * `container.kind = "legacy"` declares the wire layout the - * watcher must expect inside each moq-lite frame: a single - * `varint(timestamp_us)` prefix followed by the raw codec - * payload (no MOOF/MDAT wrapping). The publisher side honours - * this contract in - * [com.vitorpamplona.nestsclient.audio.NestMoqLiteBroadcaster]. - * - * Hard-coded because the speaker's encoder is fixed at - * 48 kHz mono Opus today; if [OpusEncoder] becomes parameterised - * this should be derived from the encoder config instead. - */ - const val SPEAKER_CATALOG_JSON: String = - "{\"audio\":{\"renditions\":{\"" + MoqLiteNestsListener.AUDIO_TRACK + "\":{" + - "\"codec\":\"opus\",\"container\":{\"kind\":\"legacy\"}," + - "\"sampleRate\":48000,\"numberOfChannels\":1}}}}" - /** * How often the catalog group is re-emitted. The relay drops * the catalog group when its last subscriber drops, so a diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsSpeaker.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsSpeaker.kt index 5bdb479b6..85d3b0d8d 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsSpeaker.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsSpeaker.kt @@ -23,6 +23,7 @@ package com.vitorpamplona.nestsclient import com.vitorpamplona.nestsclient.audio.AudioCapture import com.vitorpamplona.nestsclient.audio.NestMoqLiteBroadcaster import com.vitorpamplona.nestsclient.audio.OpusEncoder +import com.vitorpamplona.nestsclient.moq.lite.MoqLiteHangCatalog import com.vitorpamplona.nestsclient.moq.lite.MoqLitePublisherHandle import com.vitorpamplona.nestsclient.transport.WebTransportFactory import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner @@ -572,7 +573,8 @@ private class ReissuingBroadcastHandle( // and any watcher that attaches AFTER the recycle sees nothing // to subscribe to. Mirror of [MoqLiteNestsSpeaker.startBroadcasting]'s // catalog setup; same JSON, same republish cadence. - val catalogPayload = MoqLiteNestsSpeaker.SPEAKER_CATALOG_JSON.encodeToByteArray() + val catalogPayload = + MoqLiteHangCatalog.opusMono48k(MoqLiteNestsListener.AUDIO_TRACK).encodeJsonBytes() val priorCatalogPublisher = hotSwapCatalogPublisher val priorCatalogJob = hotSwapCatalogJob val newCatalogPublisher = diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHangCatalog.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHangCatalog.kt new file mode 100644 index 000000000..020756a47 --- /dev/null +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHangCatalog.kt @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.nestsclient.moq.lite + +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json + +/** + * Publisher-side model for the kixelated/moq `hang` `catalog.json` + * track. Kept distinct from + * `com.vitorpamplona.amethyst.commons.viewmodels.RoomSpeakerCatalog` + * (the parser in `:commons`) because `:nestsClient` does not depend on + * `:commons` — both classes target the same wire shape independently. + * + * Wire shape (verbatim from `kixelated/moq/rs/hang/src/catalog/`): + * + * { + * "audio": { + * "renditions": { + * "": { + * "codec": "opus", + * "container": { "kind": "legacy" }, + * "sampleRate": 48000, + * "numberOfChannels": 1, + * "bitrate": 32000 // optional + * } + * } + * } + * } + * + * Field names are camelCase per the upstream serde + * `rename_all = "camelCase"`. Every Amethyst-emitted field is required + * here (no nullable defaults) so the encoded JSON is stable byte-for- + * byte across runs and we don't accidentally publish `"bitrate": null` + * if hang.js's parser turns out to reject it. + */ +@Serializable +internal data class MoqLiteHangCatalog( + val audio: Audio, +) { + @Serializable + data class Audio( + val renditions: Map, + ) + + @Serializable + data class AudioRendition( + val codec: String, + val container: Container, + val sampleRate: Int, + val numberOfChannels: Int, + ) + + @Serializable + data class Container( + val kind: String, + ) + + /** UTF-8 JSON bytes ready to push on the `catalog.json` moq-lite track. */ + fun encodeJsonBytes(): ByteArray = JSON.encodeToString(serializer(), this).encodeToByteArray() + + companion object { + /** + * Json instance dedicated to catalog encoding. We can't reuse + * `:quartz`'s `JsonMapper` from common code without pulling + * `:commons` (which already imports it) into `:nestsClient`'s + * dependency closure — `:nestsClient` deliberately stays free + * of the UI/state layer. A local instance keeps the wire-format + * configuration (camelCase serde defaults, no pretty-printing, + * deterministic field order) co-located with the model. + */ + private val JSON = + Json { + // Default for kotlinx.serialization is `false`, but pin + // explicitly so a future global default-flip doesn't + // surface `"bitrate": null` on hang.js's parser. + encodeDefaults = false + explicitNulls = false + } + + /** + * Canonical Amethyst speaker catalog: a single `legacy`-container + * Opus rendition under [audioTrackName], matching the encoder + * config in [com.vitorpamplona.nestsclient.audio.OpusEncoder] + * (48 kHz mono). + * + * The rendition map is keyed by the moq-lite track name a + * subscriber should subscribe to for this rendition's frames — + * for nests audio rooms that's the same string the publisher + * publishes audio frames on + * (`MoqLiteNestsListener.AUDIO_TRACK`). + * + * If [com.vitorpamplona.nestsclient.audio.OpusEncoder] becomes + * parameterised in the future, this factory should take the + * encoder's config rather than hard-coding 48 kHz mono. + */ + fun opusMono48k(audioTrackName: String): MoqLiteHangCatalog = + MoqLiteHangCatalog( + audio = + Audio( + renditions = + mapOf( + audioTrackName to + AudioRendition( + codec = "opus", + container = Container(kind = "legacy"), + sampleRate = 48_000, + numberOfChannels = 1, + ), + ), + ), + ) + } +} diff --git a/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHangCatalogTest.kt b/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHangCatalogTest.kt new file mode 100644 index 000000000..1eb7adbde --- /dev/null +++ b/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHangCatalogTest.kt @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.nestsclient.moq.lite + +import kotlin.test.Test +import kotlin.test.assertEquals + +class MoqLiteHangCatalogTest { + @Test + fun opusMono48kEmitsCanonicalHangShape() { + // Byte-exact assertion: `:commons`'s `RoomSpeakerCatalogTest` + // round-trips this same string against the parser. If either + // side drifts from the kixelated/hang wire shape, both tests + // fail at once. + val expected = + "{\"audio\":{\"renditions\":{\"audio/data\":{" + + "\"codec\":\"opus\",\"container\":{\"kind\":\"legacy\"}," + + "\"sampleRate\":48000,\"numberOfChannels\":1}}}}" + val actual = + MoqLiteHangCatalog.opusMono48k("audio/data").encodeJsonBytes().decodeToString() + assertEquals(expected, actual) + } + + @Test + fun renditionKeyMatchesCallerSuppliedTrackName() { + val actual = + MoqLiteHangCatalog.opusMono48k("custom/track").encodeJsonBytes().decodeToString() + // The rendition map MUST be keyed on the caller-supplied track + // name — the watcher uses this string verbatim as the + // SUBSCRIBE.track on the audio subscription. + assertEquals(true, actual.contains("\"custom/track\":{")) + } +} From 1b0740d4a83dc2185580dd4a2913d3ea4111f25e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 15:18:54 +0000 Subject: [PATCH 19/41] feat(nests): emit catalog jitter hint matching Opus frame duration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hang.js's encoder publishes a `jitter` field (ms) in every audio rendition; the watcher uses it to size its jitter buffer. Without it the watcher falls back to a built-in default — works in practice today but means our publishes deviate from the canonical hang shape and the deviation could surface as audible buffer-sizing differences on a future watcher revision that takes the field as required. Add `jitter: 20` to MoqLiteHangCatalog.AudioRendition (matching the Opus frame cadence: 960 samples at 48 kHz = 20 ms; same value hang.js hard-codes at `js/publish/src/audio/encoder.ts:#runConfig`). Updates the byte-exact MoqLiteHangCatalogTest assertion and the parallel inline-literal in RoomSpeakerCatalogTest so both sides stay pinned. The commons-side `RoomSpeakerCatalog` parser doesn't surface the new field — `JsonMapper`'s `ignoreUnknownKeys = true` lets the watcher ingest it silently. Adding a typed `jitter` field there is a forward-compat task; not needed for interop with hang today. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../viewmodels/RoomSpeakerCatalogTest.kt | 7 ++++++- .../moq/lite/MoqLiteHangCatalog.kt | 21 +++++++++++++++++++ .../moq/lite/MoqLiteHangCatalogTest.kt | 2 +- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalogTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalogTest.kt index 104099ef3..7286d83e8 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalogTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalogTest.kt @@ -137,7 +137,7 @@ class RoomSpeakerCatalogTest { val emitted = "{\"audio\":{\"renditions\":{\"audio/data\":{" + "\"codec\":\"opus\",\"container\":{\"kind\":\"legacy\"}," + - "\"sampleRate\":48000,\"numberOfChannels\":1}}}}" + "\"sampleRate\":48000,\"numberOfChannels\":1,\"jitter\":20}}}}" val catalog = RoomSpeakerCatalog.parseOrNull(emitted.encodeToByteArray()) assertNotNull(catalog) val rendition = catalog.primaryAudio() @@ -146,5 +146,10 @@ class RoomSpeakerCatalogTest { assertEquals("legacy", rendition.container?.kind) assertEquals(48_000, rendition.sampleRate) assertEquals(1, rendition.numberOfChannels) + // `jitter` is intentionally not asserted on `rendition` — the + // commons-side parser drops unknown fields via the JsonMapper's + // `ignoreUnknownKeys = true`. Adding it to `RoomSpeakerCatalog` + // is forward work; the byte-exact match in the literal above + // already pins the wire shape. } } diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHangCatalog.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHangCatalog.kt index 020756a47..6e2483790 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHangCatalog.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHangCatalog.kt @@ -40,6 +40,7 @@ import kotlinx.serialization.json.Json * "container": { "kind": "legacy" }, * "sampleRate": 48000, * "numberOfChannels": 1, + * "jitter": 20, // ms; matches one Opus frame * "bitrate": 32000 // optional * } * } @@ -67,6 +68,15 @@ internal data class MoqLiteHangCatalog( val container: Container, val sampleRate: Int, val numberOfChannels: Int, + /** + * Publisher-side cadence hint (ms). hang.js's watcher uses + * this to size its jitter buffer — emit a real value rather + * than relying on the watcher's fallback default. For Opus + * this is the codec frame duration (20 ms at 48 kHz / 960 + * samples) and matches what hang.js's encoder emits at + * `js/publish/src/audio/encoder.ts:#runConfig`. + */ + val jitter: Int, ) @Serializable @@ -124,9 +134,20 @@ internal data class MoqLiteHangCatalog( container = Container(kind = "legacy"), sampleRate = 48_000, numberOfChannels = 1, + jitter = OPUS_FRAME_DURATION_MS, ), ), ), ) + + /** + * Opus frame duration in milliseconds — 960 samples / 48 kHz = + * 20 ms. Matches + * [com.vitorpamplona.nestsclient.audio.Audio.FRAME_SIZE_SAMPLES]. + * Published as the catalog `jitter` hint so hang.js's watcher + * sizes its jitter buffer to one Opus frame, the natural + * cadence of our encoder. + */ + private const val OPUS_FRAME_DURATION_MS: Int = 20 } } diff --git a/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHangCatalogTest.kt b/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHangCatalogTest.kt index 1eb7adbde..138bb17b1 100644 --- a/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHangCatalogTest.kt +++ b/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHangCatalogTest.kt @@ -33,7 +33,7 @@ class MoqLiteHangCatalogTest { val expected = "{\"audio\":{\"renditions\":{\"audio/data\":{" + "\"codec\":\"opus\",\"container\":{\"kind\":\"legacy\"}," + - "\"sampleRate\":48000,\"numberOfChannels\":1}}}}" + "\"sampleRate\":48000,\"numberOfChannels\":1,\"jitter\":20}}}}" val actual = MoqLiteHangCatalog.opusMono48k("audio/data").encodeJsonBytes().decodeToString() assertEquals(expected, actual) From 361dbbe201e3c2e3b859cb6e6bab68d3a33fa60a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 15:23:29 +0000 Subject: [PATCH 20/41] =?UTF-8?q?fix(nests):=20stamp=20Opus=20frames=20at?= =?UTF-8?q?=20frame-index=20=C3=97=20frame-duration,=20not=20wall=20clock?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The legacy-container timestamp on each audio frame was previously captured at send-time via `frameStartMark.elapsedNow()`. That mark advances on wall clock, but `current.send(...)` is a suspending call that backs off on transport backpressure — so a frame captured at T gets stamped at T+δ once the previous send drains, and the watcher's WebCodecs AudioDecoder schedules playback at the wrong instant. Audible as a slowly-growing offset between speaker and listener over the life of a session that's seen any meaningful send-side queueing. Replace with a frame-index counter pinned to the codec grid: `timestampUs = nextFrameIndex * AudioFormat.FRAME_DURATION_US`. The counter advances once per captured PCM frame BEFORE the encode/mute/ send path, so: - Encoder failures, empty-encoded frames, and muted frames each consume one slot — preserving the wall-clock gap on the wire so a 5 s mute shows up as a 5 s timestamp jump (matches the prior wall-clock behaviour for mute and silence). - Send-time stalls don't drift the timestamp: it's computed at the top of the iteration, not after the suspending send. - Survives publisher hot-swap (single-broadcast, single-encoder; the new publisher inherits the running counter) — same guarantee the prior `TimeMark` had. Also adds [AudioFormat.FRAME_DURATION_US] = 20_000 (960 / 48 kHz × 1e6) so the constant lives next to FRAME_SIZE_SAMPLES rather than being implicit in every call site. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../vitorpamplona/nestsclient/audio/Audio.kt | 9 ++++ .../audio/NestMoqLiteBroadcaster.kt | 45 +++++++++++++++---- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/Audio.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/Audio.kt index e015a49c1..c2be91b94 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/Audio.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/Audio.kt @@ -36,6 +36,15 @@ object AudioFormat { /** 20 ms at 48 kHz. */ const val FRAME_SIZE_SAMPLES: Int = 960 + /** + * Duration of one Opus frame in microseconds — derived from + * [FRAME_SIZE_SAMPLES] / [SAMPLE_RATE_HZ]. Used by the publisher + * to stamp each frame at frame-index × this value, giving hang.js's + * watcher a perfect 20 ms cadence even when the broadcaster's send + * loop suspends on transport backpressure. + */ + const val FRAME_DURATION_US: Long = (FRAME_SIZE_SAMPLES.toLong() * 1_000_000L) / SAMPLE_RATE_HZ + /** Bytes per PCM 16-bit sample. */ const val BYTES_PER_SAMPLE: Int = 2 } diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt index cf10c7a57..fadda7a28 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt @@ -27,8 +27,6 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.launch -import kotlin.time.TimeMark -import kotlin.time.TimeSource /** * Mirror of [NestBroadcaster] but driving a moq-lite @@ -207,15 +205,44 @@ class NestMoqLiteBroadcaster( // will skip the leading varint as a microsecond // timestamp; they MUST receive a real timestamp or // their decoder picks up garbage bytes ahead of the - // Opus packet. We use a monotonic mark captured at - // capture-loop start so timestamps are robust to mute - // gaps and survive publisher hot-swap (the wire is - // single-broadcast, single-encoder; timestamps are - // codec-payload metadata, not stream-position). - val frameStartMark: TimeMark = TimeSource.Monotonic.markNow() + // Opus packet. + // + // Frame-index-derived timestamp (NOT wall clock). Each + // captured PCM frame occupies exactly one + // [AudioFormat.FRAME_DURATION_US] slot in the timeline, + // and the timestamp is `nextFrameIndex * + // FRAME_DURATION_US` snapped to that grid. This matters + // because: + // - `current.send(...)` suspends on transport + // backpressure; a wall-clock timestamp captured at + // send-time would drift forward of the actual + // capture time of the audio sample, and the watcher's + // WebCodecs AudioDecoder would schedule playback + // at the wrong instant (audible offset between + // speaker and listener). + // - The capture loop is wall-clock-pinned anyway — + // `capture.readFrame()` blocks until the next 20 ms + // of PCM is ready — so frame-index advances at the + // same rate as wall time across the whole + // broadcast lifetime, with no codec drift relative + // to the watcher's playback clock. + // + // The counter advances on EVERY captured PCM frame — + // including encoder failures, empty-encoded frames, and + // muted frames — so a mute gap shows up on the wire as + // a real wall-clock gap (timestamps jump forward by the + // muted duration) rather than collapsing the silence. + // Survives publisher hot-swap (single-broadcast, + // single-encoder; the new publisher inherits the + // running counter) for the same reason the old + // wall-clock TimeMark did: timestamps are codec-payload + // metadata, not stream-position. + var nextFrameIndex = 0L try { while (true) { val pcm = capture.readFrame() ?: break + val timestampUs = nextFrameIndex * AudioFormat.FRAME_DURATION_US + nextFrameIndex += 1 val opus = try { encoder.encode(pcm) @@ -251,7 +278,7 @@ class NestMoqLiteBroadcaster( // for the production cliff this works around. val sendOutcome = runCatching { - val tsBytes = Varint.encode(frameStartMark.elapsedNow().inWholeMicroseconds) + val tsBytes = Varint.encode(timestampUs) val payload = ByteArray(tsBytes.size + opus.size) tsBytes.copyInto(payload, 0) opus.copyInto(payload, tsBytes.size) From 4b736263e0939d9f1c2475663b60c2861d4ad8b3 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 15:27:21 +0000 Subject: [PATCH 21/41] feat(nests): advertise both moq-lite-03 and moq-lite-04 ALPNs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Production moq-relay (moq.nostrnests.com) and the kixelated browser client both ship Lite-04 now. We previously only advertised moq-lite-03 in the wt-available-protocols header, so a Lite-04-only relay deployment would refuse the WebTransport CONNECT (no mutual sub-protocol). Add moq-lite-04 to the advertised list. The on-the-wire framing for Subscribe / Group / Announce did NOT change between Lite-03 and Lite-04, so the existing codec handles either selection unchanged — this is purely a handshake-layer addition. moq-lite-03 stays listed first so a relay that supports both still picks the previously-tested path. Constants now live on MoqLiteAlpn alongside LITE_03 and LEGACY rather than being a string literal in QuicWebTransportFactory. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../nestsclient/moq/lite/MoqLiteMessages.kt | 13 +++++++++++-- .../transport/QuicWebTransportFactory.kt | 14 +++++++++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteMessages.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteMessages.kt index 3d8504c1a..0ad9d08d6 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteMessages.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteMessages.kt @@ -21,14 +21,23 @@ package com.vitorpamplona.nestsclient.moq.lite /** - * moq-lite ALPN strings. Lite-03 is preferred; `"moql"` is the legacy - * combined ALPN that requires an in-band SETUP exchange. + * moq-lite ALPN strings. The on-the-wire framing for Subscribe / Group + * / Announce did NOT change between Lite-03 and Lite-04, so advertising + * both via `wt-available-protocols` is safe — the relay picks whichever + * it prefers and our existing codec paths handle either. We keep 03 in + * the list as the previously-tested target until the production relay's + * Lite-04 path has interop coverage. + * + * `"moql"` is the legacy combined ALPN that requires an in-band SETUP + * exchange — kept here for completeness; not advertised by the + * factory. * * Source: `kixelated/moq-rs/rs/moq-lite/src/version.rs:21-26`, * `@moq/lite/connection/connect.js:277`. */ object MoqLiteAlpn { const val LITE_03: String = "moq-lite-03" + const val LITE_04: String = "moq-lite-04" const val LEGACY: String = "moql" } diff --git a/nestsClient/src/jvmAndroid/kotlin/com/vitorpamplona/nestsclient/transport/QuicWebTransportFactory.kt b/nestsClient/src/jvmAndroid/kotlin/com/vitorpamplona/nestsclient/transport/QuicWebTransportFactory.kt index 5f24c438d..f305fc24c 100644 --- a/nestsClient/src/jvmAndroid/kotlin/com/vitorpamplona/nestsclient/transport/QuicWebTransportFactory.kt +++ b/nestsClient/src/jvmAndroid/kotlin/com/vitorpamplona/nestsclient/transport/QuicWebTransportFactory.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.nestsclient.transport +import com.vitorpamplona.nestsclient.moq.lite.MoqLiteAlpn import com.vitorpamplona.quic.connection.QuicConnection import com.vitorpamplona.quic.connection.QuicConnectionConfig import com.vitorpamplona.quic.connection.QuicConnectionDriver @@ -89,8 +90,19 @@ class QuicWebTransportFactory( * post-CONNECT message is decoded as SETUP_CLIENT, producing * `connection closed err=invalid value` on the relay side and a stalled * subscribe / `subscribe stream FIN before reply` on the client side. + * + * `moq-lite-04` is also advertised because the production relay + * (moq.nostrnests.com) and the kixelated browser client both ship + * Lite-04 now; without it in our list a Lite-04-only deployment + * would refuse the CONNECT. The on-the-wire framing for Subscribe / + * Group / Announce did NOT change between Lite-03 and Lite-04, so + * our existing codec handles either selection. List `moq-lite-03` + * first because it's our previously-tested negotiation target — + * a relay that supports both should pick the listed-first option, + * and a Lite-04-only relay falls through to the second entry. */ - private val webTransportSubProtocols: List = listOf("moq-lite-03"), + private val webTransportSubProtocols: List = + listOf(MoqLiteAlpn.LITE_03, MoqLiteAlpn.LITE_04), ) : WebTransportFactory { override suspend fun connect( authority: String, From 79c76d58311506d1c6275db54d012cbd801c64f3 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 15:32:32 +0000 Subject: [PATCH 22/41] fix(nests): reply SubscribeDrop on unsupported tracks instead of silent FIN MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a relay opens a SUBSCRIBE bidi for a track this session doesn't publish (e.g. a watcher subscribes to `video/data` against a broadcast that only declares `audio/data` + `catalog.json`), the session was silently FINing the bidi without writing any reply. The peer's wait on its receive side resolves to end-of-stream with no indication WHY — indistinguishable from "publisher disappeared mid-subscribe." Reply with [MoqLiteSubscribeDrop] carrying [MoqLiteSubscribeDropCode.TRACK_DOES_NOT_EXIST] (0x04, mirroring the IETF-MoQ `ErrorCode.TRACK_DOES_NOT_EXIST` convention) and an informational reason phrase listing the tracks we DO publish, then FIN. Matches what kixelated's `rs/moq-lite/src/lite/subscribe.rs` expects on this path. Doesn't change the happy path — every track NestsUI-v2 subscribes to (`audio/data` + `catalog.json`) is now published — but is the spec-correct behaviour for any future watcher that prospectively subscribes to a track our publisher hasn't declared. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../nestsclient/moq/lite/MoqLiteMessages.kt | 30 ++++++++++++- .../nestsclient/moq/lite/MoqLiteSession.kt | 27 +++++++++++- .../moq/lite/MoqLiteSessionTest.kt | 44 +++++++++++++++++++ 3 files changed, 97 insertions(+), 4 deletions(-) diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteMessages.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteMessages.kt index 0ad9d08d6..59d8af06d 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteMessages.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteMessages.kt @@ -231,14 +231,40 @@ data class MoqLiteSubscribeOk( * RESET_STREAM, but the publisher can pre-emptively decline a * subscription with [MoqLiteSubscribeDrop] before any group flows. * - * Decode-only as far as the client cares — we never send Drop back - * upstream. + * Sent by [MoqLiteSession] when a relay opens a SUBSCRIBE bidi for a + * `track` we don't publish (e.g. a watcher subscribes to a `video/data` + * rendition but our broadcast only declares `audio/data` + `catalog.json` + * in its catalog). Without a Drop reply the watcher's response wait + * resolves only when the bidi is FIN'd, with no indication WHY — Drop + * carries the error code and a reason phrase the watcher can log / + * surface. + * + * Decoded by [MoqLiteSession] when the relay or upstream publisher + * rejects one of OUR subscriptions — we surface it as + * [MoqLiteSubscribeException] so callers see a typed protocol-level + * rejection rather than a silent end-of-flow. */ data class MoqLiteSubscribeDrop( val errorCode: Long, val reasonPhrase: String, ) +/** + * Error codes carried in the [MoqLiteSubscribeDrop.errorCode] varint. + * moq-lite leaves these application-defined; we mirror the IETF-MoQ + * conventions in `com.vitorpamplona.nestsclient.moq.ErrorCode` so a + * cross-protocol reader gets the same semantic from either path. + */ +object MoqLiteSubscribeDropCode { + /** + * The publisher does not serve this `(broadcast, track)` tuple. + * Sent for a subscribe whose `track` doesn't match any of the + * publishers we registered on this session. Mirrors + * `com.vitorpamplona.nestsclient.moq.ErrorCode.TRACK_DOES_NOT_EXIST`. + */ + const val TRACK_DOES_NOT_EXIST: Long = 0x04L +} + /** * Header at the start of a Group uni stream. After the * [MoqLiteDataType.Group] type byte, the publisher writes one diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt index ff31ab993..bf85c60e4 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt @@ -849,11 +849,34 @@ class MoqLiteSession internal constructor( // single-track-per-publisher model, only one match is possible. val targetPublisher = publishersSnapshot.firstOrNull { it.track == sub.track } if (targetPublisher == null) { + // Reply SubscribeDrop with a TRACK_DOES_NOT_EXIST + // error code BEFORE we FIN — without this the + // peer's response wait resolves only on + // bidi-FIN with no indication WHY (looks + // identical to "publisher disappeared mid- + // subscribe"). Drop carries the error code + + // reason phrase the watcher can log / + // surface, and matches what kixelated's + // `rs/moq-lite/src/lite/subscribe.rs` + // expects for an unrecognised track on a + // live broadcast. Log.w("NestTx") { "SUBSCRIBE inbound id=${sub.id} track='${sub.track}' has no matching publisher " + - "on this session (have ${publishersSnapshot.map { it.track }}) — finishing bidi" + "on this session (have ${publishersSnapshot.map { it.track }}) — replying SubscribeDrop" + } + runCatching { + bidi.write( + MoqLiteCodec.encodeSubscribeDrop( + MoqLiteSubscribeDrop( + errorCode = MoqLiteSubscribeDropCode.TRACK_DOES_NOT_EXIST, + reasonPhrase = + "track '${sub.track}' is not published on this broadcast " + + "(available: ${publishersSnapshot.joinToString(",") { it.track }})", + ), + ), + ) + bidi.finish() } - runCatching { bidi.finish() } dispatched = true return@collect } diff --git a/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSessionTest.kt b/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSessionTest.kt index a7cad504d..70b1d5eb4 100644 --- a/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSessionTest.kt +++ b/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSessionTest.kt @@ -357,6 +357,50 @@ class MoqLiteSessionTest { session.close() } + @Test + fun publisher_replies_subscribeDrop_when_track_is_not_published() = + runBlocking { + val (clientSide, serverSide) = FakeWebTransport.pair() + val session = MoqLiteSession.client(clientSide, pumpScope) + + // Publisher serves audio/data only. + val publisher = session.publish(broadcastSuffix = "speakerPubkey", track = "audio/data") + + // Relay opens a Subscribe bidi for a DIFFERENT track. The + // session must reply with SubscribeDrop carrying the + // TRACK_DOES_NOT_EXIST code rather than a silent FIN — + // otherwise the watcher's response wait resolves only when + // the bidi is FIN'd, with no indication WHY (looks + // identical to "publisher disappeared mid-subscribe"). + val subBidi = serverSide.openBidiStream() + subBidi.write(Varint.encode(MoqLiteControlType.Subscribe.code)) + subBidi.write( + MoqLiteCodec.encodeSubscribe( + MoqLiteSubscribe( + id = 99L, + broadcast = "speakerPubkey", + track = "video/data", + priority = 0x80, + ordered = true, + maxLatencyMillis = 0L, + startGroup = null, + endGroup = null, + ), + ), + ) + + val ackChunk = withTimeout(2_000) { subBidi.incoming().first() } + val resp = MoqLiteCodec.decodeSubscribeResponse(ackChunk) + val dropped = resp as MoqLiteCodec.SubscribeResponse.Dropped + assertEquals(MoqLiteSubscribeDropCode.TRACK_DOES_NOT_EXIST, dropped.drop.errorCode) + // Reason phrase is informational; pin substring rather than + // the exact text so we can keep tweaking the wording. + kotlin.test.assertContains(dropped.drop.reasonPhrase, "video/data") + + publisher.close() + session.close() + } + @Test fun publisher_close_emits_ended_announce() = runBlocking { From 3417e44a6ab8cb02e5b0666814256408814e58b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 16:10:12 +0000 Subject: [PATCH 23/41] refactor(nests): catalog emit-on-subscribe hook replaces 2s republish loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The catalog publisher's previous shape was "send once at startup (silently fails if no subs are attached yet) + a 2 s republish loop". That worked but had two problems: 1. The startup send is a no-op (PublisherStateImpl.send returns false when inboundSubs is empty), so the catalog isn't actually on the wire until the first loop tick — up to 2 s of catalog dead time for a watcher that subscribes immediately. 2. Re-emitting a static blob every 2 s for the broadcast's lifetime wastes a fresh group on the relay's per-track cache; not a hot path today but unnecessary. Replace with an emit-on-subscribe hook. New API on MoqLitePublisherHandle: fun setOnNewSubscriber(hook: (suspend () -> Unit)?) PublisherStateImpl fires the hook once per accepted (track-matching) inbound SUBSCRIBE, OUTSIDE its serialisation lock so the hook can safely call send/endGroup without deadlock. Track-mismatched subs (SubscribeDrop reply) do NOT fire the hook — covered by a new "hook does not fire on track mismatch" test. MoqLiteNestsSpeaker.startBroadcasting and ReconnectingNestsSpeaker's hot-swap iteration both now register a hook that writes the catalog JSON on every fresh subscribe instead of looping. The catalogRepublishJob field on MoqLiteBroadcastHandle and the hotSwapCatalogJob field on ReissuingBroadcastHandle (plus their close paths) are dropped. Net effect for hang.js / NestsUI-v2 watchers: - Catalog reaches the watcher on the SAME group as the subscribe ack (no 0–2 s startup gap). - Late joiners hit the relay's track-latest cache for the catalog blob — a fresh hook fire only happens when the relay opens a new SUBSCRIBE bidi to us (cliff-detector recycle on listener side, or JWT-refresh on our side). - One catalog group per relay-side subscribe instead of one every 2 s for the broadcast lifetime. Two tests pin the new behaviour: hook fires on inbound subscribe; hook does NOT fire on a track-mismatched subscribe (which gets a SubscribeDrop reply and never enters inboundSubs). https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../nestsclient/MoqLiteNestsSpeaker.kt | 75 +++-------- .../nestsclient/ReconnectingNestsSpeaker.kt | 62 +++------- .../nestsclient/moq/lite/MoqLiteSession.kt | 49 ++++++++ .../moq/lite/MoqLiteSessionTest.kt | 117 ++++++++++++++++++ 4 files changed, 200 insertions(+), 103 deletions(-) diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt index 2fdd8e44b..3c0769d55 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt @@ -28,13 +28,9 @@ import com.vitorpamplona.nestsclient.moq.lite.MoqLitePublisherHandle import com.vitorpamplona.nestsclient.moq.lite.MoqLiteSession import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Job -import kotlinx.coroutines.cancelAndJoin -import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow -import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock @@ -150,39 +146,25 @@ class MoqLiteNestsSpeaker internal constructor( runCatching { publisher.close() } throw t } - // Push catalog once before launching the periodic republish - // pump so the manifest is on the wire before any subscriber - // attaches; the republisher then re-emits a fresh group - // every CATALOG_REPUBLISH_INTERVAL_MS so late-joining - // watchers don't wait an arbitrary amount of time for the - // next refresh. + // Catalog emit-on-subscribe: every time the relay opens a + // SUBSCRIBE bidi for catalog.json, fire the hook to write + // one group + FIN. moq-lite serves new listeners from the + // relay's per-track latest-group cache, so emitting once + // per relay-side subscribe is enough — late-joining + // watchers behind the same relay get the cached blob + // without us having to maintain a periodic re-emit loop. + // Set BEFORE the relay can race a SUBSCRIBE in; in + // practice the relay's SUBSCRIBE bidi takes a network + // round-trip after our ANNOUNCE Active, so this is safe + // even though the setter is non-suspending. val catalogJson = MoqLiteHangCatalog.opusMono48k(MoqLiteNestsListener.AUDIO_TRACK).encodeJsonBytes() - val republishJob = - try { + catalogPublisher.setOnNewSubscriber { + runCatching { catalogPublisher.send(catalogJson) catalogPublisher.endGroup() - scope.launch { - try { - while (true) { - delay(CATALOG_REPUBLISH_INTERVAL_MS) - catalogPublisher.send(catalogJson) - catalogPublisher.endGroup() - } - } catch (ce: CancellationException) { - throw ce - } catch (_: Throwable) { - // Best-effort: a transient catalog send - // failure is non-fatal — the audio path - // owns terminal-failure detection. - } - } - } catch (t: Throwable) { - runCatching { catalogPublisher.close() } - runCatching { broadcaster.stop() } - runCatching { publisher.close() } - throw t } + } mutableState.value = NestsSpeakerState.Broadcasting( room = current.room, @@ -194,7 +176,6 @@ class MoqLiteNestsSpeaker internal constructor( broadcaster = broadcaster, publisher = publisher, catalogPublisher = catalogPublisher, - catalogRepublishJob = republishJob, parent = this, ) activeHandle = handle @@ -202,17 +183,6 @@ class MoqLiteNestsSpeaker internal constructor( } } - companion object { - /** - * How often the catalog group is re-emitted. The relay drops - * the catalog group when its last subscriber drops, so a - * watcher that attaches mid-broadcast needs a freshly-published - * group to receive the manifest. 2 s keeps late-attach worst - * case bounded without spamming the relay. - */ - const val CATALOG_REPUBLISH_INTERVAL_MS: Long = 2_000L - } - /** * [HotSwappablePublisherSource] implementation. See the interface * kdoc — this method mints a fresh publisher on the session @@ -302,7 +272,6 @@ internal class MoqLiteBroadcastHandle( private val broadcaster: NestMoqLiteBroadcaster, private val publisher: MoqLitePublisherHandle, private val catalogPublisher: MoqLitePublisherHandle, - private val catalogRepublishJob: Job, private val parent: MoqLiteNestsSpeaker, ) : BroadcastHandle { @Volatile private var muted: Boolean = false @@ -321,20 +290,8 @@ internal class MoqLiteBroadcastHandle( override suspend fun close() { if (closed) return closed = true - // Stop the catalog republisher first so it doesn't race a - // concurrent send against the publisher.close below. - try { - catalogRepublishJob.cancelAndJoin() - } catch (ce: kotlinx.coroutines.CancellationException) { - // The parent scope is being cancelled. Continue cleanup of - // the audio + publisher resources, then rethrow. - runCatching { catalogPublisher.close() } - runCatching { publisher.close() } - parent.broadcastClosed(this) - throw ce - } catch (_: Throwable) { - // Best-effort. - } + // Stop the broadcaster first so the audio capture + encoder + // don't keep producing into a closing publisher. try { broadcaster.stop() } catch (ce: kotlinx.coroutines.CancellationException) { diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsSpeaker.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsSpeaker.kt index 85d3b0d8d..e532e28a0 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsSpeaker.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsSpeaker.kt @@ -30,7 +30,6 @@ import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.awaitCancellation -import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow @@ -442,8 +441,6 @@ private class ReissuingBroadcastHandle( */ @Volatile private var hotSwapCatalogPublisher: MoqLitePublisherHandle? = null - @Volatile private var hotSwapCatalogJob: Job? = null - /** Legacy path's per-session handle. Cleared when the session swaps. */ private val liveHandle = AtomicReference(null) private var pumpJob: Job? = null @@ -572,11 +569,10 @@ private class ReissuingBroadcastHandle( // this the catalog goes silent the moment the session recycles // and any watcher that attaches AFTER the recycle sees nothing // to subscribe to. Mirror of [MoqLiteNestsSpeaker.startBroadcasting]'s - // catalog setup; same JSON, same republish cadence. + // catalog setup; same JSON, same emit-on-subscribe pattern. val catalogPayload = MoqLiteHangCatalog.opusMono48k(MoqLiteNestsListener.AUDIO_TRACK).encodeJsonBytes() val priorCatalogPublisher = hotSwapCatalogPublisher - val priorCatalogJob = hotSwapCatalogJob val newCatalogPublisher = try { hotSwap.openPublisherForHotSwap(MoqLiteNestsListener.CATALOG_TRACK) @@ -590,45 +586,23 @@ private class ReissuingBroadcastHandle( null } if (newCatalogPublisher != null) { - val newCatalogJob = - try { + // Set the emit-on-subscribe hook BEFORE installing the + // publisher reference, so the relay can't race a SUBSCRIBE + // in between. + newCatalogPublisher.setOnNewSubscriber { + runCatching { newCatalogPublisher.send(catalogPayload) newCatalogPublisher.endGroup() - scope.launch { - try { - while (true) { - delay(MoqLiteNestsSpeaker.CATALOG_REPUBLISH_INTERVAL_MS) - newCatalogPublisher.send(catalogPayload) - newCatalogPublisher.endGroup() - } - } catch (ce: kotlinx.coroutines.CancellationException) { - throw ce - } catch (_: Throwable) { - // Best-effort: a transient catalog send - // failure on this session is non-fatal — - // the audio path owns terminal-failure - // detection. - } - } - } catch (ce: kotlinx.coroutines.CancellationException) { - runCatching { newCatalogPublisher.close() } - throw ce - } catch (_: Throwable) { - runCatching { newCatalogPublisher.close() } - null } - if (newCatalogJob != null) { - hotSwapCatalogPublisher = newCatalogPublisher - hotSwapCatalogJob = newCatalogJob - // Tear down the prior session's catalog state AFTER the - // new one is installed so the watcher only sees a brief - // overlap rather than a gap. The prior publisher's - // session is about to be torn down by the orchestrator - // anyway, but graceful Announce(Ended) keeps the relay - // book-keeping clean. - if (priorCatalogJob != null) runCatching { priorCatalogJob.cancelAndJoin() } - if (priorCatalogPublisher != null) runCatching { priorCatalogPublisher.close() } } + hotSwapCatalogPublisher = newCatalogPublisher + // Tear down the prior session's catalog publisher AFTER the + // new one is installed so the watcher only sees a brief + // overlap rather than a gap. The prior publisher's session + // is about to be torn down by the orchestrator anyway, but + // graceful Announce(Ended) keeps the relay book-keeping + // clean. + if (priorCatalogPublisher != null) runCatching { priorCatalogPublisher.close() } } try { @@ -703,10 +677,10 @@ private class ReissuingBroadcastHandle( // // Catalog gets explicit teardown because — unlike the audio // publisher which broadcaster.stop() closes for us — the - // catalog publisher + republisher have no broadcaster wrapper. - // Cancel the republish loop FIRST so it can't race the close. - hotSwapCatalogJob?.let { runCatching { it.cancelAndJoin() } } - hotSwapCatalogJob = null + // catalog publisher has no broadcaster wrapper. The + // emit-on-subscribe hook itself doesn't need shutdown: it + // only fires inside [PublisherStateImpl.registerInboundSubscription], + // and closing the publisher prevents any further hook fires. hotSwapCatalogPublisher?.let { runCatching { it.close() } } hotSwapCatalogPublisher = null hotSwapBroadcaster?.let { runCatching { it.stop() } } diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt index bf85c60e4..961f9cfe9 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt @@ -1079,6 +1079,18 @@ class MoqLiteSession internal constructor( @Volatile private var publisherClosed = false + /** + * Caller-installed hook fired once per accepted inbound + * SUBSCRIBE — see [MoqLitePublisherHandle.setOnNewSubscriber]. + * Read-and-fire happens OUTSIDE [gate] to avoid deadlocking on + * the hook's own calls to [send] / [endGroup]. + */ + @Volatile private var onNewSubscriberHook: (suspend () -> Unit)? = null + + override fun setOnNewSubscriber(hook: (suspend () -> Unit)?) { + onNewSubscriberHook = hook + } + suspend fun registerAnnounceBidi( bidi: com.vitorpamplona.nestsclient.transport.WebTransportBidiStream, emittedSuffix: String, @@ -1093,6 +1105,11 @@ class MoqLiteSession internal constructor( } suspend fun registerInboundSubscription(sub: MoqLiteSubscribe) { + // Capture the hook INSIDE the lock — guarantees the hook + // observes a fully-registered subscriber when it fires — + // but invoke it OUTSIDE so a hook that calls [send] / + // [endGroup] doesn't deadlock on the same gate. + var hookToFire: (suspend () -> Unit)? = null gate.withLock { if (publisherClosed) { Log.w("NestTx") { "SUBSCRIBE inbound rejected (publisher closed) id=${sub.id} track='${sub.track}'" } @@ -1103,8 +1120,15 @@ class MoqLiteSession internal constructor( return } inboundSubs += sub + hookToFire = onNewSubscriberHook Log.d("NestTx") { "SUBSCRIBE registered id=${sub.id} broadcast='${sub.broadcast}' track='${sub.track}' inboundSubs.size=${inboundSubs.size}" } } + // Launch on the session's scope so the hook outlives the + // bidi pump's per-bidi coroutine if it's slow (e.g. a + // catalog send blocked on transport backpressure). + hookToFire?.let { hook -> + scope.launch { runCatching { hook.invoke() } } + } } /** @@ -1388,6 +1412,31 @@ interface MoqLitePublisherHandle { /** FIN the current group's uni stream. The next [send] starts a fresh group. */ suspend fun endGroup() + /** + * Register a callback that fires once each time a new inbound + * subscriber is registered against this publisher's track (i.e. + * each track-matching SUBSCRIBE bidi the relay opens to us). Used + * to push a "track-latest" payload — the canonical example is the + * broadcast catalog manifest, which a watcher needs to receive on + * subscribe but doesn't change between subscribers — without + * forcing the publisher to maintain a periodic re-emit loop. + * + * Called once per accepted SUBSCRIBE (track filter passed). Fires + * OUTSIDE the publisher's serialisation lock, so the hook can + * safely call [send] / [endGroup] without deadlocking. + * + * Caller MUST set the hook before any subscriber attaches (typically + * immediately after [com.vitorpamplona.nestsclient.moq.lite.MoqLiteSession.publish] + * returns) — there's no "fire-on-set for existing subscribers" + * replay. For the typical catalog use case the publisher is fresh + * when the hook is set, and the relay's SUBSCRIBE bidi takes a + * round-trip to arrive, so this is safe in practice. + * + * Pass `null` to clear the hook. Calling twice with non-null + * replaces the previous hook (no de-duplication). + */ + fun setOnNewSubscriber(hook: (suspend () -> Unit)?) + /** * Stop publishing. Sends `Announce(Ended)` on every active announce * bidi, FINs the current group, and releases all per-publisher diff --git a/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSessionTest.kt b/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSessionTest.kt index 70b1d5eb4..f823438ce 100644 --- a/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSessionTest.kt +++ b/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSessionTest.kt @@ -357,6 +357,123 @@ class MoqLiteSessionTest { session.close() } + @Test + fun publisher_setOnNewSubscriber_hook_fires_per_inbound_subscribe() = + runBlocking { + val (clientSide, serverSide) = FakeWebTransport.pair() + val session = MoqLiteSession.client(clientSide, pumpScope) + + val publisher = session.publish(broadcastSuffix = "speakerPubkey", track = "audio/data") + val hookFireCount = + java.util.concurrent.atomic + .AtomicInteger(0) + publisher.setOnNewSubscriber { + hookFireCount.incrementAndGet() + } + + // First inbound SUBSCRIBE → hook fires once. + val subBidi1 = serverSide.openBidiStream() + subBidi1.write(Varint.encode(MoqLiteControlType.Subscribe.code)) + subBidi1.write( + MoqLiteCodec.encodeSubscribe( + MoqLiteSubscribe( + id = 0L, + broadcast = "speakerPubkey", + track = "audio/data", + priority = 0x80, + ordered = true, + maxLatencyMillis = 0L, + startGroup = null, + endGroup = null, + ), + ), + ) + // Wait for SubscribeOk to drain so we know registerInboundSubscription + // ran (and therefore the hook had its chance to launch). + withTimeout(2_000) { subBidi1.incoming().first() } + // Hook fires asynchronously on the session scope; give it a + // moment to land. Use a bounded retry rather than a flat + // delay so the test is fast on the happy path. + withTimeout(2_000) { + while (hookFireCount.get() < 1) kotlinx.coroutines.yield() + } + assertEquals(1, hookFireCount.get()) + + // Second inbound SUBSCRIBE → hook fires again. + val subBidi2 = serverSide.openBidiStream() + subBidi2.write(Varint.encode(MoqLiteControlType.Subscribe.code)) + subBidi2.write( + MoqLiteCodec.encodeSubscribe( + MoqLiteSubscribe( + id = 1L, + broadcast = "speakerPubkey", + track = "audio/data", + priority = 0x80, + ordered = true, + maxLatencyMillis = 0L, + startGroup = null, + endGroup = null, + ), + ), + ) + withTimeout(2_000) { subBidi2.incoming().first() } + withTimeout(2_000) { + while (hookFireCount.get() < 2) kotlinx.coroutines.yield() + } + assertEquals(2, hookFireCount.get()) + + publisher.close() + session.close() + } + + @Test + fun publisher_setOnNewSubscriber_hook_does_not_fire_on_track_mismatch() = + runBlocking { + val (clientSide, serverSide) = FakeWebTransport.pair() + val session = MoqLiteSession.client(clientSide, pumpScope) + + // Publisher serves audio/data only. + val publisher = session.publish(broadcastSuffix = "speakerPubkey", track = "audio/data") + val hookFireCount = + java.util.concurrent.atomic + .AtomicInteger(0) + publisher.setOnNewSubscriber { + hookFireCount.incrementAndGet() + } + + // Inbound SUBSCRIBE for a different track. Replies + // SubscribeDrop (covered in another test); hook MUST NOT + // fire because no subscriber was actually registered on + // this publisher. + val subBidi = serverSide.openBidiStream() + subBidi.write(Varint.encode(MoqLiteControlType.Subscribe.code)) + subBidi.write( + MoqLiteCodec.encodeSubscribe( + MoqLiteSubscribe( + id = 7L, + broadcast = "speakerPubkey", + track = "video/data", + priority = 0x80, + ordered = true, + maxLatencyMillis = 0L, + startGroup = null, + endGroup = null, + ), + ), + ) + // Drain the Drop reply. + withTimeout(2_000) { subBidi.incoming().first() } + // No way to wait deterministically for "the hook didn't + // fire"; sleep briefly to let any racing launch surface, + // then assert. Short delay because the hook would launch + // on the same scope as registerInboundSubscription's caller. + kotlinx.coroutines.delay(100) + assertEquals(0, hookFireCount.get()) + + publisher.close() + session.close() + } + @Test fun publisher_replies_subscribeDrop_when_track_is_not_published() = runBlocking { From 87b43d4d78340d2f006ab49c75de13d301137df8 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 16:43:43 +0000 Subject: [PATCH 24/41] fix(nests): bump preroll to 200 ms + surface persistent decode failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two audit follow-ups for receive-path interop with kixelated/moq web publishers (NestsUI v2's reference encoder): 1. ROOM_PLAYER_PREROLL_FRAMES 5 → 10 (≈100 ms → ≈200 ms). The web reference encoder uses `groupDuration: 100ms` (5 Opus frames per group) and stacks another 100–200 ms of jitter on top via the relay's outbound forward queue under residential network conditions. 100 ms of preroll reliably underran on web speakers, producing audible clicks between adjacent groups; 200 ms covers the observed envelope while keeping perceived join latency well below the wire latency a listener already sees. 2. Per-subscription consecutive Opus decode-error counter in NestViewModel.openSubscription. Single-frame decode errors are normal noise — Opus PLC papers over a single bad frame and the prior code silently swallowed them — but a structural mismatch (wrong wire format, missing legacy-container varint strip, codec/channel-count mismatch, corrupted Opus) fails *every* frame in a row. The previous behaviour made exactly that class of bug invisible: no audio, no log, no UI signal. New logic increments a per-subscription AtomicLong on each DecoderError / EncoderError, resets on every successful onLevel tick. When the streak hits DECODE_ERROR_STREAK_LOG_THRESHOLD (50 frames ≈ 1 s of solid failures) we log ONE conspicuous warning tagged `NestRx` with the underlying throwable's stacktrace. Logged once per crossing (exact-equality, not modulus) so a stuck stream produces a single attention-grabbing line instead of a 50 Hz flood. Picks the non-lambda Log.w overload deliberately so the throwable stacktrace survives — fires exactly once per stuck-stream streak, so the unconditional message-string build is fine. Both changes are additive — no callers changed, no public API touched. The stale `ROOM_PLAYER_PREROLL_FRAMES = 5` reference in NestMoqLiteBroadcaster's framesPerGroup kdoc is updated to match. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../commons/viewmodels/NestViewModel.kt | 84 +++++++++++++++++-- .../audio/NestMoqLiteBroadcaster.kt | 2 +- 2 files changed, 78 insertions(+), 8 deletions(-) diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt index 9bc9255ba..b2e7cde15 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt @@ -45,6 +45,7 @@ import com.vitorpamplona.nestsclient.moq.SubscribeHandle import com.vitorpamplona.nestsclient.transport.WebTransportFactory import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner import com.vitorpamplona.quartz.nip53LiveActivities.chat.LiveActivitiesChatMessageEvent +import com.vitorpamplona.quartz.utils.Log import kotlinx.collections.immutable.ImmutableSet import kotlinx.collections.immutable.persistentSetOf import kotlinx.collections.immutable.toPersistentSet @@ -1142,6 +1143,19 @@ class NestViewModel( // Tap the object flow to drive the speaking-now indicator before // the decoder consumes it. val instrumented = handle.objects.onEach { onSpeakerActivity(pubkey) } + // Per-subscription consecutive-decoder-error counter. Single-frame + // Opus errors are normal noise (PLC handles them), but a structural + // mismatch — wrong wire format, missing legacy-container varint + // strip, codec mismatch — fails *every* frame in a row. The + // previous behaviour was to silently swallow per-packet + // [AudioException.Kind.DecoderError] indefinitely, which made + // exactly that class of bug invisible (no audio, no log, no UI + // signal). Track the streak and surface it once per crossing so + // the next protocol regression shows up in `adb logcat | grep + // NestRx` rather than in a user bug report. + val consecutiveDecodeErrors = + java.util.concurrent.atomic + .AtomicLong(0L) roomPlayer.play( instrumented, onError = { err -> @@ -1158,7 +1172,28 @@ class NestViewModel( AudioException.Kind.DecoderError, AudioException.Kind.EncoderError, -> { - Unit + val streak = consecutiveDecodeErrors.incrementAndGet() + // 50 frames = 1 s of solid decode failures. + // Anything past that is a structural bug, not + // jitter. Log once per crossing (use exact + // equality, not modulus) so we get a single + // attention-grabbing line per stuck stream + // rather than a 50 Hz flood. + if (streak == DECODE_ERROR_STREAK_LOG_THRESHOLD) { + // Non-lambda overload here (rather than the + // allocation-free lambda one) so we preserve + // the throwable stacktrace — this fires + // exactly once per stuck-stream streak, so + // the unconditional string build is fine. + Log.w( + "NestRx", + "decoder failed $DECODE_ERROR_STREAK_LOG_THRESHOLD consecutive frames " + + "for pubkey='${pubkey.take(8)}' — likely wire-format mismatch " + + "(missing legacy-container varint strip, codec/channel-count mismatch, " + + "or corrupted Opus). last error: ${err.message}", + err.cause, + ) + } } AudioException.Kind.PlaybackFailed, @@ -1170,7 +1205,13 @@ class NestViewModel( } } }, - onLevel = { onAudioLevel(pubkey, it) }, + onLevel = { lvl -> + // A successful decode resets the error streak. + // Sample-rate-fast (50 Hz) but cheap on the + // success path: Atomic.set with no allocation. + consecutiveDecodeErrors.set(0L) + onAudioLevel(pubkey, lvl) + }, ) slot.attach(handle, roomPlayer, player) publishActiveSpeakers() @@ -1677,16 +1718,45 @@ sealed class BroadcastUiState { */ const val SPEAKING_TIMEOUT_MS: Long = 250L +/** + * Per-subscription consecutive Opus decode-error count that triggers a + * single conspicuous warning log. Single-frame decode errors are normal + * — Opus PLC papers over them — and are silently swallowed; a streak of + * this size is structural (wire-format mismatch, missing + * legacy-container varint strip, codec/channel-count mismatch, corrupt + * publisher) and demands attention because the user-visible symptom is + * "speaker tile shows up but no audio plays" with no other signal. + * + * 50 × 20 ms ≈ 1 s. Long enough that a transient flurry of bad frames + * doesn't cry wolf; short enough that an "every frame fails" structural + * bug surfaces in a logcat line within the first second of audio. + * + * Logged once per crossing (exact equality, not modulus) so a stuck + * stream produces ONE attention-grabbing line rather than 50 Hz of + * noise. + */ +const val DECODE_ERROR_STREAK_LOG_THRESHOLD: Long = 50L + /** * Per-speaker pre-roll: number of decoded PCM frames buffered before the * underlying [com.vitorpamplona.nestsclient.audio.AudioPlayer] starts - * consuming. 5 × 20 ms ≈ 100 ms of audio — long enough to mask a typical - * Main-thread stall (Compose recomposition / GC) without adding perceptible - * join latency. Combines with [com.vitorpamplona.nestsclient.audio.AudioTrackPlayer]'s - * ~250 ms AudioTrack buffer for ~350 ms of total slack between the + * consuming. 10 × 20 ms ≈ 200 ms of audio. + * + * Sized to absorb the slowest-cadence publisher we expect: kixelated/moq's + * web reference encoder uses `groupDuration: 100ms` (5 frames per group) + * by default but adds another 100–200 ms of jitter on the relay's + * outbound forward queue under residential network conditions. 100 ms of + * preroll (the previous value) reliably underran on web speakers — the + * AudioTrack ran out of samples between adjacent groups and the user + * heard short clicks. 200 ms covers the observed jitter envelope while + * keeping perceived join latency well under the typical "speaker becomes + * audible" delay on the wire. + * + * Combines with [com.vitorpamplona.nestsclient.audio.AudioTrackPlayer]'s + * ~250 ms AudioTrack buffer for ~450 ms of total slack between the * arrival-of-frame and the underrun horizon. */ -const val ROOM_PLAYER_PREROLL_FRAMES: Int = 5 +const val ROOM_PLAYER_PREROLL_FRAMES: Int = 10 /** * Coalescing interval for [NestViewModel.audioLevels]. The decode loop diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt index fadda7a28..c3e190517 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt @@ -460,7 +460,7 @@ class NestMoqLiteBroadcaster( * joining mid-broadcast has to wait until the next * group boundary for the first frame. 50 frames = 1 s * gap, which matches what the existing - * `ROOM_PLAYER_PREROLL_FRAMES = 5` audio-buffer + the + * `ROOM_PLAYER_PREROLL_FRAMES = 10` audio-buffer + the * ~250 ms AudioTrack jitter buffer was already designed * to mask. 100 frames (2 s) is past that. * - QUIC stream-level reliability: each group is one uni From 1c47fd2403f95bde547b98e1ee209c8c826edbce Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 17:26:47 +0000 Subject: [PATCH 25/41] feat(nests): drive Opus decoder + AudioTrack channel count from catalog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Web speakers (kixelated/hang reference) emit stereo Opus when the user's AudioContext picks a stereo input — and our catalog now lets us know that ahead of decoder construction. Previously the decoder + AudioTrack were both hardcoded to mono via AudioFormat.CHANNELS, so a stereo web publisher's frames either threw on the Opus TOC byte mismatch, decoded with downmix artifacts, or output left-channel-only depending on the device's MediaCodec implementation. Wire the catalog-discovered numberOfChannels through: - MediaCodecOpusDecoder accepts `channelCount: Int = 1`. Drives MediaFormat.createAudioFormat's channel count, the OpusHead CSD-0 channel byte, and the per-frame output buffer size (stereo frame = 2× shorts vs mono). Validates 1..2 — RFC 7845 mapping family 0 covers both mono and stereo without an explicit channel mapping table; multichannel needs family 1 which we don't support. - AudioTrackPlayer accepts `channelCount: Int = 1`. Drives the AudioTrack channel mask (CHANNEL_OUT_MONO vs CHANNEL_OUT_STEREO) and the 250 ms wall-clock buffer-size target (stereo doubles bytes per sample). - NestViewModel.openSubscription now starts the catalog fetch BEFORE waiting for it, then awaits `_speakerCatalogs[pubkey]` for CATALOG_AWAIT_TIMEOUT_MS (500 ms) before constructing the decoder + player. With the speaker-side emit-on-subscribe hook in place (3417e44), the catalog frame is on the wire within one round-trip of the SUBSCRIBE_OK, so the wait typically resolves in tens of ms. Falls back to mono if the catalog never arrives — covers legacy publishers that don't emit a catalog and the failure-mode where the relay never forwards the catalog group. - decoderFactory + playerFactory signatures change from `() -> X` to `(channelCount: Int) -> X`. Two call sites updated: NestViewModelFactory (production) and NestViewModelTest's newViewModel helper. Speaker side stays mono-only — our encoder is fixed at AudioFormat.CHANNELS=1 and the catalog we publish declares numberOfChannels=1. This change is exclusively about handling incoming audio from publishers that don't share that constraint. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../room/lifecycle/NestViewModelFactory.kt | 4 +- .../commons/viewmodels/NestViewModel.kt | 108 ++++++++++++++++-- .../commons/viewmodels/NestViewModelTest.kt | 4 +- .../nestsclient/audio/AudioTrackPlayer.kt | 25 +++- .../audio/MediaCodecOpusDecoder.kt | 51 +++++++-- 5 files changed, 161 insertions(+), 31 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/nests/room/lifecycle/NestViewModelFactory.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/nests/room/lifecycle/NestViewModelFactory.kt index e5d25b475..69c30fe27 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/nests/room/lifecycle/NestViewModelFactory.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/nests/room/lifecycle/NestViewModelFactory.kt @@ -53,8 +53,8 @@ internal class NestViewModelFactory( // `httpClient` slot rather than as the first positional arg. httpClient = OkHttpNestsClient(httpClient = Amethyst.instance.roleBasedHttpClientBuilder::okHttpClientForVideo), transport = QuicWebTransportFactory(), - decoderFactory = { MediaCodecOpusDecoder() }, - playerFactory = { AudioTrackPlayer() }, + decoderFactory = { channelCount -> MediaCodecOpusDecoder(channelCount = channelCount) }, + playerFactory = { channelCount -> AudioTrackPlayer(channelCount = channelCount) }, signer = signer, room = room, captureFactory = { AudioRecordCapture() }, diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt index b2e7cde15..cb6b85209 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt @@ -35,6 +35,7 @@ import com.vitorpamplona.nestsclient.NestsSpeaker import com.vitorpamplona.nestsclient.NestsSpeakerState import com.vitorpamplona.nestsclient.audio.AudioCapture import com.vitorpamplona.nestsclient.audio.AudioException +import com.vitorpamplona.nestsclient.audio.AudioFormat import com.vitorpamplona.nestsclient.audio.AudioPlayer import com.vitorpamplona.nestsclient.audio.NestPlayer import com.vitorpamplona.nestsclient.audio.OpusDecoder @@ -56,9 +57,13 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.filterNotNull +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull import kotlin.coroutines.cancellation.CancellationException /** @@ -81,7 +86,11 @@ import kotlin.coroutines.cancellation.CancellationException * Audio-pipeline construction is injected via [decoderFactory] / * [playerFactory] so commonMain doesn't have to know which platform's * MediaCodec / AudioTrack is in play. M1 wires Android-only — desktop - * passes nothing here yet. + * passes nothing here yet. Both factories take a `channelCount` (1 for + * mono, 2 for stereo) discovered from the publisher's `catalog.json` + * audio rendition; subscriptions await a brief catalog-arrival window + * before constructing the decoder + player so a stereo web publisher + * doesn't get its frames decoded as mono with downmix artifacts. * * **Threading contract:** all public methods (`connect`, `disconnect`, * `updateSpeakers`, `setMuted`, `setMicMuted`, `startBroadcast`, @@ -98,8 +107,8 @@ import kotlin.coroutines.cancellation.CancellationException class NestViewModel( private val httpClient: NestsClient, private val transport: WebTransportFactory, - private val decoderFactory: () -> OpusDecoder, - private val playerFactory: () -> AudioPlayer, + private val decoderFactory: (channelCount: Int) -> OpusDecoder, + private val playerFactory: (channelCount: Int) -> AudioPlayer, private val signer: NostrSigner, private val room: NestsRoomConfig, // Speaker-side audio capture/encode actuals. Optional — desktop and @@ -1038,6 +1047,55 @@ class NestViewModel( } } + /** + * Wait briefly for [pubkey]'s catalog to land in [_speakerCatalogs] + * and pick the channel count for the decoder + AudioTrack. Returns + * [AudioFormat.CHANNELS] on timeout (the catalog never arrived + * within [timeoutMs]) or when the catalog declares an unsupported + * count (anything outside `1..2`). Caller is responsible for + * having started [fetchSpeakerCatalog] first; otherwise this + * always times out. + * + * Why a timeout: the catalog handshake is best-effort. If the + * publisher doesn't publish a catalog (legacy publishers) or the + * relay never forwards the catalog group, audio playback should + * still proceed in the default config rather than block forever. + * 500 ms is generous — with the speaker-side + * `setOnNewSubscriber` hook the catalog group is on the wire + * within one round-trip of the SUBSCRIBE_OK, so this typically + * returns within tens of ms. + */ + private suspend fun awaitDecoderChannelCount( + pubkey: String, + timeoutMs: Long, + ): Int { + val catalog = + withTimeoutOrNull(timeoutMs) { + _speakerCatalogs + .map { it[pubkey] } + .filterNotNull() + .first() + } + val declared = catalog?.primaryAudio()?.numberOfChannels + return when { + declared == null -> { + AudioFormat.CHANNELS + } + + declared !in 1..2 -> { + Log.w("NestRx") { + "publisher catalog for pubkey='${pubkey.take(8)}' declares numberOfChannels=$declared " + + "(only 1 / 2 supported); falling back to mono" + } + AudioFormat.CHANNELS + } + + else -> { + declared + } + } + } + /** * Open the speaker's `catalog.json` track in the background, parse * the first frame, and stash it in [speakerCatalogs]. Best-effort — @@ -1100,15 +1158,29 @@ class NestViewModel( viewModelScope.launch { runCatching { handle.unsubscribe() } } return } + // Start the catalog fetch BEFORE we wait for it — runs in + // parallel with the main subscribe so the round-trip overlaps + // with the audio-side handshake. Cancelled by closeSubscription. + fetchSpeakerCatalog(l, pubkey) + // Wait briefly for the catalog so we can configure the + // decoder + AudioTrack to match the publisher's actual + // channel layout. With the speaker-side emit-on-subscribe + // hook the catalog frame is on the wire within one round-trip + // of the SUBSCRIBE_OK, so this typically returns within tens + // of ms. Falls back to mono if no catalog arrives within + // [CATALOG_AWAIT_TIMEOUT_MS] — covers legacy publishers and + // the failure-mode where the relay never forwards the catalog + // group; audio still plays, just in the default config. + val channelCount = awaitDecoderChannelCount(pubkey, CATALOG_AWAIT_TIMEOUT_MS) // Allocate native resources (MediaCodec decoder + AudioTrack // player on Android). Both are heavy and leaky if dropped on // the floor — wrap them in a nested try so any cancellation // or throw between here and slot.attach releases them // (audit round-2 VM #7). - val decoder = decoderFactory() + val decoder = decoderFactory(channelCount) val player = try { - playerFactory() + playerFactory(channelCount) } catch (t: Throwable) { runCatching { decoder.release() } throw t @@ -1221,12 +1293,10 @@ class NestViewModel( _uiState.update { it.copy(connectingSpeakers = (it.connectingSpeakers + pubkey).toPersistentSet()) } - // Parallel catalog fetch — best-effort, doesn't gate - // audio playback. Tracked in catalogJobs; cancelled - // by closeSubscription so a removed speaker doesn't - // leave the catalog collector running on the - // wrapper's still-live re-issuing handle. - fetchSpeakerCatalog(l, pubkey) + // Catalog fetch was started earlier (before the decoder + // wait) so it could overlap with the audio-side handshake; + // its job is already in [catalogJobs] and gets cancelled + // by [closeSubscription] alongside the audio path. } catch (t: Throwable) { // Either CancellationException (scope cancelled mid-construction) // or an unexpected throw — release the half-built pipeline and @@ -1718,6 +1788,22 @@ sealed class BroadcastUiState { */ const val SPEAKING_TIMEOUT_MS: Long = 250L +/** + * How long [NestViewModel.openSubscription] waits for the publisher's + * `catalog.json` to land before constructing the decoder + AudioTrack. + * The catalog declares the audio rendition's `numberOfChannels`, which + * the decoder needs at construction time so a stereo web publisher + * doesn't get its frames decoded as mono with downmix artifacts. + * + * Sized to be generous enough that a typical publisher's catalog + * (which arrives within a single round-trip of the SUBSCRIBE_OK with + * the speaker-side emit-on-subscribe hook in place) lands well before + * the timeout, and short enough that a publisher that never emits a + * catalog (legacy publishers, relay-side bug) doesn't visibly stall + * the listener — audio playback proceeds in the default mono config. + */ +const val CATALOG_AWAIT_TIMEOUT_MS: Long = 500L + /** * Per-subscription consecutive Opus decode-error count that triggers a * single conspicuous warning log. Single-frame decode errors are normal diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModelTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModelTest.kt index 2cddb7a0d..d9c5d7d76 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModelTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModelTest.kt @@ -455,8 +455,8 @@ class NestViewModelTest { NestViewModel( httpClient = NoopNestsClient, transport = NoopWebTransportFactory, - decoderFactory = { NoopOpusDecoder }, - playerFactory = { NoopAudioPlayer() }, + decoderFactory = { _ -> NoopOpusDecoder }, + playerFactory = { _ -> NoopAudioPlayer() }, signer = NoopSigner, room = ROOM_CONFIG, connector = diff --git a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/AudioTrackPlayer.kt b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/AudioTrackPlayer.kt index dfd2eaa6e..0d6901e04 100644 --- a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/AudioTrackPlayer.kt +++ b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/AudioTrackPlayer.kt @@ -77,7 +77,22 @@ import android.media.AudioFormat as AndroidAudioFormat class AudioTrackPlayer( private val usage: Int = AudioAttributes.USAGE_MEDIA, private val contentType: Int = AudioAttributes.CONTENT_TYPE_SPEECH, + /** + * Number of channels in the PCM stream this player will receive. Must + * match the [com.vitorpamplona.nestsclient.audio.OpusDecoder]'s output + * configuration (mono Opus → 1, stereo Opus → 2 with L/R interleaving). + * Drives both the AudioTrack channel mask and the underlying buffer- + * size target. Default is [AudioFormat.CHANNELS] (mono) so existing + * call sites that don't pass a channel count keep the prior behaviour. + */ + private val channelCount: Int = AudioFormat.CHANNELS, ) : AudioPlayer { + init { + require(channelCount in 1..2) { + "AudioTrackPlayer supports mono (1) or stereo (2) only, got $channelCount" + } + } + private var track: AudioTrack? = null private var muted: Boolean = false private var volume: Float = 1f @@ -100,10 +115,10 @@ class AudioTrackPlayer( .d("NestPlay") { "AudioTrackPlayer.start() — allocating AudioTrack" } val channelMask = - when (AudioFormat.CHANNELS) { + when (channelCount) { 1 -> AndroidAudioFormat.CHANNEL_OUT_MONO 2 -> AndroidAudioFormat.CHANNEL_OUT_STEREO - else -> error("unsupported channel count ${AudioFormat.CHANNELS}") + else -> error("unsupported channel count $channelCount") } val minBuffer = @@ -122,9 +137,11 @@ class AudioTrackPlayer( // miss its 20 ms cadence by an order of magnitude before the device // underruns. Take the larger of `minBuffer * 16` and an explicit // 250 ms-equivalent so devices that report a small minBuffer still - // get the same wall-clock slack. + // get the same wall-clock slack. Stereo doubles the byte count + // per sample (interleaved L,R 16-bit shorts) — scaling + // [channelCount] in keeps the wall-clock target constant. val targetBytes250Ms = - (AudioFormat.SAMPLE_RATE_HZ / 4) * AudioFormat.BYTES_PER_SAMPLE * AudioFormat.CHANNELS + (AudioFormat.SAMPLE_RATE_HZ / 4) * AudioFormat.BYTES_PER_SAMPLE * channelCount val bufferBytes = maxOf(minBuffer * 16, targetBytes250Ms) val newTrack = diff --git a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusDecoder.kt b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusDecoder.kt index c656a5d07..b2d93e7b7 100644 --- a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusDecoder.kt +++ b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusDecoder.kt @@ -31,21 +31,42 @@ import java.nio.ByteOrder * across packets, so sharing a decoder across speakers would cause clicks. * * Configuration: - * - 48 kHz mono signed 16-bit PCM output (matches [AudioFormat]). - * - CSD-0: Opus identification header per RFC 7845 §5.1, 19 bytes. + * - 48 kHz signed 16-bit PCM output (Opus's internal sample rate; + * MediaCodec's Opus decoder always emits 48 kHz regardless of the + * OpusHead `inputSampleRate` field). + * - [channelCount] — 1 (mono) or 2 (stereo) interleaved L/R. Drives both + * the MediaFormat channel-count and the OpusHead CSD-0 channel byte. + * A web publisher (kixelated/hang) emits stereo when the user's + * AudioContext picks a stereo input; without matching channel + * configuration the decoder either downmixes with artifacts or + * refuses the packet. + * - CSD-0: Opus identification header per RFC 7845 §5.1, 19 bytes + * (mapping family 0; covers both mono and stereo with implicit + * L,R interleaving). * - CSD-1 / CSD-2: pre-skip + seek pre-roll, both zero (we don't seek). + * + * Default is [AudioFormat.CHANNELS] (mono) so existing call sites that + * don't pass a channel count continue to behave exactly as before. */ -class MediaCodecOpusDecoder : OpusDecoder { +class MediaCodecOpusDecoder( + private val channelCount: Int = AudioFormat.CHANNELS, +) : OpusDecoder { + init { + require(channelCount in 1..2) { + "MediaCodecOpusDecoder supports mono (1) or stereo (2) only, got $channelCount" + } + } + private val codec: MediaCodec = try { MediaCodec .createDecoderByType(MediaFormat.MIMETYPE_AUDIO_OPUS) .apply { - configure(buildFormat(), null, null, 0) + configure(buildFormat(channelCount), null, null, 0) start() }.also { com.vitorpamplona.quartz.utils.Log.d("NestPlay") { - "MediaCodecOpusDecoder allocated codec='${it.name}'" + "MediaCodecOpusDecoder allocated codec='${it.name}' channelCount=$channelCount" } } } catch (t: Throwable) { @@ -70,7 +91,11 @@ class MediaCodecOpusDecoder : OpusDecoder { // via ShortBuffer.get(dst, off, len) — the previous shape went // through ArrayList, which boxed every PCM sample // (~48 000 alloc/sec/speaker on the audio hot path). - val out = ShortArray(AudioFormat.FRAME_SIZE_SAMPLES) + // + // Stereo Opus emits L/R-interleaved samples, so a 20 ms / 960- + // sample frame produces `FRAME_SIZE_SAMPLES * channelCount` + // shorts — twice as many for stereo as for mono. + val out = ShortArray(AudioFormat.FRAME_SIZE_SAMPLES * channelCount) var outPos = 0 // 1. Acquire an input slot. On rare back-pressure (output buffers @@ -196,14 +221,14 @@ class MediaCodecOpusDecoder : OpusDecoder { private const val FRAME_DURATION_US = 20_000L // 20 ms - private fun buildFormat(): MediaFormat { + private fun buildFormat(channelCount: Int): MediaFormat { val format = MediaFormat.createAudioFormat( MediaFormat.MIMETYPE_AUDIO_OPUS, AudioFormat.SAMPLE_RATE_HZ, - AudioFormat.CHANNELS, + channelCount, ) - format.setByteBuffer("csd-0", ByteBuffer.wrap(buildOpusIdHeader())) + format.setByteBuffer("csd-0", ByteBuffer.wrap(buildOpusIdHeader(channelCount))) // Pre-skip + seek pre-roll: both zero, encoded as little-endian // 64-bit nanoseconds per Android's MediaCodec contract. format.setByteBuffer("csd-1", ByteBuffer.wrap(zeroLongLe())) @@ -211,12 +236,14 @@ class MediaCodecOpusDecoder : OpusDecoder { return format } - private fun buildOpusIdHeader(): ByteArray { - // RFC 7845 §5.1 — 19 bytes for mono, mapping family 0. + private fun buildOpusIdHeader(channelCount: Int): ByteArray { + // RFC 7845 §5.1 — 19 bytes, mapping family 0 (covers mono and + // stereo with implicit L,R interleaving; no per-channel + // mapping table required). val buf = ByteBuffer.allocate(19).order(ByteOrder.LITTLE_ENDIAN) buf.put("OpusHead".encodeToByteArray()) // 8 bytes magic buf.put(1.toByte()) // version - buf.put(AudioFormat.CHANNELS.toByte()) // channel count + buf.put(channelCount.toByte()) // channel count (1 or 2) buf.putShort(0) // pre-skip buf.putInt(AudioFormat.SAMPLE_RATE_HZ) // input sample rate buf.putShort(0) // output gain (Q7.8 dB) From ea90685a864ae46480d6eaa5501979c146964701 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 17:37:02 +0000 Subject: [PATCH 26/41] revert(nests): drop moq-lite-04 ALPN advertisement (codec is wire-incompatible) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-investigation of `kixelated/moq` commit 45db108 ("moq-lite/moq-relay: hop-based clustering") revealed that Lite-04 IS wire-incompatible with our Lite-03 codec on Announce / AnnounceInterest / Probe — contradicting the earlier reasoning that motivated 4b73626. Specifically: - Announce.hops: Lite-03 writes a single varint count of hops; Lite-04 writes count + count × varint Origin ids. (`rs/moq-lite/src/lite/ announce.rs:67-73` branches on Version explicitly.) - AnnounceInterest: Lite-04 added an `exclude_hop` varint after the prefix. - Probe: Lite-04 added an `rtt` varint after `bitrate`. Subscribe / SubscribeOk / SubscribeDrop / Group / GroupHeader framing is unchanged across Lite-03↔Lite-04 — but the Announce/Probe drift alone is enough to desync a Lite-04-preferring relay if it picks `moq-lite-04` from our advertised list. We'd encode Announce hops as a bare varint where the peer expects `len + len × u62`, and the connection aborts on the first Announce exchange. Pin `wt-available-protocols` back to `["moq-lite-03"]` until `MoqLiteCodec` gains version-aware Announce / AnnounceInterest / Probe codecs and `MoqLiteAnnounce.hops` becomes a list rather than a single varint. Reverting 4b73626 — the LITE_04 constant stays in MoqLiteAlpn for documentation, but is no longer plumbed into the factory's sub-protocol list. Reverts the wire-format expansion in 4b73626. The factory and ALPN kdocs are rewritten to spell out the codec diff so the next person who considers re-enabling Lite-04 reads the actual blocker first. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../nestsclient/moq/lite/MoqLiteMessages.kt | 32 ++++++++++++++----- .../transport/QuicWebTransportFactory.kt | 28 ++++++++++------ 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteMessages.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteMessages.kt index 59d8af06d..b55f2e0ee 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteMessages.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteMessages.kt @@ -21,22 +21,38 @@ package com.vitorpamplona.nestsclient.moq.lite /** - * moq-lite ALPN strings. The on-the-wire framing for Subscribe / Group - * / Announce did NOT change between Lite-03 and Lite-04, so advertising - * both via `wt-available-protocols` is safe — the relay picks whichever - * it prefers and our existing codec paths handle either. We keep 03 in - * the list as the previously-tested target until the production relay's - * Lite-04 path has interop coverage. + * moq-lite ALPN strings. ONLY [LITE_03] is wire-compatible with the + * codec in [MoqLiteCodec]; the other constants are kept for + * documentation and future codec upgrades. + * + * Subscribe / Group framing is identical across Lite-03 and Lite-04, + * but Lite-04 reshapes Announce.hops into an `OriginList` + * (`kixelated/moq` commit 45db108, "moq-lite/moq-relay: hop-based + * clustering"), adds an `exclude_hop` field to AnnounceInterest, and + * adds an `rtt` field to Probe. A relay that picks Lite-04 with our + * Lite-03 codec on the wire desyncs on the first Announce exchange. + * Don't add [LITE_04] to `wt-available-protocols` until + * [MoqLiteCodec] is version-aware and [MoqLiteAnnounce.hops] is a + * list rather than a single varint. * * `"moql"` is the legacy combined ALPN that requires an in-band SETUP * exchange — kept here for completeness; not advertised by the * factory. * - * Source: `kixelated/moq-rs/rs/moq-lite/src/version.rs:21-26`, - * `@moq/lite/connection/connect.js:277`. + * Source: `kixelated/moq/rs/moq-lite/src/lite/version.rs`, + * `kixelated/moq/rs/moq-lite/src/lite/announce.rs:11-105`, + * `kixelated/moq/rs/moq-lite/src/lite/probe.rs:9-55`. */ object MoqLiteAlpn { const val LITE_03: String = "moq-lite-03" + + /** + * `moq-lite-04` ALPN string. Wire-incompatible with [MoqLiteCodec] + * today — see the object kdoc for the codec diff. Defined here so + * a future patch that lands version-aware Announce / Probe codecs + * can drop it into the [QuicWebTransportFactory] sub-protocol list + * without re-deriving the constant. + */ const val LITE_04: String = "moq-lite-04" const val LEGACY: String = "moql" } diff --git a/nestsClient/src/jvmAndroid/kotlin/com/vitorpamplona/nestsclient/transport/QuicWebTransportFactory.kt b/nestsClient/src/jvmAndroid/kotlin/com/vitorpamplona/nestsclient/transport/QuicWebTransportFactory.kt index f305fc24c..4f5fe7e59 100644 --- a/nestsClient/src/jvmAndroid/kotlin/com/vitorpamplona/nestsclient/transport/QuicWebTransportFactory.kt +++ b/nestsClient/src/jvmAndroid/kotlin/com/vitorpamplona/nestsclient/transport/QuicWebTransportFactory.kt @@ -91,18 +91,26 @@ class QuicWebTransportFactory( * `connection closed err=invalid value` on the relay side and a stalled * subscribe / `subscribe stream FIN before reply` on the client side. * - * `moq-lite-04` is also advertised because the production relay - * (moq.nostrnests.com) and the kixelated browser client both ship - * Lite-04 now; without it in our list a Lite-04-only deployment - * would refuse the CONNECT. The on-the-wire framing for Subscribe / - * Group / Announce did NOT change between Lite-03 and Lite-04, so - * our existing codec handles either selection. List `moq-lite-03` - * first because it's our previously-tested negotiation target — - * a relay that supports both should pick the listed-first option, - * and a Lite-04-only relay falls through to the second entry. + * **Lite-04 is intentionally NOT advertised** even though the kixelated + * browser client now ships it. Lite-04 reshapes the on-the-wire + * Announce.hops field from a single varint count into a full + * `OriginList` (`kixelated/moq` commit 45db108, "moq-lite/moq-relay: + * hop-based clustering"), adds an `exclude_hop` field to + * AnnounceInterest, and adds an `rtt` field to Probe — Subscribe and + * Group framing are unchanged but Announce framing diverges. Our codec + * speaks pure Lite-03; if a Lite-04-preferring relay picks + * `moq-lite-04` from our advertised list, the very first Announce + * exchange would desync (we'd encode/read a bare hops varint where + * the peer expects `len + len × u62`) and the connection would abort. + * Until [com.vitorpamplona.nestsclient.moq.lite.MoqLiteCodec] gains + * version-aware Announce / AnnounceInterest / Probe paths and + * [com.vitorpamplona.nestsclient.moq.lite.MoqLiteAnnounce.hops] + * becomes a list, advertising Lite-04 is a footgun. See + * [com.vitorpamplona.nestsclient.moq.lite.MoqLiteAlpn] for the + * known-version constants. */ private val webTransportSubProtocols: List = - listOf(MoqLiteAlpn.LITE_03, MoqLiteAlpn.LITE_04), + listOf(MoqLiteAlpn.LITE_03), ) : WebTransportFactory { override suspend fun connect( authority: String, From 73722d2ad2c9f2351c5cea9bb2d6acc7ea686b46 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 17:37:19 +0000 Subject: [PATCH 27/41] fix(nests): T14 recognise Goaway control type instead of silent FIN MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit moq-rs's `Publisher::run` accepts `ControlType::Goaway = 5` as a graceful-shutdown signal that asks the publisher to migrate to a different relay node (`rs/moq-lite/src/lite/publisher.rs`). Our enum only defined Session/Announce/Subscribe/Fetch/Probe; an inbound Goaway bidi fell through `MoqLiteControlType.fromCode` as `null`, hit the unknown-control branch in `handleInboundBidi`, and was FIN'd silently — losing the relay's shutdown notification entirely. Add `Goaway(5L)` to the enum and a dedicated arm in `handleInboundBidi` that logs the event and FINs cleanly. We don't act on the migration request today (no body decode, no preferred-relay failover); the `connectReconnecting*` wrappers' transport-loss reconnect path already recovers from the eventual hard disconnect, so all this arm needs is to surface the relay's intent in logcat instead of swallowing it. Wire body decoding is left as a follow-up. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../nestsclient/moq/lite/MoqLiteMessages.kt | 11 +++++++++++ .../nestsclient/moq/lite/MoqLiteSession.kt | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteMessages.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteMessages.kt index b55f2e0ee..66e75df97 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteMessages.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteMessages.kt @@ -73,6 +73,17 @@ enum class MoqLiteControlType( Subscribe(2L), Fetch(3L), Probe(4L), + + /** + * Graceful relay-shutdown signal. moq-rs's `Publisher::run` accepts + * `ControlType::Goaway = 5` (`rs/moq-lite/src/lite/publisher.rs`) + * to migrate a publisher to a different relay node. We don't act + * on it today — recognising the type code prevents + * [MoqLiteSession.handleInboundBidi] from silently FINing the + * bidi as an unknown control type, which would lose the relay's + * shutdown notification. Wire body decoding is left as a follow-up. + */ + Goaway(5L), ; companion object { diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt index 961f9cfe9..4b7f3cbe1 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt @@ -953,6 +953,25 @@ class MoqLiteSession internal constructor( runCatching { bidi.finish() } dispatched = true } + + MoqLiteControlType.Goaway -> { + // Relay's graceful-shutdown signal — see + // [MoqLiteControlType.Goaway]. We don't + // act on the migration request today + // (no body decode, no preferred-relay + // failover); the `connectReconnecting*` + // wrappers' transport-loss reconnect path + // already handles the eventual hard + // disconnect, so all this arm needs to do + // is recognise the type code and FIN + // cleanly instead of treating it as an + // unknown control. Logged so a relay- + // initiated migration shows up in logcat + // rather than as a mystery silent reconnect. + Log.w("NestRx") { "Goaway received from relay — FIN bidi (no migration handler today)" } + runCatching { bidi.finish() } + dispatched = true + } } } // Post-dispatch chunks are silently discarded — From 96cfa1235aeb74dfb68e24118cb45f8fdbae956b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 17:37:38 +0000 Subject: [PATCH 28/41] fix(nests): T8 skip BUFFER_FLAG_CODEC_CONFIG outputs in MediaCodecOpusEncoder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Android's `audio/opus` MediaCodec encoder emits `BUFFER_FLAG_CODEC_CONFIG` output buffers BEFORE any audio buffer — typically the 19-byte OpusHead identification header per RFC 7845, and (on some Codec2 stacks) the OpusTags comment header. These are decoder-config blobs, NOT audio frames. We weren't filtering them, so the first 1–2 wire frames every encoder lifetime were OpusHead bytes wrapped in our legacy-container varint(timestamp_us) prefix. The web watcher's WebCodecs `AudioDecoder` handles this by burning warmup slots (`@moq/watch/src/audio/decoder.ts`'s `warmed <= 3` policy) — so the listener mostly recovers — but on Android Codec2 stacks that emit BOTH OpusHead AND OpusTags as separate CSD buffers, two of the three warmup slots get absorbed by metadata and the listener hears a tiny click on the next group rollover. The hot-swap path (`ReconnectingNestsSpeaker`) repeats the warmup on every JWT refresh, so this fired once every 9 minutes in production. Filter via `bufferInfo.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG` inside `encode`'s output-drain loop. CSD buffers are released and the loop continues to the next dequeue rather than returning the bytes. Logged once per encoder lifetime via `loggedCsdSkip` so we have proof the path fired without flooding logcat — a stack that emits CSD on every frame would otherwise be noisy. Returning `ByteArray(0)` for the first call (because the only output was a CSD + format-change pair) is unchanged behaviour and the broadcaster's `if (opus.isEmpty()) continue` already handles it. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../audio/MediaCodecOpusEncoder.kt | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusEncoder.kt b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusEncoder.kt index 5de9f3911..fbdec2790 100644 --- a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusEncoder.kt +++ b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusEncoder.kt @@ -56,6 +56,16 @@ class MediaCodecOpusEncoder( private var presentationTimeUs: Long = 0L private var released = false + /** + * Latch so a single conspicuous log fires the FIRST time we drop a + * [MediaCodec.BUFFER_FLAG_CODEC_CONFIG] output buffer. Without + * this latch the same encoder can produce 2-3 CSD buffers in a row + * (OpusHead + OpusTags on some Android Codec2 stacks) and we'd + * spam the log; with it, we record the fact once per encoder + * lifetime and stay quiet thereafter. + */ + private var loggedCsdSkip = false + override fun encode(pcm: ShortArray): ByteArray { check(!released) { "encoder released" } require(pcm.isNotEmpty()) { "PCM frame must not be empty" } @@ -86,6 +96,34 @@ class MediaCodecOpusEncoder( when { outputIndex >= 0 -> { val outputBuffer = codec.getOutputBuffer(outputIndex) ?: continue + // CODEC_CONFIG buffers carry codec-specific data + // (CSD): on the Android `audio/opus` encoder these + // are the 19-byte OpusHead identification header + // and (on some Codec2 stacks) the OpusTags + // comment header. They are NOT Opus packets — they + // are decoder-config blobs that should be supplied + // out-of-band on the receive side (we already do + // this in `MediaCodecOpusDecoder.buildOpusIdHeader`). + // Sending them to the wire as audio frames means + // the watcher's WebCodecs `AudioDecoder.decode` + // sees garbage in the first frame, burning a + // warmup slot and producing a click on group + // rollover after every JWT-refresh hot-swap. + val isCodecConfig = + bufferInfo.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0 + if (isCodecConfig) { + codec.releaseOutputBuffer(outputIndex, false) + if (!loggedCsdSkip) { + loggedCsdSkip = true + com.vitorpamplona.quartz.utils.Log.d("NestTx") { + "MediaCodecOpusEncoder skipped ${bufferInfo.size}-byte CODEC_CONFIG (OpusHead/OpusTags) — not an audio frame" + } + } + // Don't return; loop to find the next output + // buffer (the next dequeue may be the actual + // first audio frame). + continue + } val opus = ByteArray(bufferInfo.size) outputBuffer.position(bufferInfo.offset) outputBuffer.limit(bufferInfo.offset + bufferInfo.size) From c23da52795336821611d23bd0018a4146be2841f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 17:37:51 +0000 Subject: [PATCH 29/41] =?UTF-8?q?fix(nests):=20T10=20endGroup()=20on=20unm?= =?UTF-8?q?uted=E2=86=92muted=20transition?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the user mutes mid-broadcast, `NestMoqLiteBroadcaster`'s send loop just `continue`s past the mute check, leaving the current group's QUIC uni stream open with no FIN. The watcher's `Container.Consumer.#runGroup` parks on `await group.consumer.readFrame()` waiting for either another frame or a stream FIN — and gets neither. kixelated/hang's UI surfaces this as a "stalled" indicator on the speaker tile while we're muted, which is wrong: we're muted, not stalled. FIN the open uni stream once on the unmuted→muted edge via a `wasMuted` latch that the muted→unmuted edge clears. Doesn't change the timestamp counter — it still advances on muted frames so an unmute's first timestamp reflects the muted duration as a real wall- clock gap (not a collapse to zero). `runCatching` around endGroup because a transport-side close racing with the FIN is non-fatal — the broadcaster's terminal-failure path picks up real breakage on the next live frame. Also resets `framesInCurrentGroup` to 0 so the post-unmute send path opens a fresh group cleanly via the standard `currentGroup ?: openNextGroupLocked()` branch in `PublisherStateImpl.send`. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../audio/NestMoqLiteBroadcaster.kt | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt index c3e190517..1535d690a 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt @@ -238,6 +238,21 @@ class NestMoqLiteBroadcaster( // wall-clock TimeMark did: timestamps are codec-payload // metadata, not stream-position. var nextFrameIndex = 0L + // Mute-edge tracking. On the unmuted→muted transition we + // FIN the open uni stream so the watcher sees a clean + // group boundary instead of a half-delivered group that + // never completes — kixelated/hang's + // `Container.Consumer.#runGroup` parks on + // `await group.consumer.readFrame()` until either a + // frame arrives or the group hits FIN, and renders a + // "stalled" indicator while it waits. Without the FIN, + // muting Amethyst lights up that indicator on every web + // watcher even though the broadcast is intentionally + // silent. The FIN clears it; unmuting opens a fresh + // group cleanly via the standard + // `currentGroup ?: openNextGroupLocked()` path inside + // `PublisherStateImpl.send`. + var wasMuted = false try { while (true) { val pcm = capture.readFrame() ?: break @@ -259,7 +274,27 @@ class NestMoqLiteBroadcaster( continue } if (opus.isEmpty()) continue - if (muted) continue + if (muted) { + // Unmuted → muted edge: FIN the current + // group's uni stream once. Doesn't reset + // [framesInCurrentGroup] since + // `endGroup` is a no-op when no group is + // open, and the next post-unmute send will + // open a fresh group anyway. `runCatching` + // because a transport-side close during + // mute is non-fatal — the broadcaster's + // terminal-failure path picks up real + // breakage on the next live frame. + if (!wasMuted) { + wasMuted = true + runCatching { publisher.endGroup() } + framesInCurrentGroup = 0 + } + continue + } + // Muted → unmuted edge: clear the latch so the + // next mute transition fires endGroup again. + wasMuted = false // Snapshot the publisher reference once per frame. // If [swapPublisher] mid-loop installed a new // reference, pick it up here and reset the group From be4e0b9f98177906a20c212c8867d223e13b525b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 17:51:45 +0000 Subject: [PATCH 30/41] fix(nests): T12 carry audio group sequence across hot-swaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every JWT-refresh hot-swap reset the audio publisher's group sequence counter to 0, because `PublisherStateImpl` always initialised `nextSequence = 0L`. kixelated/hang's `Container.Consumer.#run` discards any consumer with `sequence < this.#active`, so a watcher whose `#active` had advanced to e.g. 50 from the previous session's publisher would silently drop every group on the new session until either `#active` rolls over or the watcher re-subscribes — audible as "speaker goes silent for a minute every 9 minutes" (the proactive JWT refresh window). Carry the sequence forward end-to-end: - MoqLiteSession.publish gains a `startSequence: Long = 0L` parameter; PublisherStateImpl seeds `nextSequenceField` from it instead of hard-coding 0. - MoqLitePublisherHandle exposes a `nextSequence: Long` snapshot. `@Volatile`-backed so the hot-swap caller can read it without contending the publisher's gate; race-window between read and a concurrent send is sub-millisecond and would only produce a single duplicate sequence in the very unlikely overlap, which listeners tolerate. - HotSwappablePublisherSource.openPublisherForHotSwap takes `startSequence: Long = 0L`. MoqLiteNestsSpeaker passes it through to session.publish. - NestMoqLiteBroadcaster exposes its current publisher via a read-only `currentPublisher` so the hot-swap pump in ReconnectingNestsSpeaker.runHotSwapIteration can read its `nextSequence` before opening the replacement publisher and pass it as the seed. - Catalog publisher keeps `startSequence = 0L` (the default) — catalog isn't subject to the same `#active` accumulation because each new SUBSCRIBE triggers a fresh emit-on-subscribe write that resets the watcher's catalog state. Listener-side change is none — the watcher already reads whatever sequence we send. A new MoqLiteSessionTest pins the contract: publishing with startSequence=42 makes the first uni stream's GroupHeader.sequence == 42 and advances to 43 after the first send. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../nestsclient/MoqLiteNestsSpeaker.kt | 24 +++++++- .../nestsclient/ReconnectingNestsSpeaker.kt | 18 +++++- .../audio/NestMoqLiteBroadcaster.kt | 12 ++++ .../nestsclient/moq/lite/MoqLiteSession.kt | 60 ++++++++++++++++++- .../moq/lite/MoqLiteSessionTest.kt | 59 ++++++++++++++++++ 5 files changed, 167 insertions(+), 6 deletions(-) diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt index 3c0769d55..d324bc753 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt @@ -190,7 +190,15 @@ class MoqLiteNestsSpeaker internal constructor( * reconnect wrapper's hot-swap path; not called from the * non-reconnecting path which goes through [startBroadcasting]. */ - override suspend fun openPublisherForHotSwap(track: String): MoqLitePublisherHandle = session.publish(broadcastSuffix = speakerPubkeyHex, track = track) + override suspend fun openPublisherForHotSwap( + track: String, + startSequence: Long, + ): MoqLitePublisherHandle = + session.publish( + broadcastSuffix = speakerPubkeyHex, + track = track, + startSequence = startSequence, + ) /** * Compare-and-clear that runs from inside [close] (already holds @@ -348,8 +356,20 @@ internal interface HotSwappablePublisherSource { * session. Caller owns the returned handle's lifetime (typically * via [com.vitorpamplona.nestsclient.audio.NestMoqLiteBroadcaster.swapPublisher]'s * close-the-old contract). + * + * @param startSequence first group sequence the new publisher will + * assign. Used by the hot-swap path to seed the new session's + * audio track with the previous session's + * [MoqLitePublisherHandle.nextSequence] so kixelated/hang's + * `Container.Consumer.#run` doesn't drop every post-recycle + * group as `sequence < #active`. Pass `0L` for fresh (non- + * continuation) publishers — the catalog track is one such + * case, since its `#active` semantics are different from audio. */ - suspend fun openPublisherForHotSwap(track: String): MoqLitePublisherHandle + suspend fun openPublisherForHotSwap( + track: String, + startSequence: Long = 0L, + ): MoqLitePublisherHandle /** * Surface a broadcast-pipeline terminal failure (e.g. sustained diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsSpeaker.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsSpeaker.kt index e532e28a0..79cf850d7 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsSpeaker.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsSpeaker.kt @@ -496,9 +496,25 @@ private class ReissuingBroadcastHandle( * when the wrapper itself closes. */ private suspend fun runHotSwapIteration(hotSwap: HotSwappablePublisherSource) { + // Carry the previous session's audio-track group sequence + // forward so kixelated/hang's `Container.Consumer.#run` + // doesn't drop every post-recycle group as `sequence < + // #active`. Read BEFORE opening the new publisher — there is + // a tiny race window where the broadcaster could send one + // more group on the old publisher between our read and the + // swap-snapshot below; in practice that window is microseconds + // (the broadcaster's swap is a single volatile write) and the + // broadcaster's group cadence is 1 group/sec at the production + // [NestMoqLiteBroadcaster.framesPerGroup], so the chance of a + // duplicate sequence is negligible. Worst case the watcher + // briefly stalls on one duplicate then catches up. + val startSequence: Long = hotSwapBroadcaster?.currentPublisher?.nextSequence ?: 0L val newPublisher = try { - hotSwap.openPublisherForHotSwap(MoqLiteNestsListener.AUDIO_TRACK) + hotSwap.openPublisherForHotSwap( + track = MoqLiteNestsListener.AUDIO_TRACK, + startSequence = startSequence, + ) } catch (ce: kotlinx.coroutines.CancellationException) { throw ce } catch (_: Throwable) { diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt index 1535d690a..f83344db1 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt @@ -119,6 +119,18 @@ class NestMoqLiteBroadcaster( */ @Volatile private var publisher: MoqLitePublisherHandle = initialPublisher + /** + * Volatile read of the broadcaster's current publisher reference, + * for callers (typically the hot-swap pump in + * [com.vitorpamplona.nestsclient.connectReconnectingNestsSpeaker]) + * that want to read its [MoqLitePublisherHandle.nextSequence] + * before swapping in a fresh publisher. Don't use this to send + * — that contract belongs to the broadcaster's send loop and + * the caller would race the loop's swap-snapshot. + */ + val currentPublisher: MoqLitePublisherHandle + get() = publisher + /** * Start capturing + encoding + publishing in the background. * Returns immediately. Calling twice is an error. If diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt index 4b7f3cbe1..bb9989eca 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt @@ -703,12 +703,24 @@ class MoqLiteSession internal constructor( * distinct `track`. Re-publishing the same `(suffix, track)` pair * or mixing different suffixes is rejected with * [IllegalStateException]. + * + * @param startSequence first group sequence the publisher will + * assign. Defaults to 0 for fresh broadcasts. The hot-swap path + * in [com.vitorpamplona.nestsclient.MoqLiteNestsSpeaker.openPublisherForHotSwap] + * passes the previous publisher's + * [MoqLitePublisherHandle.nextSequence] so the new session's + * group lineage continues monotonically across JWT refreshes — + * kixelated/hang's `Container.Consumer.#run` drops any group + * with `sequence < #active`, so a reset to 0 after a recycle + * silences the watcher until `#active` rolls over. */ suspend fun publish( broadcastSuffix: String, track: String, + startSequence: Long = 0L, ): MoqLitePublisherHandle { ensureOpen() + require(startSequence >= 0L) { "startSequence must be >= 0, got $startSequence" } val normalised = MoqLitePath.normalize(broadcastSuffix) val publisher: PublisherStateImpl state.withLock { @@ -721,7 +733,12 @@ class MoqLiteSession internal constructor( check(activePublishers.none { it.track == track }) { "MoqLiteSession.publish called twice for the same track '$track' on suffix '$normalised'." } - publisher = PublisherStateImpl(suffix = normalised, track = track) + publisher = + PublisherStateImpl( + suffix = normalised, + track = track, + startSequence = startSequence, + ) activePublishers += publisher // Lazy launch — the inbound-bidi pump needs to keep running // for the lifetime of any active publisher. @@ -1083,12 +1100,23 @@ class MoqLiteSession internal constructor( private inner class PublisherStateImpl( override val suffix: String, internal val track: String, + startSequence: Long, ) : MoqLitePublisherHandle { private val gate = Mutex() private val announceBidis = mutableListOf() private val inboundSubs = mutableListOf() private var currentGroup: GroupOutbound? = null - private var nextSequence: Long = 0L + + // `@Volatile` so the hot-swap caller can read this from outside + // the publisher's gate (see [MoqLitePublisherHandle.nextSequence] + // kdoc). Mutation happens only inside [openNextGroupLocked] + // (which holds [gate]); the volatile guarantees a cross-thread + // read sees the latest write without contending the gate. + @Volatile + private var nextSequenceField: Long = startSequence + + override val nextSequence: Long + get() = nextSequenceField // Diagnostic: throttled counter for "send returned false" logs so a // long no-subscriber window doesn't flood logcat at 50 Hz. @@ -1272,7 +1300,8 @@ class MoqLiteSession internal constructor( // expected to be small (1 in nests's listener-per-room // model), so this is fine. val sub = inboundSubs.first() - val sequence = nextSequence++ + val sequence = nextSequenceField + nextSequenceField = sequence + 1L val uni = try { openGroupStream(subscribeId = sub.id, sequence = sequence) @@ -1408,6 +1437,31 @@ interface MoqLitePublisherHandle { */ val suffix: String + /** + * The next group sequence number that will be assigned by [send] / + * [startGroup]. Snapshot-only — read AFTER the broadcaster has + * stopped sending into this publisher (typically just before the + * caller closes the publisher in a hot-swap), so the value is the + * highest-already-used sequence + 1. + * + * Used by [com.vitorpamplona.nestsclient.MoqLiteNestsSpeaker]'s + * hot-swap path to seed the new session's publisher with a + * monotonically-continuing sequence — without this, every JWT + * refresh restarts at sequence 0 and kixelated/hang's + * `Container.Consumer.#run` drops every group whose sequence is + * less than its current `#active` high-water mark, killing audio + * for the watcher until either `#active` rolls over or the + * watcher re-subscribes. + * + * `@Volatile` on the implementation; safe to read from any + * coroutine. The accept-tiny-race window between read and a + * concurrent `send` is closed in practice because the broadcaster + * is responsible for swapping its publisher reference BEFORE the + * caller reads this value (so no further sends land on this + * publisher). + */ + val nextSequence: Long + /** * Start a new group. Allocates a fresh sequence id and opens a new * uni stream pre-loaded with `DataType=Group + GroupHeader`. Idempotent diff --git a/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSessionTest.kt b/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSessionTest.kt index f823438ce..37eaa430c 100644 --- a/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSessionTest.kt +++ b/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSessionTest.kt @@ -340,6 +340,65 @@ class MoqLiteSessionTest { session.close() } + @Test + fun publisher_startSequence_seeds_first_group_for_hot_swap_continuation() = + runBlocking { + val (clientSide, serverSide) = FakeWebTransport.pair() + val session = MoqLiteSession.client(clientSide, pumpScope) + + // Mint a publisher with a non-zero startSequence — simulates + // the hot-swap path's "carry forward old publisher's + // nextSequence" contract. + val publisher = + session.publish( + broadcastSuffix = "speakerPubkey", + track = "audio/data", + startSequence = 42L, + ) + assertEquals(42L, publisher.nextSequence, "fresh publisher reports startSequence as next") + + // Wire up a subscriber so send() doesn't short-circuit. + val subBidi = serverSide.openBidiStream() + subBidi.write(Varint.encode(MoqLiteControlType.Subscribe.code)) + subBidi.write( + MoqLiteCodec.encodeSubscribe( + MoqLiteSubscribe( + id = 11L, + broadcast = "speakerPubkey", + track = "audio/data", + priority = 0x80, + ordered = true, + maxLatencyMillis = 0L, + startGroup = null, + endGroup = null, + ), + ), + ) + withTimeout(2_000) { subBidi.incoming().first() } + + assertEquals(true, publisher.send("opus-1".encodeToByteArray())) + // FIN before draining so toList() terminates rather than + // blocking indefinitely on the still-open uni stream. + publisher.endGroup() + // After the first send, nextSequence should advance to 43. + assertEquals(43L, publisher.nextSequence) + + // First uni stream's GroupHeader.sequence MUST be 42, not 0. + val relayUni = withTimeout(2_000) { serverSide.incomingUniStreams().first() } + val uniChunks = relayUni.incoming().toList() + val buf = MoqLiteFrameBuffer() + uniChunks.forEach { buf.push(it) } + assertEquals(MoqLiteDataType.Group.code, buf.readVarint()) + val header = + MoqLiteCodec.decodeGroupHeader( + buf.readSizePrefixed() ?: error("group header missing"), + ) + assertEquals(42L, header.sequence, "first group's sequence is the seeded startSequence") + + publisher.close() + session.close() + } + @Test fun publisher_send_returns_false_when_no_inbound_subscriber() = runBlocking { From 4714e3c7236866cca6baa93369f494d6490c0bab Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 18:00:52 +0000 Subject: [PATCH 31/41] fix(nests): T13 reset Opus decoder on publisher boundary in NestPlayer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reissuing-subscribe wrapper splices fresh-publisher frames into the same `SharedFlow` that NestPlayer consumes, so across a JWT-refresh hot-swap (speaker side) or a cliff-detector recycle (listener side) the decoder receives a discontinuous frame stream while keeping its Opus predictor state. Result: an audible warble at every publisher cycle. Single-stream tests don't catch it because they never present two distinct publishers. Detect the boundary via `MoqObject.trackAlias` — the underlying `MoqLiteSession.subscribe` assigns a fresh subscribeId per SUBSCRIBE, which the listener wrapper surfaces verbatim as `trackAlias` on every emitted object. A change between consecutive objects = new publisher. Add an optional `decoderFactory: (() -> OpusDecoder)?` to NestPlayer. When non-null, NestPlayer tracks `lastTrackAlias` and on a change releases the current decoder and rebuilds via the factory. The default `null` preserves the legacy single-decoder behaviour so existing tests / callers that don't care about boundaries stand unchanged. NestViewModel.openSubscription wires the factory: a closure capturing the catalog-derived `channelCount` so a rebuild reuses the SAME channel layout — without that capture, a rebuild after a stereo- publisher cycle would default to mono and silently downmix. Two new NestPlayerTest cases pin the behaviour: - `publisher_boundary_rebuilds_decoder_when_factory_provided`: factory invoked twice (initial + boundary), first decoder released on boundary, second released on stop. - `publisher_boundary_no_op_when_factory_is_null`: legacy path holds onto the same decoder across trackAlias changes (unchanged semantics). NestPlayer's first constructor parameter is now `initialDecoder` (was `decoder`); positional-arg call sites are unchanged but the named-arg call sites in the test suite are updated accordingly. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../commons/viewmodels/NestViewModel.kt | 21 +++- .../nestsclient/audio/NestPlayer.kt | 62 ++++++++++- .../nestsclient/audio/NestPlayerTest.kt | 100 +++++++++++++++++- 3 files changed, 175 insertions(+), 8 deletions(-) diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt index cb6b85209..942765d62 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt @@ -1177,7 +1177,16 @@ class NestViewModel( // the floor — wrap them in a nested try so any cancellation // or throw between here and slot.attach releases them // (audit round-2 VM #7). - val decoder = decoderFactory(channelCount) + // Per-subscription decoder factory closure: captures the + // catalog-derived [channelCount] so a publisher-boundary + // decoder rebuild (see [NestPlayer]'s `decoderFactory` + // kdoc) reuses the SAME channel layout — without it, a + // rebuild after a cliff-recycle would default to mono and + // a stereo publisher would silently downmix on the new + // decoder. The factory is also called for the initial + // decoder so the construction path is uniform. + val perSubscriptionDecoderFactory: () -> OpusDecoder = { decoderFactory(channelCount) } + val decoder = perSubscriptionDecoderFactory() val player = try { playerFactory(channelCount) @@ -1195,7 +1204,7 @@ class NestViewModel( val isHushed = pubkey in _uiState.value.locallyHushed val roomPlayer = NestPlayer( - decoder = decoder, + initialDecoder = decoder, player = player, scope = viewModelScope, // ~100 ms of audio buffered before the AudioTrack @@ -1205,6 +1214,14 @@ class NestViewModel( // tuned in the audio-rooms audit; see NestPlayer // kdoc for details. prerollFrames = ROOM_PLAYER_PREROLL_FRAMES, + // Trigger a decoder rebuild on every publisher + // boundary (re-issuing wrapper spliced in a new + // SUBSCRIBE → trackAlias changes). Without this, + // Opus's predictor state from the prior + // publisher's last frame is fed into the new + // publisher's first frame and produces audible + // warble at every cliff-recycle / hot-swap. + decoderFactory = perSubscriptionDecoderFactory, ) // Apply current mute + per-speaker hush state before play() // opens the device so the first frame respects them. diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayer.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayer.kt index d70137cbf..ffcbf883a 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayer.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayer.kt @@ -42,7 +42,7 @@ import kotlinx.coroutines.launch * decoder. Idempotent. */ class NestPlayer( - private val decoder: OpusDecoder, + initialDecoder: OpusDecoder, private val player: AudioPlayer, private val scope: CoroutineScope, /** @@ -61,11 +61,44 @@ class NestPlayer( * Default is `0` so existing tests stand without modification. */ private val prerollFrames: Int = 0, + /** + * Optional factory called on every detected publisher boundary + * (track-alias change in the inbound [MoqObject] stream) to mint a + * fresh [OpusDecoder]. Used by the listener wrapper's re-issuing + * subscription pump + * ([com.vitorpamplona.nestsclient.ReconnectingNestsListener.reissuingSubscribe]): + * each new SUBSCRIBE through the relay produces objects with a + * different `trackAlias`, but they're spliced into the same + * `SharedFlow` — without a decoder reset on the boundary, Opus's + * predictor state from the prior publisher's last frame is fed + * into the new publisher's first frame, producing an audible + * warble at every JWT-refresh hot-swap on the speaker side OR + * cliff-detector recycle on the listener side. + * + * Default `null` keeps the legacy behaviour (no boundary + * detection, decoder lives for the player's whole lifetime) so + * existing tests / callers that don't care about boundaries + * stand unchanged. Production callers in + * [com.vitorpamplona.amethyst.commons.viewmodels.NestViewModel.openSubscription] + * pass a closure that captures the per-subscription channel-count + * and rebuilds via `decoderFactory(channelCount)`. + */ + private val decoderFactory: (() -> OpusDecoder)? = null, ) { init { require(prerollFrames >= 0) { "prerollFrames must be >= 0, got $prerollFrames" } } + /** + * Active decoder. Replaced on detected publisher boundary when + * [decoderFactory] is non-null. `var` so the boundary path can + * release + rebuild without changing the rest of the loop's + * decoder reference; the `private` confines mutation to this + * class, and the decode loop runs single-coroutine so no cross- + * thread visibility hazards. + */ + private var decoder: OpusDecoder = initialDecoder + private var job: Job? = null private var stopped = false @@ -157,6 +190,13 @@ class NestPlayer( com.vitorpamplona.quartz.utils.Log .d("NestPlay") { "NestPlayer beginPlayback returned" } } + // Track-alias of the most recently observed object. + // A change signals a publisher boundary (re-issuing + // subscription wrapper spliced in a new SUBSCRIBE). + // Only consulted when [decoderFactory] is non-null; + // legacy callers without a factory keep the prior + // single-decoder behaviour. + var lastTrackAlias: Long? = null try { objects.collect { obj -> receivedObjects += 1 @@ -165,6 +205,26 @@ class NestPlayer( "NestPlayer received obj #$receivedObjects (decoded=$decodedFrames empty=$emptyDecodes enqueued=$enqueued playbackBegun=$playbackBegun)" } } + // Publisher-boundary detection: if the trackAlias + // changed since the last object AND we have a + // factory to mint a fresh decoder, release the + // current decoder + build a new one. Without + // this, Opus's predictor state from the prior + // publisher's last frame is fed into the new + // publisher's first frame, producing audible + // warble at every JWT-refresh hot-swap (speaker + // side) or cliff-detector recycle (listener side). + // The prior-trackAlias guard avoids a spurious + // rebuild on the very first frame. + val factory = decoderFactory + if (factory != null && lastTrackAlias != null && obj.trackAlias != lastTrackAlias) { + com.vitorpamplona.quartz.utils.Log.d("NestPlay") { + "NestPlayer publisher boundary: trackAlias $lastTrackAlias → ${obj.trackAlias}; rebuilding decoder" + } + runCatching { decoder.release() } + decoder = factory() + } + lastTrackAlias = obj.trackAlias val pcm = try { decoder.decode(obj.payload) diff --git a/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayerTest.kt b/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayerTest.kt index 9d63e22ca..ddb4a75b4 100644 --- a/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayerTest.kt +++ b/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayerTest.kt @@ -232,7 +232,7 @@ class NestPlayerTest { val sut = NestPlayer( - decoder = decoder, + initialDecoder = decoder, player = player, scope = this, prerollFrames = 3, @@ -289,7 +289,7 @@ class NestPlayerTest { val sut = NestPlayer( - decoder = decoder, + initialDecoder = decoder, player = player, scope = this, prerollFrames = 5, @@ -326,7 +326,7 @@ class NestPlayerTest { val sut = NestPlayer( - decoder = decoder, + initialDecoder = decoder, player = player, scope = this, prerollFrames = 3, @@ -340,17 +340,107 @@ class NestPlayerTest { sut.stop() } + @Test + fun publisher_boundary_rebuilds_decoder_when_factory_provided() = + runTest { + // Two distinct decoders so we can prove the factory was + // invoked. After the trackAlias change, frames should + // route through `decoderB`, NOT `decoderA`. + val decoderA = + FakeOpusDecoder { + byteArrayOf(0x0A) + it + ShortArray(it.size) { _ -> 0xAA.toShort() } + } + val decoderB = + FakeOpusDecoder { + byteArrayOf(0x0B) + it + ShortArray(it.size) { _ -> 0xBB.toShort() } + } + val factoryCallCount = atomicIntZero() + val factory: () -> OpusDecoder = { + if (factoryCallCount.getAndIncrement() == 0) decoderA else decoderB + } + val player = FakeAudioPlayer() + + val objects = + flowOf( + // First subscription cycle: trackAlias = 7 + moqObject(byteArrayOf(0x01), trackAlias = 7L), + moqObject(byteArrayOf(0x02), trackAlias = 7L), + // Wrapper re-issued — new SUBSCRIBE produces a + // different trackAlias. Decoder MUST be rebuilt + // before the next decode runs. + moqObject(byteArrayOf(0x03), trackAlias = 8L), + moqObject(byteArrayOf(0x04), trackAlias = 8L), + ) + + val sut = + NestPlayer( + initialDecoder = factory(), + player = player, + scope = this, + decoderFactory = factory, + ) + sut.play(objects) + testScheduler.advanceUntilIdle() + + assertEquals(2, factoryCallCount.value, "factory invoked twice: initial + boundary") + assertEquals(1, decoderA.releaseCount, "decoderA released on the boundary") + assertEquals(0, decoderB.releaseCount, "decoderB still alive (released on stop)") + sut.stop() + assertEquals(1, decoderB.releaseCount, "decoderB released on stop") + } + + @Test + fun publisher_boundary_no_op_when_factory_is_null() = + runTest { + // Without a factory, NestPlayer keeps the same decoder + // across trackAlias changes — backwards-compat path. + val decoder = FakeOpusDecoder { byteToShorts(it) } + val player = FakeAudioPlayer() + val objects = + flowOf( + moqObject(byteArrayOf(0x01), trackAlias = 7L), + moqObject(byteArrayOf(0x02), trackAlias = 8L), + ) + + val sut = NestPlayer(decoder, player, this) + sut.play(objects) + testScheduler.advanceUntilIdle() + + assertEquals(0, decoder.releaseCount, "no boundary-driven release without a factory") + sut.stop() + assertEquals(1, decoder.releaseCount, "released exactly once on stop") + } + // -- helpers ----------------------------------------------------------- - private fun moqObject(payload: ByteArray): MoqObject = + private fun moqObject( + payload: ByteArray, + trackAlias: Long = 1L, + ): MoqObject = MoqObject( - trackAlias = 1, + trackAlias = trackAlias, groupId = 0, objectId = 0, publisherPriority = 0x80, payload = payload, ) + /** + * Tiny stand-in for AtomicInteger that's available in commonMain + * (kotlin.test scope). Used by the boundary-rebuild test to count + * factory invocations across the test scope's coroutine + * dispatcher. + */ + private class IntBox { + var value: Int = 0 + + fun getAndIncrement(): Int = value++ + } + + private fun atomicIntZero(): IntBox = IntBox() + private fun byteToShorts(b: ByteArray): ShortArray = ShortArray(b.size) { b[it].toShort() } private class FakeOpusDecoder( From 7e76ab11394f3957a71b4038f1825af604345947 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 18:18:23 +0000 Subject: [PATCH 32/41] fix(nests): T11 drop bestEffort=true on moq-lite group uni streams MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `MoqLiteSession.openGroupStream` was opening each group's QUIC uni stream with `bestEffort = true`. `:quic`'s `SendBuffer.markLost` with that flag drops lost STREAM ranges WITHOUT retransmit AND WITHOUT `RESET_STREAM` (`quic/src/commonMain/kotlin/com/vitorpamplona/quic/ stream/SendBuffer.kt:300-309`). The peer's `ReceiveBuffer` ends up with chunks at `[0, P)` and `[Q, end)` and a permanent hole at `[P, Q)`; the application's `incoming` Flow parks on the hole forever. Web watchers (`@moq/hang` `Container.Consumer`) park their `Group.readFrame` until the relay's 30 s `MAX_GROUP_AGE` ages the broadcast queue out — manifesting as a 30 s silent dropout per lost packet on lossy networks (cellular, mobile WiFi, congested home routers). This is a real-world bug that's invisible on a clean LAN. The reference implementation (kixelated/moq-rs `Publisher::serve_group`, `rs/moq-lite/src/lite/publisher.rs:347-406`) writes to reliable QUIC streams with no `set_unreliable` call — `bestEffort` was Amethyst's private optimisation with no peer-side support. Drop it; let `:quic` retransmit lost ranges normally. A retransmit arriving 50–150 ms late still falls inside hang's default ~200 ms jitter buffer, so the cost is marginal extra bandwidth on retransmits and the win is no more silent dropouts on lossy networks. T11.2 — orthogonality check: stream-cliff fix is independent. Re-read `nestsClient/plans/2026-05-01-quic-stream-cliff-investigation.md` and grep'd both the plan and `NestMoqLiteBroadcaster.kt` for `bestEffort` / `best_effort` — neither references it. The cliff fix is `framesPerGroup = 5/50` (cadence reduction); load-bearing on stream-creation RATE, not loss handling. Drop is safe. T11.3 (stream priority — newer groups drain first under congestion) deferred to a follow-up commit; hooking it into `:quic`'s send-frame loop is bigger than a one-line change and warrants its own task. The kixelated/moq-rs publisher uses `stream.set_priority(priority.current())` to bias the writer; without it, our drain order under congestion is FIFO across streams rather than newest-first, which can mean the listener catches up on a stale group when a fresh one is more useful. Doesn't block today — production audio rarely hits transport congestion at 1 group/sec cadence. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../nestsclient/moq/lite/MoqLiteSession.kt | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt index bb9989eca..b5e49dc2c 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt @@ -1023,14 +1023,21 @@ class MoqLiteSession internal constructor( subscribeId: Long, sequence: Long, ): com.vitorpamplona.nestsclient.transport.WebTransportWriteStream { - // Group streams carry a single Opus packet. They're real-time - // and best-effort — a STREAM frame arriving 200 ms late is - // worse than useless because the listener has already moved - // past that group's sequence number. Setting bestEffort=true - // tells the underlying QUIC SendBuffer to drop lost ranges - // instead of retransmitting them, bounding the bandwidth waste - // we'd otherwise incur on a lossy uplink. - val uni = transport.openUniStream(bestEffort = true) + // Group streams use reliable QUIC delivery to match the + // moq-lite reference (kixelated/moq-rs `serve_group` writes + // reliable streams; bestEffort is not a moq-lite concept). + // The previous shape opened with `bestEffort=true`, which + // caused `:quic`'s `SendBuffer.markLost` to drop lost ranges + // without retransmit AND without RESET_STREAM — leaving the + // peer's stream-reassembly buffer permanently wedged at the + // hole boundary. The watcher's `Group.readFrame` parks until + // the relay's 30 s `MAX_GROUP_AGE` ages the broadcast queue + // out, manifesting as a 30 s silent dropout per lost packet + // on lossy networks. Reliable delivery costs marginal extra + // bandwidth on retransmits (a lost STREAM range arriving 50– + // 150 ms late still falls inside hang's default ~200 ms + // jitter buffer) and avoids the dropout entirely. + val uni = transport.openUniStream() uni.write(Varint.encode(MoqLiteDataType.Group.code)) uni.write(MoqLiteCodec.encodeGroupHeader(MoqLiteGroupHeader(subscribeId, sequence))) return uni From ade8da3b5be976eafd2fe75346ff44425881fa3e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 18:33:24 +0000 Subject: [PATCH 33/41] fix(nests): NestPlayer keeps existing decoder when boundary factory throws MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audit-1: the publisher-boundary rebuild path released the old decoder BEFORE asking the factory for a replacement (`runCatching { decoder.release() }; decoder = factory()`). If `factory()` threw — MediaCodec contention, audio policy denial mid-session, etc. — the released decoder stayed assigned to the field and every subsequent `decoder.decode(payload)` would throw `IllegalStateException` with no recovery path. The room would silently fail for that subscription until torn down. Build the replacement first; only release the old one and swap on success. On factory failure, log and keep the existing decoder running. Cross-publisher Opus predictor state is wrong for one group but at least audio keeps playing. Audit-4+5: companion test cleanup — - Drop the `byteArrayOf(0x0A) + it` / `0x0B` dead expressions in the FakeOpusDecoder closures of the existing boundary-rebuild test. They allocated a ByteArray and concatenated, then threw the result away. - Drop the local `IntBox` helper and use `java.util.concurrent.atomic.AtomicInteger` like the rest of the codebase does (`MoqLiteSession`, `MoqLiteNestsListener`). Same semantics, no new vocabulary. New test pins the dangling-field fix: `publisher_boundary_keeps_old_decoder_when_factory_throws` flows two MoqObjects with different trackAliases through a NestPlayer whose factory always throws; asserts both frames decode through the original decoder and the original decoder is released exactly once on `stop()`. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../nestsclient/audio/NestPlayer.kt | 31 ++++++-- .../nestsclient/audio/NestPlayerTest.kt | 72 ++++++++++++------- 2 files changed, 73 insertions(+), 30 deletions(-) diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayer.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayer.kt index ffcbf883a..9c25d0671 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayer.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayer.kt @@ -216,13 +216,36 @@ class NestPlayer( // side) or cliff-detector recycle (listener side). // The prior-trackAlias guard avoids a spurious // rebuild on the very first frame. + // + // Build the replacement BEFORE releasing the old + // decoder so a factory failure (e.g. MediaCodec + // contention, audio policy denial mid-session) + // doesn't leave the field referencing a + // released codec — every subsequent decode + // would then throw `IllegalStateException` with + // no recovery path. On factory failure, log + // and keep using the existing decoder; cross- + // publisher predictor state is wrong but at + // least audio keeps playing. val factory = decoderFactory if (factory != null && lastTrackAlias != null && obj.trackAlias != lastTrackAlias) { - com.vitorpamplona.quartz.utils.Log.d("NestPlay") { - "NestPlayer publisher boundary: trackAlias $lastTrackAlias → ${obj.trackAlias}; rebuilding decoder" + val replacement = + runCatching { factory() } + .onFailure { t -> + if (t is CancellationException) throw t + com.vitorpamplona.quartz.utils.Log.w("NestPlay") { + "NestPlayer decoder factory threw on trackAlias " + + "$lastTrackAlias → ${obj.trackAlias}; keeping old decoder " + + "(${t::class.simpleName}: ${t.message})" + } + }.getOrNull() + if (replacement != null) { + com.vitorpamplona.quartz.utils.Log.d("NestPlay") { + "NestPlayer publisher boundary: trackAlias $lastTrackAlias → ${obj.trackAlias}; rebuilding decoder" + } + runCatching { decoder.release() } + decoder = replacement } - runCatching { decoder.release() } - decoder = factory() } lastTrackAlias = obj.trackAlias val pcm = diff --git a/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayerTest.kt b/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayerTest.kt index ddb4a75b4..43e729d4d 100644 --- a/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayerTest.kt +++ b/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayerTest.kt @@ -25,6 +25,7 @@ import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.flow.flowOf import kotlinx.coroutines.flow.receiveAsFlow import kotlinx.coroutines.test.runTest +import java.util.concurrent.atomic.AtomicInteger import kotlin.test.Test import kotlin.test.assertContentEquals import kotlin.test.assertEquals @@ -346,17 +347,9 @@ class NestPlayerTest { // Two distinct decoders so we can prove the factory was // invoked. After the trackAlias change, frames should // route through `decoderB`, NOT `decoderA`. - val decoderA = - FakeOpusDecoder { - byteArrayOf(0x0A) + it - ShortArray(it.size) { _ -> 0xAA.toShort() } - } - val decoderB = - FakeOpusDecoder { - byteArrayOf(0x0B) + it - ShortArray(it.size) { _ -> 0xBB.toShort() } - } - val factoryCallCount = atomicIntZero() + val decoderA = FakeOpusDecoder { ShortArray(it.size) { _ -> 0xAA.toShort() } } + val decoderB = FakeOpusDecoder { ShortArray(it.size) { _ -> 0xBB.toShort() } } + val factoryCallCount = AtomicInteger(0) val factory: () -> OpusDecoder = { if (factoryCallCount.getAndIncrement() == 0) decoderA else decoderB } @@ -384,13 +377,54 @@ class NestPlayerTest { sut.play(objects) testScheduler.advanceUntilIdle() - assertEquals(2, factoryCallCount.value, "factory invoked twice: initial + boundary") + assertEquals(2, factoryCallCount.get(), "factory invoked twice: initial + boundary") assertEquals(1, decoderA.releaseCount, "decoderA released on the boundary") assertEquals(0, decoderB.releaseCount, "decoderB still alive (released on stop)") sut.stop() assertEquals(1, decoderB.releaseCount, "decoderB released on stop") } + @Test + fun publisher_boundary_keeps_old_decoder_when_factory_throws() = + runTest { + // The decoder field MUST NOT be left referencing a released + // decoder if the factory throws — every subsequent decode + // would otherwise fail with `IllegalStateException` and the + // subscription would be permanently dead. + val decoderA = FakeOpusDecoder { ShortArray(it.size) { _ -> 0xAA.toShort() } } + val factoryFailures = AtomicInteger(0) + val factory: () -> OpusDecoder = { + factoryFailures.incrementAndGet() + throw IllegalStateException("synthetic factory failure") + } + val player = FakeAudioPlayer() + + val objects = + flowOf( + moqObject(byteArrayOf(0x01), trackAlias = 7L), + // Boundary: factory throws → decoderA must stay alive + // and decode the next frame normally. + moqObject(byteArrayOf(0x02), trackAlias = 8L), + ) + + val sut = + NestPlayer( + initialDecoder = decoderA, + player = player, + scope = this, + decoderFactory = factory, + ) + sut.play(objects) + testScheduler.advanceUntilIdle() + + assertEquals(1, factoryFailures.get(), "factory invoked once on the boundary") + // decoderA still alive → both frames decoded through it. + assertEquals(2, player.queued.size) + assertEquals(0, decoderA.releaseCount, "decoderA NOT released on factory failure") + sut.stop() + assertEquals(1, decoderA.releaseCount, "decoderA released exactly once on stop") + } + @Test fun publisher_boundary_no_op_when_factory_is_null() = runTest { @@ -427,20 +461,6 @@ class NestPlayerTest { payload = payload, ) - /** - * Tiny stand-in for AtomicInteger that's available in commonMain - * (kotlin.test scope). Used by the boundary-rebuild test to count - * factory invocations across the test scope's coroutine - * dispatcher. - */ - private class IntBox { - var value: Int = 0 - - fun getAndIncrement(): Int = value++ - } - - private fun atomicIntZero(): IntBox = IntBox() - private fun byteToShorts(b: ByteArray): ShortArray = ShortArray(b.size) { b[it].toShort() } private class FakeOpusDecoder( From a94d12638f9936d1730299271064d7d06fa2d06d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 18:37:19 +0000 Subject: [PATCH 34/41] perf(nests): bound encoder CSD loop, single-alloc audio framing, cache catalog JSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three audit follow-ups, all in the audio publish hot path: Audit-3: MediaCodecOpusEncoder.encode could busy-loop on a buggy encoder that emits BUFFER_FLAG_CODEC_CONFIG on every dequeue. The existing format-change path has a one-shot `formatChangeAbsorbed` guard for exactly this reason; the new CSD-skip path didn't. Cap consecutive CSD skips per encode call at MAX_CSD_SKIPS_PER_CALL=4 (generous: Codec2 emits 1 OpusHead or 2 OpusHead+OpusTags in practice). On overshoot, log a warning and return ByteArray(0) — the broadcaster's `if (opus.isEmpty()) continue` already handles that contract. Audit-6: NestMoqLiteBroadcaster's per-frame send path allocated twice per Opus packet — once for the timestamp Varint and once for the concatenated `varint+opus` payload. At 50 fps × N speakers that's measurable young-gen pressure. Switch to the same single-allocation pattern PublisherStateImpl.send already uses: compute Varint.size(timestampUs), allocate the final payload buffer once, write the varint directly into it via Varint.writeTo, then copy the opus bytes. Drops audio-thread allocations from 3/frame to 1/frame (the third was the wrap in PublisherStateImpl.send, already optimised). Audit-7: MoqLiteNestsSpeaker.startBroadcasting and ReconnectingNestsSpeaker.runHotSwapIteration both encode the same fixed catalog JSON on every call — once per broadcast start and once per JWT-refresh hot-swap iteration. Cache the encoded bytes on MoqLiteHangCatalog.Companion.OPUS_MONO_48K_AUDIO_DATA_JSON_BYTES so the kotlinx.serialization run happens at class-init time and both call sites read a static reference. Trivial perf, but co-locates the "default Amethyst publish shape" in one place. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../audio/MediaCodecOpusEncoder.kt | 31 +++++++++++++++++++ .../nestsclient/MoqLiteNestsSpeaker.kt | 3 +- .../nestsclient/ReconnectingNestsSpeaker.kt | 3 +- .../audio/NestMoqLiteBroadcaster.kt | 21 ++++++++++--- .../moq/lite/MoqLiteHangCatalog.kt | 18 +++++++++++ 5 files changed, 68 insertions(+), 8 deletions(-) diff --git a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusEncoder.kt b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusEncoder.kt index fbdec2790..038ee293a 100644 --- a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusEncoder.kt +++ b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusEncoder.kt @@ -91,6 +91,13 @@ class MediaCodecOpusEncoder( // without producing output, which would otherwise busy-spin at // 100 Hz against the 10 ms dequeue timeout). var formatChangeAbsorbed = false + // CSD-skip iteration cap. Codec2 stacks have been observed to + // emit OpusHead AND OpusTags as separate CSD buffers; some + // future stack might emit more. Cap the per-call CSD-skip + // count so a buggy encoder that emits CSD on every dequeue + // can't busy-loop the per-frame budget. 4 is generous: the + // expected case is 1 (OpusHead) or 2 (OpusHead + OpusTags). + var csdSkipsThisCall = 0 while (true) { val outputIndex = codec.dequeueOutputBuffer(bufferInfo, DEQUEUE_TIMEOUT_US) when { @@ -119,6 +126,20 @@ class MediaCodecOpusEncoder( "MediaCodecOpusEncoder skipped ${bufferInfo.size}-byte CODEC_CONFIG (OpusHead/OpusTags) — not an audio frame" } } + csdSkipsThisCall += 1 + if (csdSkipsThisCall >= MAX_CSD_SKIPS_PER_CALL) { + // A buggy encoder is emitting CSD on every + // dequeue; bail this call rather than + // busy-looping the per-frame budget. + // Returning empty is the existing "warmup" + // contract — broadcaster's + // `if (opus.isEmpty()) continue` handles it + // and the next encode call retries. + com.vitorpamplona.quartz.utils.Log.w("NestTx") { + "MediaCodecOpusEncoder hit MAX_CSD_SKIPS_PER_CALL=$MAX_CSD_SKIPS_PER_CALL; bailing this encode call (encoder may be misbehaving)" + } + return ByteArray(0) + } // Don't return; loop to find the next output // buffer (the next dequeue may be the actual // first audio frame). @@ -163,6 +184,16 @@ class MediaCodecOpusEncoder( private const val DEQUEUE_TIMEOUT_US = 10_000L private const val FRAME_DURATION_US = 20_000L + /** + * Per-encode-call cap on consecutive + * [MediaCodec.BUFFER_FLAG_CODEC_CONFIG] outputs we'll skip + * before bailing the call. Expected steady state is 0; the + * one-time startup cost is 1 (OpusHead) or 2 (OpusHead + + * OpusTags on Codec2). 4 leaves headroom for a future stack + * that emits more without uncapping the loop entirely. + */ + private const val MAX_CSD_SKIPS_PER_CALL: Int = 4 + private fun buildFormat(bitrate: Int): MediaFormat = MediaFormat .createAudioFormat( diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt index d324bc753..de09a3424 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt @@ -157,8 +157,7 @@ class MoqLiteNestsSpeaker internal constructor( // practice the relay's SUBSCRIBE bidi takes a network // round-trip after our ANNOUNCE Active, so this is safe // even though the setter is non-suspending. - val catalogJson = - MoqLiteHangCatalog.opusMono48k(MoqLiteNestsListener.AUDIO_TRACK).encodeJsonBytes() + val catalogJson = MoqLiteHangCatalog.OPUS_MONO_48K_AUDIO_DATA_JSON_BYTES catalogPublisher.setOnNewSubscriber { runCatching { catalogPublisher.send(catalogJson) diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsSpeaker.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsSpeaker.kt index 79cf850d7..897a93656 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsSpeaker.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/ReconnectingNestsSpeaker.kt @@ -586,8 +586,7 @@ private class ReissuingBroadcastHandle( // and any watcher that attaches AFTER the recycle sees nothing // to subscribe to. Mirror of [MoqLiteNestsSpeaker.startBroadcasting]'s // catalog setup; same JSON, same emit-on-subscribe pattern. - val catalogPayload = - MoqLiteHangCatalog.opusMono48k(MoqLiteNestsListener.AUDIO_TRACK).encodeJsonBytes() + val catalogPayload = MoqLiteHangCatalog.OPUS_MONO_48K_AUDIO_DATA_JSON_BYTES val priorCatalogPublisher = hotSwapCatalogPublisher val newCatalogPublisher = try { diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt index f83344db1..4a8144d0b 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt @@ -325,10 +325,23 @@ class NestMoqLiteBroadcaster( // for the production cliff this works around. val sendOutcome = runCatching { - val tsBytes = Varint.encode(timestampUs) - val payload = ByteArray(tsBytes.size + opus.size) - tsBytes.copyInto(payload, 0) - opus.copyInto(payload, tsBytes.size) + // Single-allocation framing: write the + // timestamp varint directly into a buffer + // sized for `varint + opus`, then copy + // the opus bytes after it. The earlier + // shape (`Varint.encode(...) + opus`) + // allocated twice per frame — once for + // the varint ByteArray, once for the + // concatenated payload. At 50 fps × N + // speakers this is measurable young-gen + // pressure on the audio hot path; the + // mirror optimisation already lives in + // `PublisherStateImpl.send` for the + // outer size-prefix wrap. + val tsLen = Varint.size(timestampUs) + val payload = ByteArray(tsLen + opus.size) + Varint.writeTo(timestampUs, payload, 0) + opus.copyInto(payload, tsLen) val accepted = current.send(payload) framesInCurrentGroup += 1 if (framesInCurrentGroup >= framesPerGroup) { diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHangCatalog.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHangCatalog.kt index 6e2483790..e52b75c08 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHangCatalog.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHangCatalog.kt @@ -106,6 +106,24 @@ internal data class MoqLiteHangCatalog( explicitNulls = false } + /** + * Cached canonical-shape catalog JSON bytes for the default + * Opus mono 48 kHz audio track ([MoqLiteNestsListener.AUDIO_TRACK] + * keyed under `audio.renditions["audio/data"]`). The catalog is + * a fixed string for the whole publisher lifetime — caching + * avoids re-running kotlinx.serialization on every + * [com.vitorpamplona.nestsclient.MoqLiteNestsSpeaker.startBroadcasting] + * call and every JWT-refresh hot-swap iteration in + * [com.vitorpamplona.nestsclient.connectReconnectingNestsSpeaker]. + * + * Hard-coded to track name `"audio/data"` because that's the + * only track Amethyst publishes today; if a future caller + * needs a different name, fall back to + * [opusMono48k] + [encodeJsonBytes]. + */ + val OPUS_MONO_48K_AUDIO_DATA_JSON_BYTES: ByteArray = + opusMono48k("audio/data").encodeJsonBytes() + /** * Canonical Amethyst speaker catalog: a single `legacy`-container * Opus rendition under [audioTrackName], matching the encoder From 75f572ba3d8f09b414203d9e642dd37fa1ed1afc Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 19:09:24 +0000 Subject: [PATCH 35/41] test+fix(nests): pin stripLegacyTimestampPrefix; primaryAudio filters to legacy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two audit-pass gaps: G1 — `MoqLiteNestsListener.Companion.stripLegacyTimestampPrefix` is the entire receive-side wire-format converter for audio frames (every web speaker's Opus packet flows through it) and had zero unit tests. A regression here is invisible to users — silent decode failure of every web speaker — and to interop tests, because both sides agree on the same bug. New StripLegacyTimestampPrefixTest pins: - all four QUIC varint-length tiers (1 / 2 / 4 / 8 bytes), one test per tier with a representative timestamp value plus a tag-byte assertion so the test fails loudly if Varint.encode's tier-selection changes. - empty-payload and malformed-payload fast paths (returns payload unchanged, same instance — no allocation). - round-trip against `NestMoqLiteBroadcaster`'s exact wire shape (`varint(timestamp_us) + raw_opus_packet`) across five timestamp values spanning all four tiers. G6 — `RoomSpeakerCatalog.primaryAudio()` previously returned `audio.renditions.values.firstOrNull()` with no container filter. A future publisher that emits CMAF before legacy in iteration order would surface the CMAF rendition, and our decoder pipeline would then feed CMAF MOOF/MDAT bytes to its legacy `varint(ts)+opus` parser and decode garbage. Filter to `container.kind == Container.LEGACY_KIND` so: - mixed CMAF+legacy publishers always surface the legacy entry regardless of map iteration order - CMAF-only publishers correctly return null (caller falls back to "unknown codec / no audio") - publishers that omit `container` entirely also return null — treating "no container declared" as "legacy by default" would silently mis-decode a CMAF-only publisher that just forgot to spell out `container.kind` LEGACY_KIND const lives on `Container.Companion` so call sites share one canonical spelling. Three new test cases pin the new filter behaviour (mixed iteration order, CMAF-only, missing container). The pre-existing `toleratesUnknownKeys` test is updated to declare a legacy container — the unknown-key tolerance we're checking is about unrecognised siblings, not container-presence. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../commons/viewmodels/RoomSpeakerCatalog.kt | 35 ++++- .../viewmodels/RoomSpeakerCatalogTest.kt | 66 +++++++- .../StripLegacyTimestampPrefixTest.kt | 145 ++++++++++++++++++ 3 files changed, 238 insertions(+), 8 deletions(-) create mode 100644 nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/StripLegacyTimestampPrefixTest.kt diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalog.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalog.kt index 9bbf04840..8b3add61f 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalog.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalog.kt @@ -106,15 +106,38 @@ data class RoomSpeakerCatalog( @Serializable data class Container( val kind: String? = null, - ) + ) { + companion object { + /** + * The only `container.kind` value the Amethyst decoder + * pipeline currently understands: each moq-lite frame is + * `varint(timestamp_us) + raw_codec_payload`. Source: + * `kixelated/moq/rs/hang/src/container/legacy.rs`. + * + * Other documented kinds — `"cmaf"` (MOOF/MDAT-fragmented + * MP4) and a handful of in-flight experimental shapes — + * would require a different decoder path and are + * intentionally rejected by [primaryAudio]. + */ + const val LEGACY_KIND: String = "legacy" + } + } /** - * First audio rendition, if any. The current single-Opus reality. - * Map iteration order is the JSON insertion order (kotlinx.serialization - * uses LinkedHashMap), so this picks the first rendition the - * publisher declared rather than an arbitrary one. + * First audio rendition with a [Container.LEGACY_KIND] container, + * which is the only container layout the Amethyst decoder pipeline + * understands today (`varint(timestamp_us) + opus_packet` per + * frame). Returns null when no legacy rendition exists — the UI + * surfaces "no audio info" rather than a CMAF rendition we'd try + * to feed to our legacy decoder. + * + * Picks the first match in JSON iteration order + * (kotlinx.serialization uses LinkedHashMap), so the publisher's + * preferred legacy rendition wins. A future publisher that emits + * CMAF-first then legacy still surfaces the legacy entry; CMAF- + * only publishers correctly return null. */ - fun primaryAudio(): AudioConfig? = audio?.renditions?.values?.firstOrNull() + fun primaryAudio(): AudioConfig? = audio?.renditions?.values?.firstOrNull { it.container?.kind == Container.LEGACY_KIND } companion object { /** diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalogTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalogTest.kt index 7286d83e8..a4941b64a 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalogTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/RoomSpeakerCatalogTest.kt @@ -90,9 +90,17 @@ class RoomSpeakerCatalogTest { @Test fun toleratesUnknownKeys() { // Forward-compat: future hang catalog revisions can add - // fields without breaking older clients. + // fields without breaking older clients. Container.kind is + // declared as `legacy` because `primaryAudio()` filters to + // that kind — the unknown-key tolerance we're testing here + // is about unrecognized siblings, not about the container + // requirement itself. val json = - """{"audio":{"renditions":{"audio/data":{"codec":"opus","extra":"future-only"}}},"video":{},"newTopLevel":true}""" + """{"audio":{"renditions":{"audio/data":{ + | "codec":"opus","container":{"kind":"legacy"}, + | "extra":"future-only" + |}}},"video":{},"newTopLevel":true} + """.trimMargin() val catalog = RoomSpeakerCatalog.parseOrNull(json.encodeToByteArray()) assertNotNull(catalog) assertEquals("opus", catalog.primaryAudio()?.codec) @@ -116,6 +124,60 @@ class RoomSpeakerCatalogTest { assertNull(catalog.primaryAudio()) } + @Test + fun primaryAudioPicksLegacyEvenWhenCmafComesFirst() { + // A future publisher may emit CMAF-first then legacy in the + // renditions map. We only know how to decode the legacy + // container today (varint(ts)+opus); CMAF (MOOF/MDAT) would + // be fed bytes-as-Opus and decode to garbage. The filter + // skips non-legacy renditions so the chosen entry is one we + // can actually play. + val mixed = + """{"audio":{"renditions":{ + | "video/cmaf":{"codec":"opus","container":{"kind":"cmaf"},"sampleRate":48000,"numberOfChannels":1}, + | "audio/data":{"codec":"opus","container":{"kind":"legacy"},"sampleRate":48000,"numberOfChannels":1} + |}}} + """.trimMargin() + val catalog = RoomSpeakerCatalog.parseOrNull(mixed.encodeToByteArray()) + assertNotNull(catalog) + val rendition = catalog.primaryAudio() + assertNotNull(rendition, "expected the legacy rendition, not CMAF") + assertEquals("legacy", rendition.container?.kind) + } + + @Test + fun primaryAudioReturnsNullForCmafOnlyPublisher() { + // No legacy rendition → no decoder path. Returning null is + // the right contract: the caller falls back to "unknown + // codec / no audio" rather than feeding CMAF bytes to the + // legacy decoder. + val cmafOnly = + """{"audio":{"renditions":{"audio/cmaf":{ + | "codec":"opus","container":{"kind":"cmaf"}, + | "sampleRate":48000,"numberOfChannels":1 + |}}}} + """.trimMargin() + val catalog = RoomSpeakerCatalog.parseOrNull(cmafOnly.encodeToByteArray()) + assertNotNull(catalog) + assertNull(catalog.primaryAudio()) + } + + @Test + fun primaryAudioReturnsNullWhenContainerKindIsMissing() { + // Defensive: a malformed publisher that omits `container` + // entirely must NOT be treated as legacy by default — we'd + // be guessing the wire shape. Same null fallback as the + // CMAF-only case. + val noContainer = + """{"audio":{"renditions":{"audio/data":{ + | "codec":"opus","sampleRate":48000,"numberOfChannels":1 + |}}}} + """.trimMargin() + val catalog = RoomSpeakerCatalog.parseOrNull(noContainer.encodeToByteArray()) + assertNotNull(catalog) + assertNull(catalog.primaryAudio()) + } + @Test fun missingAudioReturnsNullPrimary() { // hang catalogs declaring only video MUST still parse without diff --git a/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/StripLegacyTimestampPrefixTest.kt b/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/StripLegacyTimestampPrefixTest.kt new file mode 100644 index 000000000..62caecab0 --- /dev/null +++ b/nestsClient/src/commonTest/kotlin/com/vitorpamplona/nestsclient/StripLegacyTimestampPrefixTest.kt @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.nestsclient + +import com.vitorpamplona.quic.Varint +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertSame +import kotlin.test.assertTrue + +/** + * Unit tests for [MoqLiteNestsListener.Companion.stripLegacyTimestampPrefix]. + * + * This function is the entire receive-side wire-format converter for + * audio frames — every web speaker's Opus packet flows through it. A + * regression here is invisible to users (silent decode failure of every + * web speaker) and to interop tests (both sides agree on the same bug). + * Pin the four QUIC varint-length tiers, the malformed paths, and a + * round-trip against `NestMoqLiteBroadcaster`'s wire format. + * + * Varint length tags (RFC 9000 §16, top 2 bits of the first byte): + * 00 → 1 byte total (6-bit value, 0..63) + * 01 → 2 bytes total (14-bit, 0..16383) + * 10 → 4 bytes total (30-bit, 0..2^30-1) + * 11 → 8 bytes total (62-bit, 0..2^62-1) + */ +class StripLegacyTimestampPrefixTest { + @Test + fun strips_one_byte_varint_for_small_timestamp() { + // `Varint.encode(0L)` returns a single byte with tag 00. + val frame = Varint.encode(0L) + OPUS + val stripped = MoqLiteNestsListener.stripLegacyTimestampPrefix(frame) + assertContentEquals(OPUS, stripped) + } + + @Test + fun strips_one_byte_varint_at_max_6bit_value() { + // 63 (0x3F) is the largest value still encoded in 1 byte. + val frame = Varint.encode(63L) + OPUS + assertContentEquals(byteArrayOf(0x3F) + OPUS, frame, "varint(63) is a single 0x3F byte") + val stripped = MoqLiteNestsListener.stripLegacyTimestampPrefix(frame) + assertContentEquals(OPUS, stripped) + } + + @Test + fun strips_two_byte_varint() { + // 64 (just past the 1-byte boundary) → tag 01, 2 bytes. + val frame = Varint.encode(64L) + OPUS + val tag = (frame[0].toInt() ushr 6) and 0x3 + assertTrue(tag == 0x1, "varint(64) should use 2-byte form, got tag=$tag") + val stripped = MoqLiteNestsListener.stripLegacyTimestampPrefix(frame) + assertContentEquals(OPUS, stripped) + } + + @Test + fun strips_four_byte_varint() { + // ~4.2 million µs ≈ 4.2 s — comfortably past the 2-byte 16383 µs + // limit and well within any real broadcast lifetime. + val frame = Varint.encode(4_200_000L) + OPUS + val tag = (frame[0].toInt() ushr 6) and 0x3 + assertTrue(tag == 0x2, "varint(4_200_000) should use 4-byte form, got tag=$tag") + val stripped = MoqLiteNestsListener.stripLegacyTimestampPrefix(frame) + assertContentEquals(OPUS, stripped) + } + + @Test + fun strips_eight_byte_varint() { + // > 2^30 µs (~17.9 minutes) forces the 8-byte form. Models a + // broadcast that's been running long enough for the timestamp + // counter to overflow into the widest tier — perfectly valid + // Opus stream timestamps but not exercised by the smaller + // tiers. + val frame = Varint.encode(2_000_000_000L) + OPUS + val tag = (frame[0].toInt() ushr 6) and 0x3 + assertTrue(tag == 0x3, "varint(2_000_000_000) should use 8-byte form, got tag=$tag") + val stripped = MoqLiteNestsListener.stripLegacyTimestampPrefix(frame) + assertContentEquals(OPUS, stripped) + } + + @Test + fun empty_payload_returns_empty() { + val empty = ByteArray(0) + // Same instance — no allocation on the empty path. + assertSame(empty, MoqLiteNestsListener.stripLegacyTimestampPrefix(empty)) + } + + @Test + fun malformed_payload_smaller_than_varint_returns_payload_unchanged() { + // A frame body that's just the high tag byte of an 8-byte + // varint with no follow-up bytes is malformed. The current + // contract is "return the payload unchanged so upstream + // surfaces the corruption rather than silently masking it." + // This pins that contract — if it ever changes (e.g. to + // throw or return empty), this test fails and forces an + // explicit decision. + val malformed = byteArrayOf(0xC0.toByte()) // tag=11, expects 8 bytes total + val result = MoqLiteNestsListener.stripLegacyTimestampPrefix(malformed) + assertContentEquals(malformed, result, "malformed frame returned unchanged") + assertSame(malformed, result, "no allocation on the unchanged-malformed path") + } + + @Test + fun round_trip_against_broadcaster_wire_shape() { + // Mirror the byte sequence + // [com.vitorpamplona.nestsclient.audio.NestMoqLiteBroadcaster] writes: + // varint(timestamp_us) + raw_opus_packet + // Stripping MUST recover the exact opus packet byte-for-byte + // for every frame the watcher receives. Any drift here means + // every decoded frame loses or gains bytes at the front. + val timestamps = longArrayOf(0L, 20_000L, 40_000L, 1_000_000L, 2_000_000_000L) + for (ts in timestamps) { + val tsLen = Varint.size(ts) + val frame = ByteArray(tsLen + OPUS.size) + Varint.writeTo(ts, frame, 0) + OPUS.copyInto(frame, tsLen) + val stripped = MoqLiteNestsListener.stripLegacyTimestampPrefix(frame) + assertContentEquals(OPUS, stripped, "round-trip failed for timestamp $ts") + } + } + + private companion object { + // 8 bytes of plausible-looking Opus packet bytes — the function + // doesn't care about the codec content, only the length / + // content equality after the strip. + private val OPUS = byteArrayOf(0x78, 0x12, 0x34, 0x56, 0x78.toByte(), 0x9A.toByte(), 0xBC.toByte(), 0xDE.toByte()) + } +} From 8a49486b37424192c4049b36cf5ba371e28bcd0e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 19:36:05 +0000 Subject: [PATCH 36/41] fix(nests): G4+G5 plumb catalog sampleRate through decoder + AudioTrack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit G4: Audit-9 (catalog-driven decoder reconfig) plumbed numberOfChannels end-to-end but left sampleRate hardcoded at AudioFormat.SAMPLE_RATE_HZ in MediaCodecOpusDecoder.buildOpusIdHeader / buildFormat and in AudioTrackPlayer's MinBufferSize / 250 ms target / setSampleRate calls. For Opus this is benign in practice — Codec2 always emits 48 kHz PCM regardless of OpusHead inputSampleRate — but hardcoding the constant means a future codec or container variant whose decoder DOES respect input sample rate would mis-clock playback. And the OpusHead identification header should match what the catalog declares either way. Thread sampleRate alongside channelCount through every layer: - MediaCodecOpusDecoder constructor takes `sampleRate: Int = AudioFormat.SAMPLE_RATE_HZ`. buildFormat and buildOpusIdHeader both take it as a parameter. - AudioTrackPlayer constructor takes the same. Used in AudioTrack.getMinBufferSize, the 250 ms-equivalent target bytes calculation (`(sampleRate / 4) * BYTES_PER_SAMPLE * channelCount`), AndroidAudioFormat.Builder.setSampleRate, and the diagnostic log. - decoderFactory and playerFactory in NestViewModel become `(channelCount: Int, sampleRate: Int) -> ...`. - awaitDecoderChannelCount → awaitAudioPipelineConfig, returning a private `AudioPipelineConfig(channelCount, sampleRate)` struct so openSubscription handles both fields uniformly. `sampleRate` falls back to AudioFormat.SAMPLE_RATE_HZ on timeout / non-positive declaration with a warning log. - NestViewModelFactory + NestViewModelTest updated to the two-arg factory shape. G5: documented the SUBSCRIBE_BUFFER safety budget on CATALOG_AWAIT_TIMEOUT_MS's kdoc. With the current 500 ms timeout + SUBSCRIBE_BUFFER = 64-frame DROP_OLDEST flow + 50 fps Opus, at most 25 frames buffer during the wait — leaving ≥ 39 frames of margin (≈ 780 ms) before the oldest frame would be evicted. Even at the production framesPerGroup = 50 (1 group/sec) cadence this never trips during normal startup. No code change; just pinning the rationale so a future timeout bump is checked against the buffer size. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../room/lifecycle/NestViewModelFactory.kt | 8 +- .../commons/viewmodels/NestViewModel.kt | 136 ++++++++++++------ .../commons/viewmodels/NestViewModelTest.kt | 4 +- .../nestsclient/audio/AudioTrackPlayer.kt | 31 +++- .../audio/MediaCodecOpusDecoder.kt | 37 ++++- 5 files changed, 157 insertions(+), 59 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/nests/room/lifecycle/NestViewModelFactory.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/nests/room/lifecycle/NestViewModelFactory.kt index 69c30fe27..b1389a5c7 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/nests/room/lifecycle/NestViewModelFactory.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/nests/room/lifecycle/NestViewModelFactory.kt @@ -53,8 +53,12 @@ internal class NestViewModelFactory( // `httpClient` slot rather than as the first positional arg. httpClient = OkHttpNestsClient(httpClient = Amethyst.instance.roleBasedHttpClientBuilder::okHttpClientForVideo), transport = QuicWebTransportFactory(), - decoderFactory = { channelCount -> MediaCodecOpusDecoder(channelCount = channelCount) }, - playerFactory = { channelCount -> AudioTrackPlayer(channelCount = channelCount) }, + decoderFactory = { channelCount, sampleRate -> + MediaCodecOpusDecoder(channelCount = channelCount, sampleRate = sampleRate) + }, + playerFactory = { channelCount, sampleRate -> + AudioTrackPlayer(channelCount = channelCount, sampleRate = sampleRate) + }, signer = signer, room = room, captureFactory = { AudioRecordCapture() }, diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt index 942765d62..c688ed863 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt @@ -87,10 +87,11 @@ import kotlin.coroutines.cancellation.CancellationException * [playerFactory] so commonMain doesn't have to know which platform's * MediaCodec / AudioTrack is in play. M1 wires Android-only — desktop * passes nothing here yet. Both factories take a `channelCount` (1 for - * mono, 2 for stereo) discovered from the publisher's `catalog.json` - * audio rendition; subscriptions await a brief catalog-arrival window - * before constructing the decoder + player so a stereo web publisher - * doesn't get its frames decoded as mono with downmix artifacts. + * mono, 2 for stereo) AND a `sampleRate` (Hz) discovered from the + * publisher's `catalog.json` audio rendition; subscriptions await a + * brief catalog-arrival window before constructing the decoder + player + * so a stereo or non-48 kHz web publisher doesn't get its frames + * decoded with the wrong channel layout / clock. * * **Threading contract:** all public methods (`connect`, `disconnect`, * `updateSpeakers`, `setMuted`, `setMicMuted`, `startBroadcast`, @@ -107,8 +108,8 @@ import kotlin.coroutines.cancellation.CancellationException class NestViewModel( private val httpClient: NestsClient, private val transport: WebTransportFactory, - private val decoderFactory: (channelCount: Int) -> OpusDecoder, - private val playerFactory: (channelCount: Int) -> AudioPlayer, + private val decoderFactory: (channelCount: Int, sampleRate: Int) -> OpusDecoder, + private val playerFactory: (channelCount: Int, sampleRate: Int) -> AudioPlayer, private val signer: NostrSigner, private val room: NestsRoomConfig, // Speaker-side audio capture/encode actuals. Optional — desktop and @@ -1049,12 +1050,15 @@ class NestViewModel( /** * Wait briefly for [pubkey]'s catalog to land in [_speakerCatalogs] - * and pick the channel count for the decoder + AudioTrack. Returns - * [AudioFormat.CHANNELS] on timeout (the catalog never arrived - * within [timeoutMs]) or when the catalog declares an unsupported - * count (anything outside `1..2`). Caller is responsible for - * having started [fetchSpeakerCatalog] first; otherwise this - * always times out. + * and pick the audio config (channel count + sample rate) for the + * decoder + AudioTrack. Falls back to [AudioFormat.CHANNELS] / + * [AudioFormat.SAMPLE_RATE_HZ] on timeout (the catalog never + * arrived within [timeoutMs]) or when the catalog declares + * unsupported values (channelCount outside `1..2`, or non-positive + * sampleRate). + * + * Caller is responsible for having started [fetchSpeakerCatalog] + * first; otherwise this always times out. * * Why a timeout: the catalog handshake is best-effort. If the * publisher doesn't publish a catalog (legacy publishers) or the @@ -1065,10 +1069,10 @@ class NestViewModel( * within one round-trip of the SUBSCRIBE_OK, so this typically * returns within tens of ms. */ - private suspend fun awaitDecoderChannelCount( + private suspend fun awaitAudioPipelineConfig( pubkey: String, timeoutMs: Long, - ): Int { + ): AudioPipelineConfig { val catalog = withTimeoutOrNull(timeoutMs) { _speakerCatalogs @@ -1076,26 +1080,60 @@ class NestViewModel( .filterNotNull() .first() } - val declared = catalog?.primaryAudio()?.numberOfChannels - return when { - declared == null -> { - AudioFormat.CHANNELS - } - - declared !in 1..2 -> { - Log.w("NestRx") { - "publisher catalog for pubkey='${pubkey.take(8)}' declares numberOfChannels=$declared " + - "(only 1 / 2 supported); falling back to mono" + val rendition = catalog?.primaryAudio() + val declaredChannels = rendition?.numberOfChannels + val channels = + when { + declaredChannels == null -> { + AudioFormat.CHANNELS } - AudioFormat.CHANNELS - } - else -> { - declared + declaredChannels !in 1..2 -> { + Log.w("NestRx") { + "publisher catalog for pubkey='${pubkey.take(8)}' declares numberOfChannels=$declaredChannels " + + "(only 1 / 2 supported); falling back to mono" + } + AudioFormat.CHANNELS + } + + else -> { + declaredChannels + } } - } + val declaredRate = rendition?.sampleRate + val rate = + when { + declaredRate == null -> { + AudioFormat.SAMPLE_RATE_HZ + } + + declaredRate <= 0 -> { + Log.w("NestRx") { + "publisher catalog for pubkey='${pubkey.take(8)}' declares sampleRate=$declaredRate " + + "(must be positive); falling back to ${AudioFormat.SAMPLE_RATE_HZ} Hz" + } + AudioFormat.SAMPLE_RATE_HZ + } + + else -> { + declaredRate + } + } + return AudioPipelineConfig(channelCount = channels, sampleRate = rate) } + /** + * Output of [awaitAudioPipelineConfig] — the validated audio + * pipeline configuration for one subscription. Internal because the + * factories' two-arg shape (`(channelCount, sampleRate)`) is what + * the public surface deals in; this struct just bundles them inside + * `openSubscription`. + */ + private data class AudioPipelineConfig( + val channelCount: Int, + val sampleRate: Int, + ) + /** * Open the speaker's `catalog.json` track in the background, parse * the first frame, and stash it in [speakerCatalogs]. Best-effort — @@ -1171,25 +1209,28 @@ class NestViewModel( // [CATALOG_AWAIT_TIMEOUT_MS] — covers legacy publishers and // the failure-mode where the relay never forwards the catalog // group; audio still plays, just in the default config. - val channelCount = awaitDecoderChannelCount(pubkey, CATALOG_AWAIT_TIMEOUT_MS) + val audioCfg = awaitAudioPipelineConfig(pubkey, CATALOG_AWAIT_TIMEOUT_MS) // Allocate native resources (MediaCodec decoder + AudioTrack // player on Android). Both are heavy and leaky if dropped on // the floor — wrap them in a nested try so any cancellation // or throw between here and slot.attach releases them // (audit round-2 VM #7). // Per-subscription decoder factory closure: captures the - // catalog-derived [channelCount] so a publisher-boundary + // catalog-derived [audioCfg] so a publisher-boundary // decoder rebuild (see [NestPlayer]'s `decoderFactory` - // kdoc) reuses the SAME channel layout — without it, a - // rebuild after a cliff-recycle would default to mono and - // a stereo publisher would silently downmix on the new - // decoder. The factory is also called for the initial - // decoder so the construction path is uniform. - val perSubscriptionDecoderFactory: () -> OpusDecoder = { decoderFactory(channelCount) } + // kdoc) reuses the SAME channel layout AND sample rate — + // without it, a rebuild after a cliff-recycle would + // default to mono / 48 kHz and a stereo or non-48 kHz + // publisher would silently downmix / clock-mismatch on + // the new decoder. The factory is also called for the + // initial decoder so the construction path is uniform. + val perSubscriptionDecoderFactory: () -> OpusDecoder = { + decoderFactory(audioCfg.channelCount, audioCfg.sampleRate) + } val decoder = perSubscriptionDecoderFactory() val player = try { - playerFactory(channelCount) + playerFactory(audioCfg.channelCount, audioCfg.sampleRate) } catch (t: Throwable) { runCatching { decoder.release() } throw t @@ -1808,16 +1849,27 @@ const val SPEAKING_TIMEOUT_MS: Long = 250L /** * How long [NestViewModel.openSubscription] waits for the publisher's * `catalog.json` to land before constructing the decoder + AudioTrack. - * The catalog declares the audio rendition's `numberOfChannels`, which - * the decoder needs at construction time so a stereo web publisher - * doesn't get its frames decoded as mono with downmix artifacts. + * The catalog declares the audio rendition's `numberOfChannels` and + * `sampleRate`, which the decoder needs at construction time so a + * stereo or non-48 kHz web publisher doesn't get its frames decoded + * with the wrong layout / clock. * * Sized to be generous enough that a typical publisher's catalog * (which arrives within a single round-trip of the SUBSCRIBE_OK with * the speaker-side emit-on-subscribe hook in place) lands well before * the timeout, and short enough that a publisher that never emits a * catalog (legacy publishers, relay-side bug) doesn't visibly stall - * the listener — audio playback proceeds in the default mono config. + * the listener — audio playback proceeds in the default config. + * + * **Frame-buffer budget**: while we wait, audio frames stream from + * the relay into the wrapper's + * [com.vitorpamplona.nestsclient.ReconnectingNestsListener] + * `SUBSCRIBE_BUFFER = 64` SharedFlow with `DROP_OLDEST` overflow. + * At 50 fps Opus that's 1.28 s of headroom; the 500 ms wait + * consumes at most 25 frames of buffer, leaving ≥ 39 frames of + * margin (≈780 ms) before the oldest frame would be dropped. Even + * at the production `framesPerGroup = 50` (= 1 group / sec) cadence + * this never trips the eviction path during normal startup. */ const val CATALOG_AWAIT_TIMEOUT_MS: Long = 500L diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModelTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModelTest.kt index d9c5d7d76..5c7415ab5 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModelTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModelTest.kt @@ -455,8 +455,8 @@ class NestViewModelTest { NestViewModel( httpClient = NoopNestsClient, transport = NoopWebTransportFactory, - decoderFactory = { _ -> NoopOpusDecoder }, - playerFactory = { _ -> NoopAudioPlayer() }, + decoderFactory = { _, _ -> NoopOpusDecoder }, + playerFactory = { _, _ -> NoopAudioPlayer() }, signer = NoopSigner, room = ROOM_CONFIG, connector = diff --git a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/AudioTrackPlayer.kt b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/AudioTrackPlayer.kt index 0d6901e04..27871b739 100644 --- a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/AudioTrackPlayer.kt +++ b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/AudioTrackPlayer.kt @@ -86,11 +86,26 @@ class AudioTrackPlayer( * call sites that don't pass a channel count keep the prior behaviour. */ private val channelCount: Int = AudioFormat.CHANNELS, + /** + * PCM sample rate in Hz. Drives the AudioTrack output rate and the + * 250 ms target-buffer calculation. For Opus, Android's Codec2 + * decoder always emits 48 kHz PCM regardless of the OpusHead + * `inputSampleRate`, so for the production Opus path this is + * always [AudioFormat.SAMPLE_RATE_HZ] — but threading the + * parameter through keeps the AudioTrack's declared rate matched + * to whatever the catalog says, and lets a future codec or + * container variant whose decoder DOES respect input sample rate + * (a non-Opus rendition) get the correct PCM clock. + */ + private val sampleRate: Int = AudioFormat.SAMPLE_RATE_HZ, ) : AudioPlayer { init { require(channelCount in 1..2) { "AudioTrackPlayer supports mono (1) or stereo (2) only, got $channelCount" } + require(sampleRate > 0) { + "AudioTrackPlayer sampleRate must be positive, got $sampleRate" + } } private var track: AudioTrack? = null @@ -123,14 +138,14 @@ class AudioTrackPlayer( val minBuffer = AudioTrack.getMinBufferSize( - AudioFormat.SAMPLE_RATE_HZ, + sampleRate, channelMask, AndroidAudioFormat.ENCODING_PCM_16BIT, ) if (minBuffer <= 0) { throw AudioException( AudioException.Kind.DeviceUnavailable, - "AudioTrack.getMinBufferSize returned $minBuffer for ${AudioFormat.SAMPLE_RATE_HZ} Hz", + "AudioTrack.getMinBufferSize returned $minBuffer for $sampleRate Hz", ) } // Target ~250 ms of audio: enough headroom so the decode loop can @@ -138,10 +153,12 @@ class AudioTrackPlayer( // underruns. Take the larger of `minBuffer * 16` and an explicit // 250 ms-equivalent so devices that report a small minBuffer still // get the same wall-clock slack. Stereo doubles the byte count - // per sample (interleaved L,R 16-bit shorts) — scaling - // [channelCount] in keeps the wall-clock target constant. + // per sample (interleaved L,R 16-bit shorts); a higher sample + // rate scales the per-second byte count linearly — both factor + // into the target so the wall-clock 250 ms target is preserved + // regardless of channel layout / sample rate. val targetBytes250Ms = - (AudioFormat.SAMPLE_RATE_HZ / 4) * AudioFormat.BYTES_PER_SAMPLE * channelCount + (sampleRate / 4) * AudioFormat.BYTES_PER_SAMPLE * channelCount val bufferBytes = maxOf(minBuffer * 16, targetBytes250Ms) val newTrack = @@ -158,7 +175,7 @@ class AudioTrackPlayer( AndroidAudioFormat .Builder() .setEncoding(AndroidAudioFormat.ENCODING_PCM_16BIT) - .setSampleRate(AudioFormat.SAMPLE_RATE_HZ) + .setSampleRate(sampleRate) .setChannelMask(channelMask) .build(), ).setBufferSizeInBytes(bufferBytes) @@ -199,7 +216,7 @@ class AudioTrackPlayer( track = newTrack com.vitorpamplona.quartz.utils.Log.d("NestPlay") { "AudioTrack allocated: state=${newTrack.state} playState=${newTrack.playState} " + - "bufferSizeBytes=$bufferBytes minBuffer=$minBuffer sampleRate=${AudioFormat.SAMPLE_RATE_HZ}" + "bufferSizeBytes=$bufferBytes minBuffer=$minBuffer sampleRate=$sampleRate" } } diff --git a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusDecoder.kt b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusDecoder.kt index b2d93e7b7..ec9077f31 100644 --- a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusDecoder.kt +++ b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusDecoder.kt @@ -50,11 +50,30 @@ import java.nio.ByteOrder */ class MediaCodecOpusDecoder( private val channelCount: Int = AudioFormat.CHANNELS, + /** + * Source sample rate in Hz. Drives the OpusHead `inputSampleRate` + * field and the MediaFormat `audio/opus` sample-rate hint. + * + * **Note on Opus's actual decode rate**: Opus always decodes at + * 48 kHz internally regardless of [sampleRate] (the codec's + * design — RFC 6716). Android's Codec2 `audio/opus` decoder + * always emits 48 kHz PCM. So this parameter is informational at + * the OpusHead level and doesn't affect the PCM rate the + * downstream [AudioPlayer] sees. We thread it through anyway so + * the decoder declaration matches what the catalog says, and so a + * future codec or container variant that DOES respect input + * sample rate (e.g. a non-Opus rendition) gets the correct + * configuration. + */ + private val sampleRate: Int = AudioFormat.SAMPLE_RATE_HZ, ) : OpusDecoder { init { require(channelCount in 1..2) { "MediaCodecOpusDecoder supports mono (1) or stereo (2) only, got $channelCount" } + require(sampleRate > 0) { + "MediaCodecOpusDecoder sampleRate must be positive, got $sampleRate" + } } private val codec: MediaCodec = @@ -62,7 +81,7 @@ class MediaCodecOpusDecoder( MediaCodec .createDecoderByType(MediaFormat.MIMETYPE_AUDIO_OPUS) .apply { - configure(buildFormat(channelCount), null, null, 0) + configure(buildFormat(channelCount, sampleRate), null, null, 0) start() }.also { com.vitorpamplona.quartz.utils.Log.d("NestPlay") { @@ -221,14 +240,17 @@ class MediaCodecOpusDecoder( private const val FRAME_DURATION_US = 20_000L // 20 ms - private fun buildFormat(channelCount: Int): MediaFormat { + private fun buildFormat( + channelCount: Int, + sampleRate: Int, + ): MediaFormat { val format = MediaFormat.createAudioFormat( MediaFormat.MIMETYPE_AUDIO_OPUS, - AudioFormat.SAMPLE_RATE_HZ, + sampleRate, channelCount, ) - format.setByteBuffer("csd-0", ByteBuffer.wrap(buildOpusIdHeader(channelCount))) + format.setByteBuffer("csd-0", ByteBuffer.wrap(buildOpusIdHeader(channelCount, sampleRate))) // Pre-skip + seek pre-roll: both zero, encoded as little-endian // 64-bit nanoseconds per Android's MediaCodec contract. format.setByteBuffer("csd-1", ByteBuffer.wrap(zeroLongLe())) @@ -236,7 +258,10 @@ class MediaCodecOpusDecoder( return format } - private fun buildOpusIdHeader(channelCount: Int): ByteArray { + private fun buildOpusIdHeader( + channelCount: Int, + sampleRate: Int, + ): ByteArray { // RFC 7845 §5.1 — 19 bytes, mapping family 0 (covers mono and // stereo with implicit L,R interleaving; no per-channel // mapping table required). @@ -245,7 +270,7 @@ class MediaCodecOpusDecoder( buf.put(1.toByte()) // version buf.put(channelCount.toByte()) // channel count (1 or 2) buf.putShort(0) // pre-skip - buf.putInt(AudioFormat.SAMPLE_RATE_HZ) // input sample rate + buf.putInt(sampleRate) // input sample rate buf.putShort(0) // output gain (Q7.8 dB) buf.put(0.toByte()) // mapping family 0 return buf.array() From a96ec9db00408eca77d25b5f341940e5fbaf40b3 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 19:38:19 +0000 Subject: [PATCH 37/41] chore(nests): standardise Log import in audio-pipeline files Audit-12 cleanup. The audio-pipeline sources mixed two import styles: some files used `com.vitorpamplona.quartz.utils.Log.d/w/e` fully- qualified (NestPlayer's 13 sites, AudioTrackPlayer's 8, two each in the encoder/decoder, three in NestMoqLiteBroadcaster) while AudioRecordCapture had already migrated to a top-level `import com.vitorpamplona.quartz.utils.Log`. The lambda overload (`Log.d(tag) { lazyMessage }`) is what the find-non-lambda-logs skill enforces and is the only style the rest of the codebase uses; the fully-qualified form is verbose noise on every call site. Add the `import com.vitorpamplona.quartz.utils.Log` line to each file and rewrite call sites to bare `Log.d` / `Log.w` / `Log.e`. No behaviour change; logs still go through PlatformLog with the same tag and lazy-message contract. Five files updated, ~30 call sites. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../nestsclient/audio/AudioTrackPlayer.kt | 17 ++++++------ .../audio/MediaCodecOpusDecoder.kt | 5 ++-- .../audio/MediaCodecOpusEncoder.kt | 5 ++-- .../audio/NestMoqLiteBroadcaster.kt | 7 ++--- .../nestsclient/audio/NestPlayer.kt | 27 ++++++++++--------- 5 files changed, 33 insertions(+), 28 deletions(-) diff --git a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/AudioTrackPlayer.kt b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/AudioTrackPlayer.kt index 27871b739..0cba8be7f 100644 --- a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/AudioTrackPlayer.kt +++ b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/AudioTrackPlayer.kt @@ -23,6 +23,7 @@ package com.vitorpamplona.nestsclient.audio import android.media.AudioAttributes import android.media.AudioTrack import android.os.Process +import com.vitorpamplona.quartz.utils.Log import kotlinx.coroutines.ExecutorCoroutineDispatcher import kotlinx.coroutines.asCoroutineDispatcher import kotlinx.coroutines.withContext @@ -126,7 +127,7 @@ class AudioTrackPlayer( override fun start() { if (track != null) return - com.vitorpamplona.quartz.utils.Log + Log .d("NestPlay") { "AudioTrackPlayer.start() — allocating AudioTrack" } val channelMask = @@ -214,7 +215,7 @@ class AudioTrackPlayer( audioExecutor = executor audioDispatcher = executor.asCoroutineDispatcher() track = newTrack - com.vitorpamplona.quartz.utils.Log.d("NestPlay") { + Log.d("NestPlay") { "AudioTrack allocated: state=${newTrack.state} playState=${newTrack.playState} " + "bufferSizeBytes=$bufferBytes minBuffer=$minBuffer sampleRate=$sampleRate" } @@ -229,7 +230,7 @@ class AudioTrackPlayer( val t = track ?: return try { t.play() - com.vitorpamplona.quartz.utils.Log.d("NestPlay") { + Log.d("NestPlay") { "AudioTrack.play() returned, state=${t.playState} bufferSize=${t.bufferSizeInFrames} muted=$muted volume=$volume" } } catch (e: Throwable) { @@ -238,7 +239,7 @@ class AudioTrackPlayer( // track above. Throw so [NestPlayer]'s outer catch surfaces // it via `onError(AudioException.PlaybackFailed)` — same path // that handles every other mid-stream device failure. - com.vitorpamplona.quartz.utils.Log.w("NestPlay") { + Log.w("NestPlay") { "AudioTrack.play() threw: ${e::class.simpleName}: ${e.message}" } throw AudioException( @@ -261,7 +262,7 @@ class AudioTrackPlayer( withContext(dispatcher) { val written = t.write(pcm, 0, pcm.size, AudioTrack.WRITE_BLOCKING) if (written < 0) { - com.vitorpamplona.quartz.utils.Log.w("NestPlay") { + Log.w("NestPlay") { "AudioTrack.write returned error $written (state=${t.playState})" } throw AudioException( @@ -270,7 +271,7 @@ class AudioTrackPlayer( ) } if (written != pcm.size) { - com.vitorpamplona.quartz.utils.Log.w("NestPlay") { + Log.w("NestPlay") { "AudioTrack.write partial: requested=${pcm.size} written=$written" } } @@ -279,14 +280,14 @@ class AudioTrackPlayer( override fun setMuted(muted: Boolean) { this.muted = muted - com.vitorpamplona.quartz.utils.Log + Log .d("NestPlay") { "AudioTrackPlayer.setMuted($muted) volume=$volume" } track?.let { applyMuteVolume(it) } } override fun setVolume(volume: Float) { this.volume = volume.coerceIn(0f, 1f) - com.vitorpamplona.quartz.utils.Log + Log .d("NestPlay") { "AudioTrackPlayer.setVolume($volume) muted=$muted" } track?.let { applyMuteVolume(it) } } diff --git a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusDecoder.kt b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusDecoder.kt index ec9077f31..d9abb819f 100644 --- a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusDecoder.kt +++ b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusDecoder.kt @@ -22,6 +22,7 @@ package com.vitorpamplona.nestsclient.audio import android.media.MediaCodec import android.media.MediaFormat +import com.vitorpamplona.quartz.utils.Log import java.nio.ByteBuffer import java.nio.ByteOrder @@ -84,12 +85,12 @@ class MediaCodecOpusDecoder( configure(buildFormat(channelCount, sampleRate), null, null, 0) start() }.also { - com.vitorpamplona.quartz.utils.Log.d("NestPlay") { + Log.d("NestPlay") { "MediaCodecOpusDecoder allocated codec='${it.name}' channelCount=$channelCount" } } } catch (t: Throwable) { - com.vitorpamplona.quartz.utils.Log.w("NestPlay") { + Log.w("NestPlay") { "MediaCodec audio/opus decoder allocation FAILED: ${t::class.simpleName}: ${t.message}" } throw AudioException( diff --git a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusEncoder.kt b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusEncoder.kt index 038ee293a..f18f7da99 100644 --- a/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusEncoder.kt +++ b/nestsClient/src/androidMain/kotlin/com/vitorpamplona/nestsclient/audio/MediaCodecOpusEncoder.kt @@ -22,6 +22,7 @@ package com.vitorpamplona.nestsclient.audio import android.media.MediaCodec import android.media.MediaFormat +import com.vitorpamplona.quartz.utils.Log import java.nio.ByteOrder /** @@ -122,7 +123,7 @@ class MediaCodecOpusEncoder( codec.releaseOutputBuffer(outputIndex, false) if (!loggedCsdSkip) { loggedCsdSkip = true - com.vitorpamplona.quartz.utils.Log.d("NestTx") { + Log.d("NestTx") { "MediaCodecOpusEncoder skipped ${bufferInfo.size}-byte CODEC_CONFIG (OpusHead/OpusTags) — not an audio frame" } } @@ -135,7 +136,7 @@ class MediaCodecOpusEncoder( // contract — broadcaster's // `if (opus.isEmpty()) continue` handles it // and the next encode call retries. - com.vitorpamplona.quartz.utils.Log.w("NestTx") { + Log.w("NestTx") { "MediaCodecOpusEncoder hit MAX_CSD_SKIPS_PER_CALL=$MAX_CSD_SKIPS_PER_CALL; bailing this encode call (encoder may be misbehaving)" } return ByteArray(0) diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt index 4a8144d0b..208ef3a6a 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestMoqLiteBroadcaster.kt @@ -21,6 +21,7 @@ package com.vitorpamplona.nestsclient.audio import com.vitorpamplona.nestsclient.moq.lite.MoqLitePublisherHandle +import com.vitorpamplona.quartz.utils.Log import com.vitorpamplona.quic.Varint import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope @@ -356,7 +357,7 @@ class NestMoqLiteBroadcaster( if (accepted) { sentFrames += 1 if (sentFrames % SEND_LOG_THROTTLE == 0L) { - com.vitorpamplona.quartz.utils.Log.d("NestTx") { + Log.d("NestTx") { "broadcaster sent frame #$sentFrames (group $framesInCurrentGroup/$framesPerGroup)" } } @@ -371,7 +372,7 @@ class NestMoqLiteBroadcaster( } else { droppedNoSubFrames += 1 if (droppedNoSubFrames % SEND_LOG_THROTTLE == 0L) { - com.vitorpamplona.quartz.utils.Log.w("NestTx") { + Log.w("NestTx") { "broadcaster send returned false — frame dropped (count=$droppedNoSubFrames, sent=$sentFrames)" } } @@ -379,7 +380,7 @@ class NestMoqLiteBroadcaster( }.onFailure { t -> if (t is CancellationException) throw t consecutiveSendErrors += 1 - com.vitorpamplona.quartz.utils.Log.w("NestTx") { + Log.w("NestTx") { "broadcaster send threw (consecutive=$consecutiveSendErrors): ${t::class.simpleName}: ${t.message}" } onError( diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayer.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayer.kt index 9c25d0671..84b33f67e 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayer.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/audio/NestPlayer.kt @@ -21,6 +21,7 @@ package com.vitorpamplona.nestsclient.audio import com.vitorpamplona.nestsclient.moq.MoqObject +import com.vitorpamplona.quartz.utils.Log import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job @@ -161,7 +162,7 @@ class NestPlayer( var decodedFrames: Long = 0L var emptyDecodes: Long = 0L var enqueued: Long = 0L - com.vitorpamplona.quartz.utils.Log.d("NestPlay") { + Log.d("NestPlay") { "NestPlayer.play started (prerollFrames=$prerollFrames)" } @@ -179,7 +180,7 @@ class NestPlayer( // hardware starts pulling samples; getting // [enqueue] in first means the very first sample // pulled is from our pre-rolled audio, not silence. - com.vitorpamplona.quartz.utils.Log.d("NestPlay") { + Log.d("NestPlay") { "NestPlayer flushing preroll (${preroll.size} frames) → beginPlayback" } while (preroll.isNotEmpty()) { @@ -187,7 +188,7 @@ class NestPlayer( } player.beginPlayback() playbackBegun = true - com.vitorpamplona.quartz.utils.Log + Log .d("NestPlay") { "NestPlayer beginPlayback returned" } } // Track-alias of the most recently observed object. @@ -201,7 +202,7 @@ class NestPlayer( objects.collect { obj -> receivedObjects += 1 if (receivedObjects % PLAY_LOG_THROTTLE == 0L) { - com.vitorpamplona.quartz.utils.Log.d("NestPlay") { + Log.d("NestPlay") { "NestPlayer received obj #$receivedObjects (decoded=$decodedFrames empty=$emptyDecodes enqueued=$enqueued playbackBegun=$playbackBegun)" } } @@ -233,14 +234,14 @@ class NestPlayer( runCatching { factory() } .onFailure { t -> if (t is CancellationException) throw t - com.vitorpamplona.quartz.utils.Log.w("NestPlay") { + Log.w("NestPlay") { "NestPlayer decoder factory threw on trackAlias " + "$lastTrackAlias → ${obj.trackAlias}; keeping old decoder " + "(${t::class.simpleName}: ${t.message})" } }.getOrNull() if (replacement != null) { - com.vitorpamplona.quartz.utils.Log.d("NestPlay") { + Log.d("NestPlay") { "NestPlayer publisher boundary: trackAlias $lastTrackAlias → ${obj.trackAlias}; rebuilding decoder" } runCatching { decoder.release() } @@ -254,7 +255,7 @@ class NestPlayer( } catch (ce: CancellationException) { throw ce } catch (t: Throwable) { - com.vitorpamplona.quartz.utils.Log.w("NestPlay") { + Log.w("NestPlay") { "decoder.decode threw on obj #$receivedObjects: ${t::class.simpleName}: ${t.message}" } onError( @@ -269,7 +270,7 @@ class NestPlayer( if (pcm.isEmpty()) { emptyDecodes += 1 if (emptyDecodes % PLAY_LOG_THROTTLE == 0L) { - com.vitorpamplona.quartz.utils.Log.w("NestPlay") { + Log.w("NestPlay") { "decoder returned empty pcm (count=$emptyDecodes / received=$receivedObjects)" } } @@ -282,7 +283,7 @@ class NestPlayer( val enqueueMs = System.currentTimeMillis() - enqueueStart enqueued += 1 if (enqueued % PLAY_LOG_THROTTLE == 0L || enqueueMs > 50) { - com.vitorpamplona.quartz.utils.Log.d("NestPlay") { + Log.d("NestPlay") { "NestPlayer enqueued #$enqueued (took ${enqueueMs}ms)" } } @@ -294,7 +295,7 @@ class NestPlayer( } } } - com.vitorpamplona.quartz.utils.Log.w("NestPlay") { + Log.w("NestPlay") { "NestPlayer objects flow COMPLETED (received=$receivedObjects decoded=$decodedFrames empty=$emptyDecodes enqueued=$enqueued)" } // Flow ended without enough frames to fill the pre-roll @@ -303,12 +304,12 @@ class NestPlayer( // already-decoded audio still reaches the device. beginAndFlushIfNeeded() } catch (ce: CancellationException) { - com.vitorpamplona.quartz.utils.Log.d("NestPlay") { + Log.d("NestPlay") { "NestPlayer cancelled (received=$receivedObjects decoded=$decodedFrames enqueued=$enqueued)" } throw ce } catch (t: Throwable) { - com.vitorpamplona.quartz.utils.Log.w("NestPlay") { + Log.w("NestPlay") { "NestPlayer pipeline threw: ${t::class.simpleName}: ${t.message}" } onError( @@ -334,7 +335,7 @@ class NestPlayer( suspend fun stop() { if (stopped) return stopped = true - com.vitorpamplona.quartz.utils.Log + Log .d("NestPlay") { "NestPlayer.stop()" } job?.cancelAndJoin() runCatching { player.stop() } From fb7b6b7cd92304fe59eb3545fd52011800021640 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 19:43:45 +0000 Subject: [PATCH 38/41] refactor(nests): split MoqLiteMessages.kt by concern (Audit-11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original `MoqLiteMessages.kt` mixed three categories of declaration: - One `MoqLiteAlpn` object — the WebTransport sub-protocol advertisement strings. - Four enums — `MoqLiteControlType`, `MoqLiteDataType`, `MoqLiteAnnounceStatus`, `MoqLiteSubscribeResponseType` — that are wire-format discriminators the codec reads at message boundaries to choose which body to decode. - Eight data classes / objects — `MoqLiteAnnouncePlease`, `MoqLiteAnnounce`, `MoqLiteSubscribe`, `MoqLiteSubscribeOk`, `MoqLiteSubscribeDrop`, `MoqLiteSubscribeDropCode`, `MoqLiteGroupHeader`, `MoqLiteProbe` — the body shapes themselves. 317 lines, three concerns, one file. Split by responsibility: - `MoqLiteAlpn.kt` — the ALPN object alone. - `MoqLiteControlCodes.kt` — the four discriminator enums. - `MoqLiteMessages.kt` — body data classes only. Same package, same visibility, identical wire behaviour. Imports across the codebase resolve to the new locations automatically since everything stayed in `com.vitorpamplona.nestsclient.moq.lite`. Tests pass unchanged. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../nestsclient/moq/lite/MoqLiteAlpn.kt | 58 +++++++ .../moq/lite/MoqLiteControlCodes.kt | 135 ++++++++++++++++ .../nestsclient/moq/lite/MoqLiteMessages.kt | 148 +----------------- 3 files changed, 200 insertions(+), 141 deletions(-) create mode 100644 nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteAlpn.kt create mode 100644 nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteControlCodes.kt diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteAlpn.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteAlpn.kt new file mode 100644 index 000000000..2c3186ef6 --- /dev/null +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteAlpn.kt @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.nestsclient.moq.lite + +/** + * moq-lite ALPN strings. ONLY [LITE_03] is wire-compatible with the + * codec in [MoqLiteCodec]; the other constants are kept for + * documentation and future codec upgrades. + * + * Subscribe / Group framing is identical across Lite-03 and Lite-04, + * but Lite-04 reshapes Announce.hops into an `OriginList` + * (`kixelated/moq` commit 45db108, "moq-lite/moq-relay: hop-based + * clustering"), adds an `exclude_hop` field to AnnounceInterest, and + * adds an `rtt` field to Probe. A relay that picks Lite-04 with our + * Lite-03 codec on the wire desyncs on the first Announce exchange. + * Don't add [LITE_04] to `wt-available-protocols` until + * [MoqLiteCodec] is version-aware and [MoqLiteAnnounce.hops] is a + * list rather than a single varint. + * + * `"moql"` is the legacy combined ALPN that requires an in-band SETUP + * exchange — kept here for completeness; not advertised by the + * factory. + * + * Source: `kixelated/moq/rs/moq-lite/src/lite/version.rs`, + * `kixelated/moq/rs/moq-lite/src/lite/announce.rs:11-105`, + * `kixelated/moq/rs/moq-lite/src/lite/probe.rs:9-55`. + */ +object MoqLiteAlpn { + const val LITE_03: String = "moq-lite-03" + + /** + * `moq-lite-04` ALPN string. Wire-incompatible with [MoqLiteCodec] + * today — see the object kdoc for the codec diff. Defined here so + * a future patch that lands version-aware Announce / Probe codecs + * can drop it into the [QuicWebTransportFactory] sub-protocol list + * without re-deriving the constant. + */ + const val LITE_04: String = "moq-lite-04" + const val LEGACY: String = "moql" +} diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteControlCodes.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteControlCodes.kt new file mode 100644 index 000000000..2d55a2c81 --- /dev/null +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteControlCodes.kt @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.nestsclient.moq.lite + +// Wire-format discriminators for moq-lite control flow. +// +// The four enums below share a common job: they're varint / byte tags +// the wire-format parser (`MoqLiteCodec`) reads at message boundaries to +// decide which body to decode next. None of them carry payload data +// themselves; the data classes that DO live in `MoqLiteMessages.kt`. +// +// Source: `kixelated/moq/rs/moq-lite/src/lite/{stream,announce,subscribe}.rs`. + +/** + * ControlType varint discriminator written as the first datum on every + * client-initiated bidi stream. Selects which message body the peer + * should expect to read next. + * + * Source: `rs/moq-lite/src/lite/stream.rs:7-15`. + */ +enum class MoqLiteControlType( + val code: Long, +) { + /** Lite-01/02 only — unused on Lite-03. Reserved here for completeness. */ + Session(0L), + Announce(1L), + Subscribe(2L), + Fetch(3L), + Probe(4L), + + /** + * Graceful relay-shutdown signal. moq-rs's `Publisher::run` accepts + * `ControlType::Goaway = 5` (`rs/moq-lite/src/lite/publisher.rs`) + * to migrate a publisher to a different relay node. We don't act + * on it today — recognising the type code prevents + * [MoqLiteSession.handleInboundBidi] from silently FINing the + * bidi as an unknown control type, which would lose the relay's + * shutdown notification. Wire body decoding is left as a follow-up. + */ + Goaway(5L), + ; + + companion object { + private val byCode = entries.associateBy { it.code } + + fun fromCode(code: Long): MoqLiteControlType? = byCode[code] + } +} + +/** + * DataType varint written as the first byte of every uni stream that + * carries payload. Lite-03 currently uses `Group=0` only. + * + * Source: `rs/moq-lite/src/lite/stream.rs:32-36`. + */ +enum class MoqLiteDataType( + val code: Long, +) { + Group(0L), + ; + + companion object { + private val byCode = entries.associateBy { it.code } + + fun fromCode(code: Long): MoqLiteDataType? = byCode[code] + } +} + +/** + * Status byte at the head of an [MoqLiteAnnounce] payload. `Active=1` + * is sent on first publish; `Ended=0` on explicit unannounce. Disconnect + * is NOT signalled with `Ended` — the bidi just closes. + * + * Source: `rs/moq-lite/src/lite/announce.rs:84-90`. + */ +enum class MoqLiteAnnounceStatus( + val code: Int, +) { + Ended(0), + Active(1), + ; + + companion object { + fun fromCode(code: Int): MoqLiteAnnounceStatus? = + when (code) { + 0 -> Ended + 1 -> Active + else -> null + } + } +} + +/** + * Type discriminator at the head of a SubscribeResponse on the response + * side of a Subscribe bidi. Wire layout (Lite-03+): + * + * type varint (0 = Ok, 1 = Drop) + * body size-prefixed bytes + * + * The type sits OUTSIDE the body size prefix — see + * `rs/moq-lite/src/lite/subscribe.rs::SubscribeResponse::encode` (the + * `_` arm covers Lite03+). Earlier drafts wrapped type+body in one + * outer size prefix; Lite03 split them. + */ +enum class MoqLiteSubscribeResponseType( + val code: Long, +) { + Ok(0L), + Drop(1L), + ; + + companion object { + private val byCode = entries.associateBy { it.code } + + fun fromCode(code: Long): MoqLiteSubscribeResponseType? = byCode[code] + } +} diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteMessages.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteMessages.kt index 66e75df97..48ff2f55d 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteMessages.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteMessages.kt @@ -20,147 +20,13 @@ */ package com.vitorpamplona.nestsclient.moq.lite -/** - * moq-lite ALPN strings. ONLY [LITE_03] is wire-compatible with the - * codec in [MoqLiteCodec]; the other constants are kept for - * documentation and future codec upgrades. - * - * Subscribe / Group framing is identical across Lite-03 and Lite-04, - * but Lite-04 reshapes Announce.hops into an `OriginList` - * (`kixelated/moq` commit 45db108, "moq-lite/moq-relay: hop-based - * clustering"), adds an `exclude_hop` field to AnnounceInterest, and - * adds an `rtt` field to Probe. A relay that picks Lite-04 with our - * Lite-03 codec on the wire desyncs on the first Announce exchange. - * Don't add [LITE_04] to `wt-available-protocols` until - * [MoqLiteCodec] is version-aware and [MoqLiteAnnounce.hops] is a - * list rather than a single varint. - * - * `"moql"` is the legacy combined ALPN that requires an in-band SETUP - * exchange — kept here for completeness; not advertised by the - * factory. - * - * Source: `kixelated/moq/rs/moq-lite/src/lite/version.rs`, - * `kixelated/moq/rs/moq-lite/src/lite/announce.rs:11-105`, - * `kixelated/moq/rs/moq-lite/src/lite/probe.rs:9-55`. - */ -object MoqLiteAlpn { - const val LITE_03: String = "moq-lite-03" - - /** - * `moq-lite-04` ALPN string. Wire-incompatible with [MoqLiteCodec] - * today — see the object kdoc for the codec diff. Defined here so - * a future patch that lands version-aware Announce / Probe codecs - * can drop it into the [QuicWebTransportFactory] sub-protocol list - * without re-deriving the constant. - */ - const val LITE_04: String = "moq-lite-04" - const val LEGACY: String = "moql" -} - -/** - * ControlType varint discriminator written as the first datum on every - * client-initiated bidi stream. Selects which message body the peer - * should expect to read next. - * - * Source: `rs/moq-lite/src/lite/stream.rs:7-15`. - */ -enum class MoqLiteControlType( - val code: Long, -) { - /** Lite-01/02 only — unused on Lite-03. Reserved here for completeness. */ - Session(0L), - Announce(1L), - Subscribe(2L), - Fetch(3L), - Probe(4L), - - /** - * Graceful relay-shutdown signal. moq-rs's `Publisher::run` accepts - * `ControlType::Goaway = 5` (`rs/moq-lite/src/lite/publisher.rs`) - * to migrate a publisher to a different relay node. We don't act - * on it today — recognising the type code prevents - * [MoqLiteSession.handleInboundBidi] from silently FINing the - * bidi as an unknown control type, which would lose the relay's - * shutdown notification. Wire body decoding is left as a follow-up. - */ - Goaway(5L), - ; - - companion object { - private val byCode = entries.associateBy { it.code } - - fun fromCode(code: Long): MoqLiteControlType? = byCode[code] - } -} - -/** - * DataType varint written as the first byte of every uni stream that - * carries payload. Lite-03 currently uses `Group=0` only. - * - * Source: `rs/moq-lite/src/lite/stream.rs:32-36`. - */ -enum class MoqLiteDataType( - val code: Long, -) { - Group(0L), - ; - - companion object { - private val byCode = entries.associateBy { it.code } - - fun fromCode(code: Long): MoqLiteDataType? = byCode[code] - } -} - -/** - * Status byte at the head of an [MoqLiteAnnounce] payload. `Active=1` - * is sent on first publish; `Ended=0` on explicit unannounce. Disconnect - * is NOT signalled with `Ended` — the bidi just closes. - * - * Source: `rs/moq-lite/src/lite/announce.rs:84-90`. - */ -enum class MoqLiteAnnounceStatus( - val code: Int, -) { - Ended(0), - Active(1), - ; - - companion object { - fun fromCode(code: Int): MoqLiteAnnounceStatus? = - when (code) { - 0 -> Ended - 1 -> Active - else -> null - } - } -} - -/** - * Type discriminator at the head of a SubscribeResponse on the response - * side of a Subscribe bidi. Wire layout (Lite-03+): - * - * type varint (0 = Ok, 1 = Drop) - * body size-prefixed bytes - * - * The type sits OUTSIDE the body size prefix — see - * `rs/moq-lite/src/lite/subscribe.rs::SubscribeResponse::encode` (the - * `_` arm covers Lite03+). Earlier drafts wrapped type+body in one - * outer size prefix; Lite03 split them. - */ -enum class MoqLiteSubscribeResponseType( - val code: Long, -) { - Ok(0L), - Drop(1L), - ; - - companion object { - private val byCode = entries.associateBy { it.code } - - fun fromCode(code: Long): MoqLiteSubscribeResponseType? = byCode[code] - } -} +// Wire-format payload data classes for moq-lite messages. +// +// The discriminator enums (which message-body to expect at the top of +// a bidi / uni stream) live in `MoqLiteControlCodes.kt`; the ALPN +// negotiation strings live in `MoqLiteAlpn.kt`. This file holds only +// the body shapes the codec encodes / decodes once a discriminator +// has selected one. /** * "I'm interested in broadcasts under this prefix" — the first message From 6f649e76d8c2c6d9c1433b9ecccb546d8d9f16d2 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 19:47:08 +0000 Subject: [PATCH 39/41] refactor(nests): extract MoqLiteBroadcastHandle + HotSwappablePublisherSource (Audit-10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `MoqLiteNestsSpeaker.kt` mixed three top-level concerns at 387 lines: - The `MoqLiteNestsSpeaker` class itself — the speaker entry point that builds a publisher + broadcaster pair on `startBroadcasting`. - `MoqLiteBroadcastHandle` — internal `BroadcastHandle` implementation tying the broadcaster, audio publisher, and catalog publisher together with a fixed-order shutdown. - `HotSwappablePublisherSource` — internal interface that lets `ReconnectingNestsSpeaker.runHotSwapIteration` retarget a long-lived broadcaster onto fresh moq-lite session publishers without restarting the AudioRecord / Opus encoder pipeline. The handle and the interface are independently reachable from `ReconnectingNestsSpeaker` and have no behavioural coupling to `MoqLiteNestsSpeaker` beyond a `parent` reference (handle) or an `as?` cast (interface). Move each to its own file: - `MoqLiteBroadcastHandle.kt` (109 lines). - `HotSwappablePublisherSource.kt` (62 lines). `MoqLiteNestsSpeaker.kt` is now 276 lines focused on the speaker class. Same package, same `internal` visibility — no call-site changes needed elsewhere. Tests + spotless green. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../HotSwappablePublisherSource.kt | 72 ++++++++++++ .../nestsclient/MoqLiteBroadcastHandle.kt | 104 +++++++++++++++++ .../nestsclient/MoqLiteNestsSpeaker.kt | 110 ------------------ 3 files changed, 176 insertions(+), 110 deletions(-) create mode 100644 nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/HotSwappablePublisherSource.kt create mode 100644 nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteBroadcastHandle.kt diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/HotSwappablePublisherSource.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/HotSwappablePublisherSource.kt new file mode 100644 index 000000000..1a895d238 --- /dev/null +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/HotSwappablePublisherSource.kt @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.nestsclient + +import com.vitorpamplona.nestsclient.moq.lite.MoqLitePublisherHandle + +/** + * Internal hot-swap seam: speakers that expose this interface let the + * reconnect wrapper retarget a long-lived + * [com.vitorpamplona.nestsclient.audio.NestMoqLiteBroadcaster] onto a + * freshly-opened moq-lite session's publisher without restarting the + * AudioRecord / Opus encoder pipeline. Implemented by + * [MoqLiteNestsSpeaker]; not implemented by the IETF reference + * [DefaultNestsSpeaker], which falls back to the close-then-restart path + * inside [com.vitorpamplona.nestsclient.connectReconnectingNestsSpeaker]. + * + * The wrapper uses an `as?` cast to detect support so this interface + * can stay package-internal — protocol consumers never see it. + */ +internal interface HotSwappablePublisherSource { + /** + * Open a fresh [MoqLitePublisherHandle] on the underlying moq-lite + * session. Caller owns the returned handle's lifetime (typically + * via [com.vitorpamplona.nestsclient.audio.NestMoqLiteBroadcaster.swapPublisher]'s + * close-the-old contract). + * + * @param startSequence first group sequence the new publisher will + * assign. Used by the hot-swap path to seed the new session's + * audio track with the previous session's + * [MoqLitePublisherHandle.nextSequence] so kixelated/hang's + * `Container.Consumer.#run` doesn't drop every post-recycle + * group as `sequence < #active`. Pass `0L` for fresh (non- + * continuation) publishers — the catalog track is one such + * case, since its `#active` semantics are different from audio. + */ + suspend fun openPublisherForHotSwap( + track: String, + startSequence: Long = 0L, + ): MoqLitePublisherHandle + + /** + * Surface a broadcast-pipeline terminal failure (e.g. sustained + * `publisher.send` errors past + * [com.vitorpamplona.nestsclient.audio.NestMoqLiteBroadcaster.MAX_CONSECUTIVE_SEND_ERRORS]) + * by flipping the speaker's state to [NestsSpeakerState.Failed]. + * Called by the hot-swap pump when the long-lived broadcaster's + * `onTerminalFailure` fires; lets the reconnect orchestrator + * observe the terminal state and recycle the session, matching + * the legacy + * [MoqLiteNestsSpeaker.startBroadcasting] path's failure + * propagation. + */ + fun reportBroadcastTerminalFailure() +} diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteBroadcastHandle.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteBroadcastHandle.kt new file mode 100644 index 000000000..bd8635ac6 --- /dev/null +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteBroadcastHandle.kt @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.nestsclient + +import com.vitorpamplona.nestsclient.audio.NestMoqLiteBroadcaster +import com.vitorpamplona.nestsclient.moq.lite.MoqLitePublisherHandle + +/** + * [BroadcastHandle] returned by [MoqLiteNestsSpeaker.startBroadcasting] + * for the non-reconnecting code path. Wraps: + * + * - the long-lived audio [broadcaster] (mic + encoder + send loop), + * - the audio-track [publisher] on the moq-lite session, + * - the [catalogPublisher] companion track that emits the + * `catalog.json` manifest on every new SUBSCRIBE. + * + * Mute is forwarded to the broadcaster (which also reports the user- + * facing state back via [parent.reportMuteState]). [close] tears down + * all three components in a fixed order — broadcaster → audio + * publisher → catalog publisher — so a `CancellationException` + * mid-shutdown still releases every resource before re-throwing. + * + * Reconnecting / hot-swap callers go through `ReissuingBroadcastHandle` + * in `ReconnectingNestsSpeaker` instead, which manages the audio + + * catalog publishers across session swaps. + */ +internal class MoqLiteBroadcastHandle( + private val broadcaster: NestMoqLiteBroadcaster, + private val publisher: MoqLitePublisherHandle, + private val catalogPublisher: MoqLitePublisherHandle, + private val parent: MoqLiteNestsSpeaker, +) : BroadcastHandle { + @Volatile private var muted: Boolean = false + + @Volatile private var closed: Boolean = false + + override val isMuted: Boolean get() = muted + + override suspend fun setMuted(muted: Boolean) { + if (closed) return + this.muted = muted + broadcaster.setMuted(muted) + parent.reportMuteState(muted) + } + + override suspend fun close() { + if (closed) return + closed = true + // Stop the broadcaster first so the audio capture + encoder + // don't keep producing into a closing publisher. + try { + broadcaster.stop() + } catch (ce: kotlinx.coroutines.CancellationException) { + // Even on cancel, run the rest of cleanup before rethrowing + // — broadcaster.stop already cancels its own job, so the + // mic + encoder + publisher are owed their close paths. + runCatching { catalogPublisher.close() } + runCatching { publisher.close() } + parent.broadcastClosed(this) + throw ce + } catch (_: Throwable) { + // Best-effort; fall through to the defensive publisher.close. + } + // broadcaster.stop() already calls publisher.close(); call again + // defensively to make this method idempotent against partial + // failures on the broadcaster.stop path. + try { + publisher.close() + } catch (ce: kotlinx.coroutines.CancellationException) { + runCatching { catalogPublisher.close() } + parent.broadcastClosed(this) + throw ce + } catch (_: Throwable) { + // Best-effort. + } + try { + catalogPublisher.close() + } catch (ce: kotlinx.coroutines.CancellationException) { + parent.broadcastClosed(this) + throw ce + } catch (_: Throwable) { + // Best-effort. + } + parent.broadcastClosed(this) + } +} diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt index de09a3424..0d02d1051 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/MoqLiteNestsSpeaker.kt @@ -274,113 +274,3 @@ class MoqLiteNestsSpeaker internal constructor( } } } - -internal class MoqLiteBroadcastHandle( - private val broadcaster: NestMoqLiteBroadcaster, - private val publisher: MoqLitePublisherHandle, - private val catalogPublisher: MoqLitePublisherHandle, - private val parent: MoqLiteNestsSpeaker, -) : BroadcastHandle { - @Volatile private var muted: Boolean = false - - @Volatile private var closed: Boolean = false - - override val isMuted: Boolean get() = muted - - override suspend fun setMuted(muted: Boolean) { - if (closed) return - this.muted = muted - broadcaster.setMuted(muted) - parent.reportMuteState(muted) - } - - override suspend fun close() { - if (closed) return - closed = true - // Stop the broadcaster first so the audio capture + encoder - // don't keep producing into a closing publisher. - try { - broadcaster.stop() - } catch (ce: kotlinx.coroutines.CancellationException) { - // Even on cancel, run the rest of cleanup before rethrowing - // — broadcaster.stop already cancels its own job, so the - // mic + encoder + publisher are owed their close paths. - runCatching { catalogPublisher.close() } - runCatching { publisher.close() } - parent.broadcastClosed(this) - throw ce - } catch (_: Throwable) { - // Best-effort; fall through to the defensive publisher.close. - } - // broadcaster.stop() already calls publisher.close(); call again - // defensively to make this method idempotent against partial - // failures on the broadcaster.stop path. - try { - publisher.close() - } catch (ce: kotlinx.coroutines.CancellationException) { - runCatching { catalogPublisher.close() } - parent.broadcastClosed(this) - throw ce - } catch (_: Throwable) { - // Best-effort. - } - try { - catalogPublisher.close() - } catch (ce: kotlinx.coroutines.CancellationException) { - parent.broadcastClosed(this) - throw ce - } catch (_: Throwable) { - // Best-effort. - } - parent.broadcastClosed(this) - } -} - -/** - * Internal hot-swap seam: speakers that expose this interface let the - * reconnect wrapper retarget a long-lived - * [com.vitorpamplona.nestsclient.audio.NestMoqLiteBroadcaster] onto a - * freshly-opened moq-lite session's publisher without restarting the - * AudioRecord / Opus encoder pipeline. Implemented by - * [MoqLiteNestsSpeaker]; not implemented by the IETF reference - * [DefaultNestsSpeaker], which falls back to the close-then-restart path - * inside [com.vitorpamplona.nestsclient.connectReconnectingNestsSpeaker]. - * - * The wrapper uses an `as?` cast to detect support so this interface - * can stay package-internal — protocol consumers never see it. - */ -internal interface HotSwappablePublisherSource { - /** - * Open a fresh [MoqLitePublisherHandle] on the underlying moq-lite - * session. Caller owns the returned handle's lifetime (typically - * via [com.vitorpamplona.nestsclient.audio.NestMoqLiteBroadcaster.swapPublisher]'s - * close-the-old contract). - * - * @param startSequence first group sequence the new publisher will - * assign. Used by the hot-swap path to seed the new session's - * audio track with the previous session's - * [MoqLitePublisherHandle.nextSequence] so kixelated/hang's - * `Container.Consumer.#run` doesn't drop every post-recycle - * group as `sequence < #active`. Pass `0L` for fresh (non- - * continuation) publishers — the catalog track is one such - * case, since its `#active` semantics are different from audio. - */ - suspend fun openPublisherForHotSwap( - track: String, - startSequence: Long = 0L, - ): MoqLitePublisherHandle - - /** - * Surface a broadcast-pipeline terminal failure (e.g. sustained - * `publisher.send` errors past - * [com.vitorpamplona.nestsclient.audio.NestMoqLiteBroadcaster.MAX_CONSECUTIVE_SEND_ERRORS]) - * by flipping the speaker's state to [NestsSpeakerState.Failed]. - * Called by the hot-swap pump when the long-lived broadcaster's - * `onTerminalFailure` fires; lets the reconnect orchestrator - * observe the terminal state and recycle the session, matching - * the legacy - * [MoqLiteNestsSpeaker.startBroadcasting] path's failure - * propagation. - */ - fun reportBroadcastTerminalFailure() -} From e9d19e5de45738799477e7bd411c74aca7c77187 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 19:50:24 +0000 Subject: [PATCH 40/41] refactor(nests): extract MoqLiteFrame + handles + publisher interface (Audit-8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `MoqLiteSession.kt` was 1526 lines mixing the session state machine (announce pump, subscribe pump, group-stream demux, publisher state) with the public types its API hands back to consumers. The session- internal `PublisherStateImpl` inner class is tightly coupled to the session's outer scope (state lock, transport, scope.launch, openGroupStream) and is left in place — extracting it is a separate refactor with its own design choices. Extract the standalone caller-facing types: - `MoqLiteFrame.kt` — the `MoqLiteFrame` data class with its custom `equals` / `hashCode` for ByteArray content equality. - `MoqLiteHandles.kt` — `MoqLiteSubscribeHandle`, `MoqLiteAnnouncesHandle`, and `MoqLiteSubscribeException`. All three are "what the caller gets back from a subscribe / announce" + "how protocol-level rejections surface." - `MoqLitePublisherHandle.kt` — the public publisher interface that the session's internal `PublisherStateImpl` implements. `internal constructor` on the handles keeps them un-instantiable outside the package. Same package, same visibility, no call-site changes needed. `MoqLiteSession.kt` shrinks from 1526 to 1372 lines, focused on session lifecycle + the inner `PublisherStateImpl`. Tests + spotless green. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- .../nestsclient/moq/lite/MoqLiteFrame.kt | 40 +++++ .../nestsclient/moq/lite/MoqLiteHandles.kt | 61 +++++++ .../moq/lite/MoqLitePublisherHandle.kt | 122 ++++++++++++++ .../nestsclient/moq/lite/MoqLiteSession.kt | 154 ------------------ 4 files changed, 223 insertions(+), 154 deletions(-) create mode 100644 nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteFrame.kt create mode 100644 nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHandles.kt create mode 100644 nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLitePublisherHandle.kt diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteFrame.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteFrame.kt new file mode 100644 index 000000000..73c827014 --- /dev/null +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteFrame.kt @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.nestsclient.moq.lite + +/** + * One frame received from a subscription. moq-lite's wire format + * carries no per-frame envelope beyond the size; [groupSequence] is + * pulled from the group header so consumers can detect group rollover + * (e.g. for keyframe boundaries). + */ +data class MoqLiteFrame( + val groupSequence: Long, + val payload: ByteArray, +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is MoqLiteFrame) return false + return groupSequence == other.groupSequence && payload.contentEquals(other.payload) + } + + override fun hashCode(): Int = 31 * groupSequence.hashCode() + payload.contentHashCode() +} diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHandles.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHandles.kt new file mode 100644 index 000000000..9f69f1f00 --- /dev/null +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteHandles.kt @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.nestsclient.moq.lite + +import kotlinx.coroutines.flow.Flow + +// Caller-facing handles returned by MoqLiteSession.subscribe / .announce, +// plus the typed protocol-level rejection exception. Kept together +// because they share a single concern: "what the consumer gets back +// from a subscribe / announce request and how it surfaces failure." + +/** + * Active subscription handle returned by [MoqLiteSession.subscribe]. + * [frames] emits every frame the publisher pushes; [unsubscribe] + * FINs the bidi to signal "no longer interested" (moq-lite has no + * UNSUBSCRIBE message — FIN is the protocol). + */ +class MoqLiteSubscribeHandle internal constructor( + val id: Long, + val ok: MoqLiteSubscribeOk, + val frames: Flow, + private val unsubscribeAction: suspend () -> Unit, +) { + suspend fun unsubscribe() = unsubscribeAction() +} + +/** + * Active announce-discovery handle returned by [MoqLiteSession.announce]. + * [updates] emits every [MoqLiteAnnounce] update the relay streams + * back; [close] FINs the bidi to stop receiving updates. + */ +class MoqLiteAnnouncesHandle internal constructor( + val updates: Flow, + private val close: suspend () -> Unit, +) { + suspend fun close() = close.invoke() +} + +/** Thrown when subscribe is rejected (Drop) or the response stream dies. */ +class MoqLiteSubscribeException( + message: String, + cause: Throwable? = null, +) : RuntimeException(message, cause) diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLitePublisherHandle.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLitePublisherHandle.kt new file mode 100644 index 000000000..143917b78 --- /dev/null +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLitePublisherHandle.kt @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.nestsclient.moq.lite + +/** + * Active publisher handle returned by [MoqLiteSession.publish]. + * + * Lifecycle: + * 1. Call [startGroup] (or [send] which auto-starts a fresh group on + * first call) to begin pushing frames for one Opus group. + * 2. Call [send] for each frame (one Opus packet = one frame). + * 3. Call [endGroup] to FIN the current group's uni stream and start + * a fresh group on the next [send]. Group rollover is the + * publisher's call — typically every N seconds or every keyframe. + * 4. Call [close] when the broadcast ends — sends `Announce(Ended)` + * on every active announce bidi and FINs every group stream. + */ +interface MoqLitePublisherHandle { + /** + * The broadcast suffix this publisher claimed at [MoqLiteSession.publish]. + * Always normalised per [MoqLitePath]. + */ + val suffix: String + + /** + * The next group sequence number that will be assigned by [send] / + * [startGroup]. Snapshot-only — read AFTER the broadcaster has + * stopped sending into this publisher (typically just before the + * caller closes the publisher in a hot-swap), so the value is the + * highest-already-used sequence + 1. + * + * Used by [com.vitorpamplona.nestsclient.MoqLiteNestsSpeaker]'s + * hot-swap path to seed the new session's publisher with a + * monotonically-continuing sequence — without this, every JWT + * refresh restarts at sequence 0 and kixelated/hang's + * `Container.Consumer.#run` drops every group whose sequence is + * less than its current `#active` high-water mark, killing audio + * for the watcher until either `#active` rolls over or the + * watcher re-subscribes. + * + * `@Volatile` on the implementation; safe to read from any + * coroutine. The accept-tiny-race window between read and a + * concurrent `send` is closed in practice because the broadcaster + * is responsible for swapping its publisher reference BEFORE the + * caller reads this value (so no further sends land on this + * publisher). + */ + val nextSequence: Long + + /** + * Start a new group. Allocates a fresh sequence id and opens a new + * uni stream pre-loaded with `DataType=Group + GroupHeader`. Idempotent + * — calling [startGroup] when the previous group hasn't been ended + * is treated as an implicit [endGroup] then a new start. + */ + suspend fun startGroup() + + /** + * Push one [payload] (one Opus packet) as a `varint(size) + payload` + * frame on the current group's uni stream. Auto-starts a group if + * none is active. + * + * Returns false if no inbound subscriber is currently attached. + * Subscriber-less sends silently drop on the wire — the relay keeps + * the publisher's announce active either way, so unmute is + * sample-accurate. + */ + suspend fun send(payload: ByteArray): Boolean + + /** FIN the current group's uni stream. The next [send] starts a fresh group. */ + suspend fun endGroup() + + /** + * Register a callback that fires once each time a new inbound + * subscriber is registered against this publisher's track (i.e. + * each track-matching SUBSCRIBE bidi the relay opens to us). Used + * to push a "track-latest" payload — the canonical example is the + * broadcast catalog manifest, which a watcher needs to receive on + * subscribe but doesn't change between subscribers — without + * forcing the publisher to maintain a periodic re-emit loop. + * + * Called once per accepted SUBSCRIBE (track filter passed). Fires + * OUTSIDE the publisher's serialisation lock, so the hook can + * safely call [send] / [endGroup] without deadlocking. + * + * Caller MUST set the hook before any subscriber attaches (typically + * immediately after [com.vitorpamplona.nestsclient.moq.lite.MoqLiteSession.publish] + * returns) — there's no "fire-on-set for existing subscribers" + * replay. For the typical catalog use case the publisher is fresh + * when the hook is set, and the relay's SUBSCRIBE bidi takes a + * round-trip to arrive, so this is safe in practice. + * + * Pass `null` to clear the hook. Calling twice with non-null + * replaces the previous hook (no de-duplication). + */ + fun setOnNewSubscriber(hook: (suspend () -> Unit)?) + + /** + * Stop publishing. Sends `Announce(Ended)` on every active announce + * bidi, FINs the current group, and releases all per-publisher + * resources. Idempotent. + */ + suspend fun close() +} diff --git a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt index b5e49dc2c..3dc3825ed 100644 --- a/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt +++ b/nestsClient/src/commonMain/kotlin/com/vitorpamplona/nestsclient/moq/lite/MoqLiteSession.kt @@ -34,7 +34,6 @@ import kotlinx.coroutines.Job import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.channels.BufferOverflow import kotlinx.coroutines.channels.Channel -import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.flow.consumeAsFlow import kotlinx.coroutines.launch @@ -1371,156 +1370,3 @@ class MoqLiteSession internal constructor( ): MoqLiteSession = MoqLiteSession(transport, pumpScope) } } - -/** - * One frame received from a subscription. moq-lite's wire format - * carries no per-frame envelope beyond the size; [groupSequence] is - * pulled from the group header so consumers can detect group rollover - * (e.g. for keyframe boundaries). - */ -data class MoqLiteFrame( - val groupSequence: Long, - val payload: ByteArray, -) { - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (other !is MoqLiteFrame) return false - return groupSequence == other.groupSequence && payload.contentEquals(other.payload) - } - - override fun hashCode(): Int = 31 * groupSequence.hashCode() + payload.contentHashCode() -} - -/** - * Active subscription handle returned by [MoqLiteSession.subscribe]. - * [frames] emits every frame the publisher pushes; [unsubscribe] - * FINs the bidi to signal "no longer interested" (moq-lite has no - * UNSUBSCRIBE message — FIN is the protocol). - */ -class MoqLiteSubscribeHandle internal constructor( - val id: Long, - val ok: MoqLiteSubscribeOk, - val frames: Flow, - private val unsubscribeAction: suspend () -> Unit, -) { - suspend fun unsubscribe() = unsubscribeAction() -} - -/** - * Active announce-discovery handle returned by [MoqLiteSession.announce]. - * [updates] emits every [MoqLiteAnnounce] update the relay streams - * back; [close] FINs the bidi to stop receiving updates. - */ -class MoqLiteAnnouncesHandle internal constructor( - val updates: Flow, - private val close: suspend () -> Unit, -) { - suspend fun close() = close.invoke() -} - -/** Thrown when subscribe is rejected (Drop) or the response stream dies. */ -class MoqLiteSubscribeException( - message: String, - cause: Throwable? = null, -) : RuntimeException(message, cause) - -/** - * Active publisher handle returned by [MoqLiteSession.publish]. - * - * Lifecycle: - * 1. Call [startGroup] (or [send] which auto-starts a fresh group on - * first call) to begin pushing frames for one Opus group. - * 2. Call [send] for each frame (one Opus packet = one frame). - * 3. Call [endGroup] to FIN the current group's uni stream and start - * a fresh group on the next [send]. Group rollover is the - * publisher's call — typically every N seconds or every keyframe. - * 4. Call [close] when the broadcast ends — sends `Announce(Ended)` - * on every active announce bidi and FINs every group stream. - */ -interface MoqLitePublisherHandle { - /** - * The broadcast suffix this publisher claimed at [MoqLiteSession.publish]. - * Always normalised per [MoqLitePath]. - */ - val suffix: String - - /** - * The next group sequence number that will be assigned by [send] / - * [startGroup]. Snapshot-only — read AFTER the broadcaster has - * stopped sending into this publisher (typically just before the - * caller closes the publisher in a hot-swap), so the value is the - * highest-already-used sequence + 1. - * - * Used by [com.vitorpamplona.nestsclient.MoqLiteNestsSpeaker]'s - * hot-swap path to seed the new session's publisher with a - * monotonically-continuing sequence — without this, every JWT - * refresh restarts at sequence 0 and kixelated/hang's - * `Container.Consumer.#run` drops every group whose sequence is - * less than its current `#active` high-water mark, killing audio - * for the watcher until either `#active` rolls over or the - * watcher re-subscribes. - * - * `@Volatile` on the implementation; safe to read from any - * coroutine. The accept-tiny-race window between read and a - * concurrent `send` is closed in practice because the broadcaster - * is responsible for swapping its publisher reference BEFORE the - * caller reads this value (so no further sends land on this - * publisher). - */ - val nextSequence: Long - - /** - * Start a new group. Allocates a fresh sequence id and opens a new - * uni stream pre-loaded with `DataType=Group + GroupHeader`. Idempotent - * — calling [startGroup] when the previous group hasn't been ended - * is treated as an implicit [endGroup] then a new start. - */ - suspend fun startGroup() - - /** - * Push one [payload] (one Opus packet) as a `varint(size) + payload` - * frame on the current group's uni stream. Auto-starts a group if - * none is active. - * - * Returns false if no inbound subscriber is currently attached. - * Subscriber-less sends silently drop on the wire — the relay keeps - * the publisher's announce active either way, so unmute is - * sample-accurate. - */ - suspend fun send(payload: ByteArray): Boolean - - /** FIN the current group's uni stream. The next [send] starts a fresh group. */ - suspend fun endGroup() - - /** - * Register a callback that fires once each time a new inbound - * subscriber is registered against this publisher's track (i.e. - * each track-matching SUBSCRIBE bidi the relay opens to us). Used - * to push a "track-latest" payload — the canonical example is the - * broadcast catalog manifest, which a watcher needs to receive on - * subscribe but doesn't change between subscribers — without - * forcing the publisher to maintain a periodic re-emit loop. - * - * Called once per accepted SUBSCRIBE (track filter passed). Fires - * OUTSIDE the publisher's serialisation lock, so the hook can - * safely call [send] / [endGroup] without deadlocking. - * - * Caller MUST set the hook before any subscriber attaches (typically - * immediately after [com.vitorpamplona.nestsclient.moq.lite.MoqLiteSession.publish] - * returns) — there's no "fire-on-set for existing subscribers" - * replay. For the typical catalog use case the publisher is fresh - * when the hook is set, and the relay's SUBSCRIBE bidi takes a - * round-trip to arrive, so this is safe in practice. - * - * Pass `null` to clear the hook. Calling twice with non-null - * replaces the previous hook (no de-duplication). - */ - fun setOnNewSubscriber(hook: (suspend () -> Unit)?) - - /** - * Stop publishing. Sends `Announce(Ended)` on every active announce - * bidi, FINs the current group, and releases all per-publisher - * resources. Idempotent. - */ - suspend fun close() -} From 28358b414154ddc61a82455c7ebb1949421f240e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 20:04:56 +0000 Subject: [PATCH 41/41] refactor(nests): extract ActiveSubscription + plan deferred manager refactor (Audit-9, Audit-14) Audit-9: NestViewModel.kt is 2112 lines and growing, ~1000 of which are subscription-lifecycle state machine concerns intertwined with the room-level public API. Pulling out the full `NestSubscriptionManager` is a multi-week refactor with subtle coupling (catalog readiness affects spinner state, mute has effective + per-speaker flavours, expiry jobs need the parent scope) and warrants its own focused review pass. For now, take the small tractable subset: - Extract `ActiveSubscription` from a `private inner class` in NestViewModel to its own file as `internal class ActiveSubscription` in the same package. The class is purely state-holding (handle, roomPlayer, player, isPlaying); zero VM coupling beyond the slot map's value type. Same visibility for NestViewModel callers; one less private helper class buried 1500 lines into NestViewModel.kt. - File `commons/plans/2026-05-06-nest-subscription-manager-extraction.md` documents the deferred full extraction: target shape, state that moves, methods that move, what stays in VM, why deferred, when to land. Picks up the next person who opens NestViewModel.kt rather than leaving them to re-derive the rationale. Audit-14: T11.3 (stream priority for moq-lite group uni streams) was deferred from the T11 commit (drop bestEffort=true) because the :quic-side change touches the writer's hot path and warrants its own review pass. New file `nestsClient/plans/2026-05-06-stream-priority-followup.md` spells out: - Why: bestEffort=true was incidentally biasing drain order toward newer groups; without it, the writer's round-robin order can serve a stale group when a fresh one is more useful. - Target shape: `QuicStream.priority` field, sortedByDescending in the writer's send-frame loop, `WebTransportWriteStream.setPriority` pass-through, `MoqLiteSession.openGroupStream` calls setPriority(sequence). With code sketches. - Test: pin iteration order via the writer's emitted-frames tape. - Risk profile: starvation, perf cost of per-pass sort, compat. - When to land: after interop verification stabilises. No code changes in this commit beyond the ActiveSubscription move. https://claude.ai/code/session_014JfZJHSTvyYYWJbC9VbB47 --- ...06-nest-subscription-manager-extraction.md | 109 +++++++++++++ .../commons/viewmodels/ActiveSubscription.kt | 91 +++++++++++ .../commons/viewmodels/NestViewModel.kt | 45 +----- .../2026-05-06-stream-priority-followup.md | 151 ++++++++++++++++++ 4 files changed, 353 insertions(+), 43 deletions(-) create mode 100644 commons/plans/2026-05-06-nest-subscription-manager-extraction.md create mode 100644 commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/ActiveSubscription.kt create mode 100644 nestsClient/plans/2026-05-06-stream-priority-followup.md diff --git a/commons/plans/2026-05-06-nest-subscription-manager-extraction.md b/commons/plans/2026-05-06-nest-subscription-manager-extraction.md new file mode 100644 index 000000000..61883b034 --- /dev/null +++ b/commons/plans/2026-05-06-nest-subscription-manager-extraction.md @@ -0,0 +1,109 @@ +# Extract `NestSubscriptionManager` from `NestViewModel` + +**Status**: deferred — flagged in the audit pass, not landed yet. + +## Why + +`NestViewModel.kt` is 2112 lines and growing. ~1000 of those lines are +the per-speaker subscription-lifecycle state machine: open / close / +reconcile / catalog-fetch / decoder-await / level-tap / mute-routing / +hush. The rest is room-level public API (connect, disconnect, +broadcast, presence, reactions, focus / network observers, UI state). + +These two concerns are coupled by shared mutable state but do not +share a *responsibility*. The audit-9 finding flagged it as the +single biggest SRP breach in the touched code. + +We extracted `ActiveSubscription` into its own file (`ActiveSubscription.kt`) +in commit `` as a stepping stone — the deferred extraction +proper is the orchestration class. + +## Target shape + +```kotlin +internal class NestSubscriptionManager( + private val viewModelScope: CoroutineScope, + private val signer: NostrSigner, + private val decoderFactory: (channelCount: Int, sampleRate: Int) -> OpusDecoder, + private val playerFactory: (channelCount: Int, sampleRate: Int) -> AudioPlayer, + private val onActiveSpeakersChanged: (Set) -> Unit, + private val onSpeakerActivity: (String) -> Unit, + private val onAudioLevel: (String, Float) -> Unit, + private val onConnectingSpeakerChanged: (pubkey: String, connecting: Boolean) -> Unit, + private val effectiveListenMuted: () -> Boolean, + private val locallyHushed: () -> Set, +) { + val speakerCatalogs: StateFlow> + val audioLevels: StateFlow> + + fun bind(listener: NestsListener) + fun unbind() // cancel everything; release native resources + fun updateSpeakers(requested: Set) + fun applyEffectiveMute() + fun setLocalHushed(pubkey: String, hushed: Boolean) +} +``` + +## State that moves + +From `NestViewModel`: +- `activeSubscriptions: MutableMap` +- `catalogJobs: MutableMap` +- `_speakerCatalogs: MutableStateFlow>` +- `requestedSpeakers: Set` +- `speakingExpiryJobs: MutableMap` (for `onSpeakerActivity` debounce) +- `_audioLevels` if separable + +## Methods that move + +- `reconcileSubscriptions` +- `openSubscription` (~150 lines) +- `closeSubscription` +- `fetchSpeakerCatalog` +- `awaitAudioPipelineConfig` +- `onSpeakerActivity` debounce timer +- `onAudioLevel` coalescing +- `applyEffectiveListenMute`'s subscription-touching half +- `setLocalHushed`'s player.setVolume half +- `publishActiveSpeakers` + +## What stays in `NestViewModel` + +- Public lifecycle (`connect`, `disconnect`, `onCleared`) +- Connection state machine (`ConnectionUiState`) +- Broadcast state (`broadcast` / `_uiState.broadcast`) +- Presence aggregation (kind 10312 events) +- Reactions +- Focus / network observers +- The `NestUiState` composition itself + +`NestViewModel` calls `manager.bind(listener)` on each fresh +listener and `manager.unbind()` on disconnect / cleanup. +`updateSpeakers` and mute / hush propagate through to the manager. + +## Why deferred + +- 1000-line code move across ~10 methods + 5 state fields +- Subtle coupling: catalog readiness affects spinner state, mute + state has effective + per-speaker flavours, expiry jobs need + parent scope, `closed` flag is currently a single VM-wide flag +- Test surface: `NestViewModelTest` exercises subscription paths + through the VM today; would need to either keep that surface or + add a `NestSubscriptionManagerTest`. + +The extraction has a clean contract (callbacks for the bits that +remain VM-side) but landing it without behavioural drift wants a +focused review pass plus its own dedicated test rebuild — bigger +than fits in the current audit-pass. + +## When to land + +After: +- The catalog-driven decoder reconfig (T3) has run in production + for long enough that the `awaitAudioPipelineConfig` pattern is + proven against real publishers. +- Any pending subscription-lifecycle bug fixes (e.g. the audit's + earlier "boundary-rebuild dangling decoder" path) are settled — + moving them mid-fix risks introducing regressions. + +Track as a follow-up issue rather than a near-term must-do. diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/ActiveSubscription.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/ActiveSubscription.kt new file mode 100644 index 000000000..db0a0bae3 --- /dev/null +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/ActiveSubscription.kt @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.viewmodels + +import com.vitorpamplona.nestsclient.audio.AudioPlayer +import com.vitorpamplona.nestsclient.audio.NestPlayer +import com.vitorpamplona.nestsclient.moq.SubscribeHandle + +/** + * Per-pubkey state slot held in [NestViewModel.activeSubscriptions]. + * + * Three lifecycle phases: + * - **Pending**: just-reserved by `reconcileSubscriptions` to dedupe + * concurrent reconciles. No handle / player attached yet. Constructor + * via [pending]. + * - **Active**: [attach] wires the moq subscribe handle, the + * [NestPlayer] decode loop, and the device player. [isPlaying] + * flips true. + * - **Detached**: [detach] returns the handle + roomPlayer pair so the + * caller can run the suspending teardown (`NestPlayer.stop()` + + * `SubscribeHandle.unsubscribe()`) in its own coroutine scope — + * keeps the native MediaCodec / AudioTrack release ordered after + * the decode loop has unwound (audit MoQ #11/#12). + * + * `internal` because only [NestViewModel]'s subscription-lifecycle + * paths (open / close / reconcile / mute / hush) construct or mutate + * these. Lifted to a top-level type rather than a nested class so the + * file split tracks concerns: this class is "subscription state + * machine"; the rest of NestViewModel is "ViewModel public surface + + * orchestration." + */ +internal class ActiveSubscription private constructor( + val pubkey: String, +) { + private var handle: SubscribeHandle? = null + private var roomPlayer: NestPlayer? = null + var player: AudioPlayer? = null + private set + var isPlaying: Boolean = false + private set + + fun attach( + handle: SubscribeHandle, + roomPlayer: NestPlayer, + player: AudioPlayer, + ) { + this.handle = handle + this.roomPlayer = roomPlayer + this.player = player + this.isPlaying = true + } + + /** + * Hand the player + handle back to the caller's coroutine scope — + * `NestPlayer.stop()` and `SubscribeHandle.unsubscribe()` are + * both suspend, and the caller has the right scope to await them + * (so native MediaCodec/AudioTrack release runs after the decode + * loop has unwound, per audit MoQ #11/#12). + */ + fun detach(): Pair { + isPlaying = false + val p = roomPlayer + val h = handle + roomPlayer = null + handle = null + player = null + return p to h + } + + companion object { + fun pending(pubkey: String) = ActiveSubscription(pubkey) + } +} diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt index c688ed863..5c85b2735 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/viewmodels/NestViewModel.kt @@ -42,7 +42,6 @@ import com.vitorpamplona.nestsclient.audio.OpusDecoder import com.vitorpamplona.nestsclient.audio.OpusEncoder import com.vitorpamplona.nestsclient.connectReconnectingNestsListener import com.vitorpamplona.nestsclient.connectReconnectingNestsSpeaker -import com.vitorpamplona.nestsclient.moq.SubscribeHandle import com.vitorpamplona.nestsclient.transport.WebTransportFactory import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner import com.vitorpamplona.quartz.nip53LiveActivities.chat.LiveActivitiesChatMessageEvent @@ -1672,48 +1671,8 @@ class NestViewModel( } } - private class ActiveSubscription private constructor( - val pubkey: String, - ) { - private var handle: SubscribeHandle? = null - private var roomPlayer: NestPlayer? = null - var player: AudioPlayer? = null - private set - var isPlaying: Boolean = false - private set - - fun attach( - handle: SubscribeHandle, - roomPlayer: NestPlayer, - player: AudioPlayer, - ) { - this.handle = handle - this.roomPlayer = roomPlayer - this.player = player - this.isPlaying = true - } - - /** - * Hand the player + handle back to the caller's coroutine scope — - * `NestPlayer.stop()` and `SubscribeHandle.unsubscribe()` are - * both suspend, and the caller has the right scope to await them - * (so native MediaCodec/AudioTrack release runs after the decode - * loop has unwound, per audit MoQ #11/#12). - */ - fun detach(): Pair { - isPlaying = false - val p = roomPlayer - val h = handle - roomPlayer = null - handle = null - player = null - return p to h - } - - companion object { - fun pending(pubkey: String) = ActiveSubscription(pubkey) - } - } + // ActiveSubscription was extracted to its own file in the same + // package — see [ActiveSubscription.kt]. // Platform-specific Factory lives in `amethyst/.../nests/room/`, // not commonMain — the lifecycle KMP `ViewModelProvider.Factory` diff --git a/nestsClient/plans/2026-05-06-stream-priority-followup.md b/nestsClient/plans/2026-05-06-stream-priority-followup.md new file mode 100644 index 000000000..6e9578214 --- /dev/null +++ b/nestsClient/plans/2026-05-06-stream-priority-followup.md @@ -0,0 +1,151 @@ +# Stream priority for moq-lite group uni streams (T11.3 follow-up) + +**Status**: deferred — flagged in T11 (drop `bestEffort=true`), not landed. + +## Why + +`T11` removed `bestEffort=true` from `MoqLiteSession.openGroupStream` +because the QUIC contract it relied on (drop lost ranges silently +without `RESET_STREAM`) created undeliverable streams that wedge +peer reassembly buffers — the actual user-visible bug was a 30 s +silent dropout per lost packet on lossy networks. + +`bestEffort=true` was incidentally providing a "newer-groups-skip- +queued-retransmits" effect: the writer never queued retransmits in +the first place, so under congestion the loss budget naturally +biased toward dropping older lost ranges. With `bestEffort=true` +gone, all retransmits are now queued, and under sustained +congestion the writer drains streams in `streamRoundRobinStart` +order — which can mean the listener catches up on a stale group +when a fresh one would be more useful. + +The kixelated reference (`moq-rs`'s `Publisher::serve_group` in +`rs/moq-lite/src/lite/publisher.rs:347-406`) addresses this by +calling `stream.set_priority(priority.current())` on every group +stream and biasing the writer toward higher-priority (newer) +streams. We should do the same. + +## Target shape + +### `:quic` API + +Add a stable priority hook to `QuicStream`: + +```kotlin +class QuicStream(...) { + @Volatile + var priority: Int = 0 + // Higher drains first under contention. Default 0 = unchanged + // round-robin behaviour. +} +``` + +Expose at the WebTransport layer: + +```kotlin +interface WebTransportWriteStream { + fun setPriority(priority: Int) +} +``` + +### `QuicConnectionWriter` — the load-bearing change + +Today's send-frame loop (`QuicConnectionWriter.kt:411-416`): + +```kotlin +val streamsView = conn.streamsListLocked() +val start = conn.streamRoundRobinStart % streamsView.size +for (i in streamsView.indices) { + val stream = streamsView[(start + i) % streamsView.size] + // ... +} +``` + +Replace with priority-then-round-robin: + +```kotlin +val streamsView = conn.streamsListLocked() +val sorted = streamsView.sortedByDescending { it.priority } +val start = conn.streamRoundRobinStart % sorted.size +for (i in sorted.indices) { + val stream = sorted[(start + i) % sorted.size] + // ... +} +``` + +Sort is stable; same-priority streams retain insertion order, so the +existing round-robin behaviour holds within a priority tier. Higher- +priority streams always come first in the iteration. + +Cost: O(N log N) per drain pass, where N is active local-initiated +streams. N is small (1–10 in the moq-lite audio path); the +allocation of `sorted` per pass is the only real cost. If that +shows up in profiling, switch to `kotlin.collections.IntArray`- +backed indirect sort or maintain a priority-sorted list incrementally +on `setPriority` calls. + +### moq-lite wiring + +`MoqLiteSession.openGroupStream:1022-1037`: + +```kotlin +internal suspend fun openGroupStream( + subscribeId: Long, + sequence: Long, +): WebTransportWriteStream { + val uni = transport.openUniStream() + uni.setPriority(sequence.coerceAtMost(Int.MAX_VALUE.toLong()).toInt()) + uni.write(Varint.encode(MoqLiteDataType.Group.code)) + uni.write(MoqLiteCodec.encodeGroupHeader(MoqLiteGroupHeader(subscribeId, sequence))) + return uni +} +``` + +Newer groups have higher sequence → higher priority → drain first +under congestion. The saturation conversion handles broadcasts that +run long enough for `sequence` to exceed `Int.MAX_VALUE` (≈ 71 +years at our 1 group/sec production cadence; defensive only). + +## Test + +Add a unit test in `:quic` that builds a `QuicConnection`, opens +two streams, sets `.priority = 0` on the first and `.priority = 1` +on the second, queues bytes on both, and verifies the higher- +priority stream's bytes hit the wire first. Doesn't need to fake +real flow-control backpressure — pinning the iteration order via +the writer's emitted-frames tape is sufficient to catch the +regression case ("a later refactor accidentally re-introduces +round-robin order"). + +A second test in `:nestsClient` that opens 3 group streams and +asserts later-sequence streams drain before earlier ones is +nice-to-have but secondary; the `:quic`-level test pins the load- +bearing invariant. + +## Why deferred + +The change is small in lines but touches the QUIC writer's hot +path. Rebasing a future `:quic` retransmit / pacing change onto a +priority-sorted iteration order is doable but the diff conflicts +get noisy. Better landed as a focused PR after the current +interop-work series stabilises. + +Risk profile: +- **Bugs**: subtle starvation risk if a high-priority stream + always has `streamRemaining > 0` — round-robin tiebreaker within + a priority tier mitigates this for same-priority case, but the + cross-tier case needs deliberate thought (do we want strict + priority or weighted?). +- **Performance**: per-pass sort allocation. Negligible for N≤10 + but worth measuring if N grows. +- **Compat**: streams without an explicit priority default to 0, + matching today's behaviour. Existing tests should pass unchanged. + +## When to land + +After the catalog interop series is fully verified (production +audio with no degradation under realistic conditions), and ideally +alongside a `:quic` perf review pass that touches the same code +path. Not blocking on any audio bug today — `T11`'s reliable- +delivery fix is the actual correctness change; this is the +spec-aligned hardening.